e2e: drive each client CLI through one long-lived batch process

The cross-language e2e matrix spawned one CLI process per operation —
~250 per client — paying a process (and, for the Java CLI, a full JVM)
cold-start every time. The Java leg alone ran ~16 minutes.

Each client CLI (dotnet, go, rust, python, java) gains a `batch`
subcommand: a single process that reads one command line from stdin,
runs it through the normal subcommand dispatch, writes the JSON result,
then a line containing exactly `__MXGW_BATCH_EOR__`. A failing command
writes its `{"error":...}` envelope and the loop continues.

run-client-e2e-tests.ps1 now launches one batch process per client and
pings every operation through its stdin/stdout, so startup is paid once
per client. The orchestration and assertions are unchanged; the parity
and auth phases now read the `{"error":...}` envelope instead of a
process exit code.

Full 5-client matrix with -VerifyWrite: ~15 min, down from ~35; the Java
leg dropped from ~16 min to ~2-3.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Joseph Doherty
2026-05-21 06:20:13 -04:00
parent c1ff8c94e8
commit 6126099cdb
10 changed files with 970 additions and 47 deletions
+142 -1
View File
@@ -1,13 +1,15 @@
"""Tests for the Python CLI."""
import io
import json
from typing import Any
import click
import pytest
from click.testing import CliRunner
from mxgateway import __version__
from mxgateway_cli.commands import _use_plaintext, main
from mxgateway_cli.commands import _BATCH_EOR, _use_plaintext, main
def test_version_json_is_deterministic() -> None:
@@ -216,3 +218,142 @@ def test_cli_localhost_endpoint_with_plaintext_flag_uses_plaintext(
assert result.exit_code != 0
assert captured.get("plaintext") is True
# ---------------------------------------------------------------------------
# batch subcommand tests
# ---------------------------------------------------------------------------
def _run_batch(lines: list[str]) -> tuple[int, list[str]]:
"""Invoke ``batch`` with the given stdin lines; return (exit_code, stdout_lines)."""
runner = CliRunner()
stdin_text = "\n".join(lines) + "\n"
result = runner.invoke(main, ["batch"], input=stdin_text)
stdout_lines = result.output.splitlines()
return result.exit_code, stdout_lines
def _split_records(stdout_lines: list[str]) -> list[list[str]]:
"""Split stdout lines on ``__MXGW_BATCH_EOR__`` sentinels into per-command records."""
records: list[list[str]] = []
current: list[str] = []
for line in stdout_lines:
if line == _BATCH_EOR:
records.append(current)
current = []
else:
current.append(line)
# Any trailing lines without a sentinel are ignored (shouldn't occur).
return records
def test_batch_version_json_produces_eor_sentinel() -> None:
"""A single ``version --json`` line produces the version JSON followed by the EOR sentinel."""
exit_code, lines = _run_batch(["version --json"])
assert exit_code == 0
records = _split_records(lines)
assert len(records) == 1
payload = json.loads(records[0][0])
assert payload == {
"client": "mxgw-py",
"package": "mxaccess-gateway-client",
"version": __version__,
}
def test_batch_two_commands_produce_two_delimited_records() -> None:
"""Two input lines produce exactly two EOR-delimited records."""
exit_code, lines = _run_batch(["version --json", "version --json"])
assert exit_code == 0
records = _split_records(lines)
assert len(records) == 2
for record in records:
payload = json.loads(record[0])
assert payload["client"] == "mxgw-py"
def test_batch_eof_exits_zero() -> None:
"""EOF on stdin exits with code 0."""
runner = CliRunner()
result = runner.invoke(main, ["batch"], input="")
assert result.exit_code == 0
def test_batch_empty_line_exits_zero() -> None:
"""An empty line signals a clean exit with code 0."""
exit_code, lines = _run_batch([""])
assert exit_code == 0
# No EOR sentinels should have been emitted.
assert _BATCH_EOR not in lines
def test_batch_empty_line_stops_processing_subsequent_commands() -> None:
"""Commands after the first empty line must not be executed."""
exit_code, lines = _run_batch(["", "version --json"])
assert exit_code == 0
# No records should appear because the empty line stopped the loop.
records = _split_records(lines)
assert records == []
def test_batch_failure_does_not_terminate_loop() -> None:
"""A failing command (bad parse) must not terminate the batch loop."""
exit_code, lines = _run_batch([
"open-session --unknown-flag",
"version --json",
])
assert exit_code == 0
records = _split_records(lines)
# Two records: one error + one success.
assert len(records) == 2
# First record must be a JSON error object.
error_payload = json.loads(records[0][0])
assert "error" in error_payload
assert "type" in error_payload
# Second record must be the version JSON.
version_payload = json.loads(records[1][0])
assert version_payload["client"] == "mxgw-py"
def test_batch_error_record_has_required_json_shape() -> None:
"""A failing command must produce ``{"error": "...", "type": "..."}`` JSON."""
exit_code, lines = _run_batch(["open-session --unknown-flag"])
assert exit_code == 0
records = _split_records(lines)
assert len(records) == 1
payload = json.loads(records[0][0])
assert isinstance(payload.get("error"), str)
assert isinstance(payload.get("type"), str)
def test_batch_network_error_produces_error_json_not_terminates(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A network-level failure (MxGatewayError) on one command must not stop the loop."""
async def _fake_connect(kwargs: dict[str, Any]) -> Any:
raise RuntimeError("injected-network-failure")
monkeypatch.setattr("mxgateway_cli.commands.GatewayClient.connect", _fake_connect)
exit_code, lines = _run_batch([
"open-session --endpoint localhost:5000 --api-key mxgw_test --plaintext --json",
"version --json",
])
assert exit_code == 0
records = _split_records(lines)
assert len(records) == 2
# First record is an error.
error_payload = json.loads(records[0][0])
assert "error" in error_payload
assert "type" in error_payload
# Second record is success.
version_payload = json.loads(records[1][0])
assert version_payload["client"] == "mxgw-py"