e2e: drive each client CLI through one long-lived batch process
The cross-language e2e matrix spawned one CLI process per operation —
~250 per client — paying a process (and, for the Java CLI, a full JVM)
cold-start every time. The Java leg alone ran ~16 minutes.
Each client CLI (dotnet, go, rust, python, java) gains a `batch`
subcommand: a single process that reads one command line from stdin,
runs it through the normal subcommand dispatch, writes the JSON result,
then a line containing exactly `__MXGW_BATCH_EOR__`. A failing command
writes its `{"error":...}` envelope and the loop continues.
run-client-e2e-tests.ps1 now launches one batch process per client and
pings every operation through its stdin/stdout, so startup is paid once
per client. The orchestration and assertions are unchanged; the parity
and auth phases now read the `{"error":...}` envelope instead of a
process exit code.
Full 5-client matrix with -VerifyWrite: ~15 min, down from ~35; the Java
leg dropped from ~16 min to ~2-3.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1,13 +1,15 @@
|
||||
"""Tests for the Python CLI."""
|
||||
|
||||
import io
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import click
|
||||
import pytest
|
||||
from click.testing import CliRunner
|
||||
|
||||
from mxgateway import __version__
|
||||
from mxgateway_cli.commands import _use_plaintext, main
|
||||
from mxgateway_cli.commands import _BATCH_EOR, _use_plaintext, main
|
||||
|
||||
|
||||
def test_version_json_is_deterministic() -> None:
|
||||
@@ -216,3 +218,142 @@ def test_cli_localhost_endpoint_with_plaintext_flag_uses_plaintext(
|
||||
|
||||
assert result.exit_code != 0
|
||||
assert captured.get("plaintext") is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# batch subcommand tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _run_batch(lines: list[str]) -> tuple[int, list[str]]:
|
||||
"""Invoke ``batch`` with the given stdin lines; return (exit_code, stdout_lines)."""
|
||||
runner = CliRunner()
|
||||
stdin_text = "\n".join(lines) + "\n"
|
||||
result = runner.invoke(main, ["batch"], input=stdin_text)
|
||||
stdout_lines = result.output.splitlines()
|
||||
return result.exit_code, stdout_lines
|
||||
|
||||
|
||||
def _split_records(stdout_lines: list[str]) -> list[list[str]]:
|
||||
"""Split stdout lines on ``__MXGW_BATCH_EOR__`` sentinels into per-command records."""
|
||||
records: list[list[str]] = []
|
||||
current: list[str] = []
|
||||
for line in stdout_lines:
|
||||
if line == _BATCH_EOR:
|
||||
records.append(current)
|
||||
current = []
|
||||
else:
|
||||
current.append(line)
|
||||
# Any trailing lines without a sentinel are ignored (shouldn't occur).
|
||||
return records
|
||||
|
||||
|
||||
def test_batch_version_json_produces_eor_sentinel() -> None:
|
||||
"""A single ``version --json`` line produces the version JSON followed by the EOR sentinel."""
|
||||
exit_code, lines = _run_batch(["version --json"])
|
||||
|
||||
assert exit_code == 0
|
||||
records = _split_records(lines)
|
||||
assert len(records) == 1
|
||||
payload = json.loads(records[0][0])
|
||||
assert payload == {
|
||||
"client": "mxgw-py",
|
||||
"package": "mxaccess-gateway-client",
|
||||
"version": __version__,
|
||||
}
|
||||
|
||||
|
||||
def test_batch_two_commands_produce_two_delimited_records() -> None:
|
||||
"""Two input lines produce exactly two EOR-delimited records."""
|
||||
exit_code, lines = _run_batch(["version --json", "version --json"])
|
||||
|
||||
assert exit_code == 0
|
||||
records = _split_records(lines)
|
||||
assert len(records) == 2
|
||||
for record in records:
|
||||
payload = json.loads(record[0])
|
||||
assert payload["client"] == "mxgw-py"
|
||||
|
||||
|
||||
def test_batch_eof_exits_zero() -> None:
|
||||
"""EOF on stdin exits with code 0."""
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(main, ["batch"], input="")
|
||||
assert result.exit_code == 0
|
||||
|
||||
|
||||
def test_batch_empty_line_exits_zero() -> None:
|
||||
"""An empty line signals a clean exit with code 0."""
|
||||
exit_code, lines = _run_batch([""])
|
||||
assert exit_code == 0
|
||||
# No EOR sentinels should have been emitted.
|
||||
assert _BATCH_EOR not in lines
|
||||
|
||||
|
||||
def test_batch_empty_line_stops_processing_subsequent_commands() -> None:
|
||||
"""Commands after the first empty line must not be executed."""
|
||||
exit_code, lines = _run_batch(["", "version --json"])
|
||||
|
||||
assert exit_code == 0
|
||||
# No records should appear because the empty line stopped the loop.
|
||||
records = _split_records(lines)
|
||||
assert records == []
|
||||
|
||||
|
||||
def test_batch_failure_does_not_terminate_loop() -> None:
|
||||
"""A failing command (bad parse) must not terminate the batch loop."""
|
||||
exit_code, lines = _run_batch([
|
||||
"open-session --unknown-flag",
|
||||
"version --json",
|
||||
])
|
||||
|
||||
assert exit_code == 0
|
||||
records = _split_records(lines)
|
||||
# Two records: one error + one success.
|
||||
assert len(records) == 2
|
||||
# First record must be a JSON error object.
|
||||
error_payload = json.loads(records[0][0])
|
||||
assert "error" in error_payload
|
||||
assert "type" in error_payload
|
||||
# Second record must be the version JSON.
|
||||
version_payload = json.loads(records[1][0])
|
||||
assert version_payload["client"] == "mxgw-py"
|
||||
|
||||
|
||||
def test_batch_error_record_has_required_json_shape() -> None:
|
||||
"""A failing command must produce ``{"error": "...", "type": "..."}`` JSON."""
|
||||
exit_code, lines = _run_batch(["open-session --unknown-flag"])
|
||||
|
||||
assert exit_code == 0
|
||||
records = _split_records(lines)
|
||||
assert len(records) == 1
|
||||
payload = json.loads(records[0][0])
|
||||
assert isinstance(payload.get("error"), str)
|
||||
assert isinstance(payload.get("type"), str)
|
||||
|
||||
|
||||
def test_batch_network_error_produces_error_json_not_terminates(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A network-level failure (MxGatewayError) on one command must not stop the loop."""
|
||||
|
||||
async def _fake_connect(kwargs: dict[str, Any]) -> Any:
|
||||
raise RuntimeError("injected-network-failure")
|
||||
|
||||
monkeypatch.setattr("mxgateway_cli.commands.GatewayClient.connect", _fake_connect)
|
||||
|
||||
exit_code, lines = _run_batch([
|
||||
"open-session --endpoint localhost:5000 --api-key mxgw_test --plaintext --json",
|
||||
"version --json",
|
||||
])
|
||||
|
||||
assert exit_code == 0
|
||||
records = _split_records(lines)
|
||||
assert len(records) == 2
|
||||
# First record is an error.
|
||||
error_payload = json.loads(records[0][0])
|
||||
assert "error" in error_payload
|
||||
assert "type" in error_payload
|
||||
# Second record is success.
|
||||
version_payload = json.loads(records[1][0])
|
||||
assert version_payload["client"] == "mxgw-py"
|
||||
|
||||
Reference in New Issue
Block a user