diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/Docker/Dockerfile b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/Docker/Dockerfile index d4614a9f3..0dce5459a 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/Docker/Dockerfile +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/Docker/Dockerfile @@ -21,16 +21,23 @@ COPY profiles/ /fixtures/ # compose profile. See Docker/README.md §exception injection. COPY exception_injector.py /fixtures/ +# #909: launcher that runs the simulator with asyncio debug mode OFF (pymodbus 3.13 +# hard-codes debug=True, which made it >10x slower and let it be swamped by one test run), +# plus the FC03 liveness probe it uses as an in-container watchdog and compose uses as +# the healthcheck. See run_simulator.py. +COPY run_simulator.py fc3_probe.py /fixtures/ + # 5020 = the shared port for the standard/dl205/mitsubishi/s7_1500/exception_injection # profiles (only one binds it at a time); 5021 = the rtu_over_tcp profile, which co-runs # with standard. Image-metadata honesty only — compose's ports: mapping doesn't need EXPOSE. EXPOSE 5020 5021 # Default to the standard profile; docker-compose.yml overrides per service. +# Never launch the stock `pymodbus.simulator` console script: it runs asyncio in debug mode (#909). # --http_port intentionally omitted; pymodbus 3.13's web UI binds on a # container-local default we don't publish, so it's not reachable from the # host and costs nothing. -CMD ["pymodbus.simulator", \ +CMD ["python", "/fixtures/run_simulator.py", \ "--modbus_server", "srv", \ "--modbus_device", "dev", \ "--json_file", "/fixtures/standard.json"] diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/Docker/README.md b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/Docker/README.md index c9ec09383..a6edf9e53 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/Docker/README.md +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/Docker/README.md @@ -14,6 +14,8 @@ Docker Desktop and nothing else. | [`docker-compose.yml`](docker-compose.yml) | One service per profile (`standard` / `dl205` / `mitsubishi` / `s7_1500` / `exception_injection` bind `:5020`, so only one of those runs at a time; `rtu_over_tcp` binds `:5021` and can run alongside `standard`) | | [`profiles/*.json`](profiles/) | Same seed-register definitions the native launcher uses — canonical source | | [`exception_injector.py`](exception_injector.py) | Pure-stdlib Modbus/TCP server that emits arbitrary exception codes per rule — used by the `exception_injection` profile | +| [`run_simulator.py`](run_simulator.py) | Starts the pymodbus simulator with asyncio debug mode off, plus the FC03 watchdog (#909) | +| [`fc3_probe.py`](fc3_probe.py) | FC03-read-with-deadline liveness probe: the compose healthcheck and the watchdog (#909) | ## Run @@ -92,6 +94,44 @@ need a specific profile (the basic round-trip set) run under any of the three pymodbus-based profiles. The `exception_injection` profile is the only one that runs `exception_injector.py` instead of pymodbus. +## Robustness under load (#909) + +The pymodbus simulator is a single asyncio loop, and it is shared: the docker-dev rig's +Modbus drivers poll it too (49 drivers on each of central-1 and central-2 in November 2026, +about 220 requests/s at idle). Four things keep it from getting swamped: + +- **No asyncio debug mode.** pymodbus 3.13's `pymodbus.simulator` console script hard-codes + `asyncio.run(..., debug=True)`. Debug mode captures a traceback for every scheduled callback + and logs any callback slower than 100 ms ("Executing took 0.146 seconds"). + Side by side on the shared host (closed-loop FC03 clients, 2 s deadline), debug mode served + about 120-170 req/s to 100 clients with a p99 of 1.6-1.9 s and client timeouts; without it, + about 2400 req/s with a p99 of about 135 ms. Every pymodbus service therefore starts through + [`run_simulator.py`](run_simulator.py), which takes the same arguments and runs with + `debug=False`. **Do not go back to the stock `pymodbus.simulator` entrypoint.** +- **A real healthcheck.** [`fc3_probe.py`](fc3_probe.py) sends an FC03 read and passes only on + a timely, well-formed reply (an exception reply counts). A TCP connect is not enough: the + #909 simulator accepted connections and answered after the client had timed out. + `docker ps` shows `(healthy)` / `(unhealthy)`. +- **Self-heal.** Docker does not restart unhealthy containers. `run_simulator.py` runs the + same probe on a watchdog thread every 10 s and exits after 3 consecutive failures, and + `restart: unless-stopped` starts a fresh container. Tune it with `SIM_WATCHDOG_INTERVAL` + (`0` disables it), `SIM_WATCHDOG_FAILURES` and `SIM_WATCHDOG_DEADLINE`. +- **Bounded logs.** json-file `max-size: 10m`, `max-file: 3`. + +With `restart: unless-stopped`, a container stopped by `docker compose down` or `docker stop` +stays stopped. A container that was running when the docker daemon restarts comes back, so +remove the old `:5020` profile container (see the profile-cycling gotcha above) before +switching profiles. + +If tests still time out, look at the load before the fixture. The `Cancel send, because not +connected!` count in `docker logs` is the number of replies that arrived after the client had +given up. Count the established connections and their owners: + +```bash +ssh dohertj2@10.100.0.35 'PID=$(docker inspect -f "{{.State.Pid}}" otopcua-pymodbus-standard); \ + sudo nsenter -t $PID -n ss -tnH state established "( sport = :5020 )" | awk "{print \$4}" | cut -d: -f1 | sort | uniq -c' +``` + ## Endpoint - Default: `localhost:5020` diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/Docker/docker-compose.yml b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/Docker/docker-compose.yml index 9ce9bb17b..26127d250 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/Docker/docker-compose.yml +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/Docker/docker-compose.yml @@ -9,70 +9,90 @@ # docker compose --profile dl205 up # docker compose --profile mitsubishi up # docker compose --profile s7_1500 up +# +# #909 hardening, shared by every pymodbus service through the anchors below: +# * the simulator starts through run_simulator.py, never the stock `pymodbus.simulator` +# console script, because that script hard-codes asyncio debug mode (>10x slower; see +# run_simulator.py for the measurements); +# * a healthcheck that does a real FC03 read with a deadline (fc3_probe.py). A TCP +# connect is not enough: the #909 sim accepted connections and answered late; +# * run_simulator.py's in-container watchdog exits after 3 failed probes (~30 s), and +# `restart: unless-stopped` brings up a fresh container (docker has no +# restart-on-unhealthy, and an autoheal sidecar would need the docker socket); +# * bounded json-file logs (the old container had kept 440k+ lines since October). +x-pymodbus: &pymodbus + image: otopcua-pymodbus:3.13.0 + build: + context: . + dockerfile: Dockerfile + labels: + project: lmxopcua + restart: unless-stopped + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + +x-healthcheck: &fc3-healthcheck + test: ["CMD", "python", "/fixtures/fc3_probe.py", "--port", "5020", "--deadline", "3"] + interval: 15s + timeout: 5s + retries: 3 + start_period: 10s + services: standard: + <<: *pymodbus profiles: ["standard"] - build: - context: . - dockerfile: Dockerfile - image: otopcua-pymodbus:3.13.0 container_name: otopcua-pymodbus-standard - restart: "no" ports: - "5020:5020" + healthcheck: *fc3-healthcheck command: [ - "pymodbus.simulator", + "python", "/fixtures/run_simulator.py", "--modbus_server", "srv", "--modbus_device", "dev", "--json_file", "/fixtures/standard.json" ] dl205: + <<: *pymodbus profiles: ["dl205"] - image: otopcua-pymodbus:3.13.0 - build: - context: . - dockerfile: Dockerfile container_name: otopcua-pymodbus-dl205 - restart: "no" ports: - "5020:5020" + healthcheck: *fc3-healthcheck command: [ - "pymodbus.simulator", + "python", "/fixtures/run_simulator.py", "--modbus_server", "srv", "--modbus_device", "dev", "--json_file", "/fixtures/dl205.json" ] mitsubishi: + <<: *pymodbus profiles: ["mitsubishi"] - image: otopcua-pymodbus:3.13.0 - build: - context: . - dockerfile: Dockerfile container_name: otopcua-pymodbus-mitsubishi - restart: "no" ports: - "5020:5020" + healthcheck: *fc3-healthcheck command: [ - "pymodbus.simulator", + "python", "/fixtures/run_simulator.py", "--modbus_server", "srv", "--modbus_device", "dev", "--json_file", "/fixtures/mitsubishi.json" ] s7_1500: + <<: *pymodbus profiles: ["s7_1500"] - image: otopcua-pymodbus:3.13.0 - build: - context: . - dockerfile: Dockerfile container_name: otopcua-pymodbus-s7_1500 - restart: "no" ports: - "5020:5020" + healthcheck: *fc3-healthcheck command: [ - "pymodbus.simulator", + "python", "/fixtures/run_simulator.py", "--modbus_server", "srv", "--modbus_device", "dev", "--json_file", "/fixtures/s7_1500.json" @@ -84,19 +104,16 @@ services: # the `standard` profile: two families, two ports, no conflict. Serves # rtu_over_tcp.json (byte-for-byte standard.json + framer/port swap). rtu_over_tcp: + <<: *pymodbus profiles: ["rtu_over_tcp"] - image: otopcua-pymodbus:3.13.0 - build: - context: . - dockerfile: Dockerfile container_name: otopcua-pymodbus-rtu_over_tcp - labels: - project: lmxopcua - restart: "no" ports: - "5021:5021" + healthcheck: + <<: *fc3-healthcheck + test: ["CMD", "python", "/fixtures/fc3_probe.py", "--port", "5021", "--framer", "rtu", "--deadline", "3"] command: [ - "pymodbus.simulator", + "python", "/fixtures/run_simulator.py", "--modbus_server", "srv", "--modbus_device", "dev", "--json_file", "/fixtures/rtu_over_tcp.json" @@ -107,17 +124,16 @@ services: # simulator — pymodbus naturally emits only exception codes 02 + 03, and # this profile extends integration coverage to the other codes the # driver's MapModbusExceptionToStatus table handles (01, 04, 05, 06, - # 0A, 0B). Rules are driven by exception_injection.json. + # 0A, 0B). Rules are driven by exception_injection.json. No asyncio debug + # mode and no watchdog here; the FC03 healthcheck still applies (an + # exception reply counts as a timely answer). exception_injection: + <<: *pymodbus profiles: ["exception_injection"] - image: otopcua-pymodbus:3.13.0 - build: - context: . - dockerfile: Dockerfile container_name: otopcua-modbus-exception-injector - restart: "no" ports: - "5020:5020" + healthcheck: *fc3-healthcheck command: [ "python", "/fixtures/exception_injector.py", "--config", "/fixtures/exception_injection.json" diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/Docker/fc3_probe.py b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/Docker/fc3_probe.py new file mode 100644 index 000000000..331aa5d3e --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/Docker/fc3_probe.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Liveness probe for the pymodbus simulator: one FC03 read with a hard deadline (#909). + +A TCP connect is not enough. The #909 fixture accepted connections while answering +every request after the client had timed out, so a connect-only check stayed green. +This probe sends a real FC03 read and passes only on a timely, well-formed reply. +A Modbus exception reply also passes: it proves the request loop is serving. + +Pure stdlib, so it runs in the image without extra packages. It is used two ways: + * as the compose ``healthcheck`` (exit code 0 = healthy, 1 = unhealthy); + * imported by ``run_simulator.py``, whose watchdog exits the container after + repeated failures so ``restart: unless-stopped`` brings up a fresh one. + +Usage: fc3_probe.py [--port 5020] [--framer socket|rtu] [--deadline 3] +""" +import argparse +import socket +import struct +import sys +import time + + +def _crc16(data: bytes) -> int: + crc = 0xFFFF + for b in data: + crc ^= b + for _ in range(8): + crc = (crc >> 1) ^ 0xA001 if crc & 1 else crc >> 1 + return crc + + +def _recv_exact(sock: socket.socket, n: int, end: float) -> bytes: + buf = b"" + while len(buf) < n: + remaining = end - time.monotonic() + if remaining <= 0: + raise TimeoutError("deadline passed") + sock.settimeout(remaining) + chunk = sock.recv(n - len(buf)) + if not chunk: + raise ConnectionError("closed mid-reply") + buf += chunk + return buf + + +def probe(host: str = "127.0.0.1", port: int = 5020, framer: str = "socket", + deadline: float = 3.0, unit: int = 1, address: int = 0) -> None: + """Raise on failure; return on a timely, well-formed FC03 reply (data or exception).""" + end = time.monotonic() + deadline + pdu = struct.pack(">BHH", 3, address, 1) + with socket.create_connection((host, port), timeout=deadline) as sock: + if framer == "rtu": + frame = bytes([unit]) + pdu + sock.sendall(frame + struct.pack("HHHB", tid, 0, len(pdu) + 1, unit) + pdu) + header = _recv_exact(sock, 7, end) + rtid, _, length, _ = struct.unpack(">HHHB", header) + if rtid != tid: + raise ValueError(f"transaction id {rtid} != {tid}") + if length < 2: + raise ValueError(f"MBAP length {length} too small") + _recv_exact(sock, length - 1, end) + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--host", default="127.0.0.1") + ap.add_argument("--port", type=int, default=5020) + ap.add_argument("--framer", choices=["socket", "rtu"], default="socket") + ap.add_argument("--deadline", type=float, default=3.0) + args = ap.parse_args() + try: + probe(args.host, args.port, args.framer, args.deadline) + except Exception as exc: # noqa: BLE001 - any failure is "unhealthy" + print(f"FC03 probe failed: {type(exc).__name__}: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/Docker/run_simulator.py b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/Docker/run_simulator.py new file mode 100644 index 000000000..45b8cef50 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/Docker/run_simulator.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Start the pymodbus simulator WITHOUT asyncio debug mode, under a self-healing watchdog (#909). + +Takes the same arguments as the stock ``pymodbus.simulator`` console script. + +Why not run the console script directly: pymodbus 3.13's +``pymodbus/server/simulator/main.py`` hard-codes ``asyncio.run(run_main(), debug=True)``. +asyncio debug mode records a traceback for every scheduled callback and times every +callback, logging any that run over 100 ms. Measured on the shared docker host +(2026-11-13, same image, side by side, closed-loop FC03 clients with a 2 s deadline): +20 clients ~170-220 req/s in debug mode against ~1300-1500 req/s without it; 100 clients +~120-170 req/s (p50 0.5-0.8 s, p99 1.6-1.9 s, client timeouts) against ~2350-2450 req/s +(p99 ~135 ms, no timeouts). Debug mode made the shared simulator the bottleneck: +that is the #909 failure: "took 0.146 seconds" warnings, thousands of +``Cancel send, because not connected!`` lines, and late replies. + +The watchdog thread runs ``fc3_probe`` against the server every +``SIM_WATCHDOG_INTERVAL`` seconds (default 10). After ``SIM_WATCHDOG_FAILURES`` +consecutive failures (default 3) it exits the process with status 1, so the compose +``restart: unless-stopped`` policy starts a fresh container. Docker has no built-in +"restart when unhealthy", and an autoheal sidecar would need the docker socket. +Set ``SIM_WATCHDOG_INTERVAL=0`` to disable the watchdog. +""" +import asyncio +import json +import os +import sys +import threading +import time + +from pymodbus.server.simulator.main import run_main + +import fc3_probe + + +def _arg(name: str, default=None): + argv = sys.argv[1:] + if name in argv and argv.index(name) + 1 < len(argv): + return argv[argv.index(name) + 1] + return default + + +def _server_endpoint(): + """Port and framer of the selected server_list entry, from the profile JSON.""" + try: + with open(_arg("--json_file"), encoding="utf-8") as fh: + servers = json.load(fh)["server_list"] + srv = servers[_arg("--modbus_server", next(iter(servers)))] + return int(srv.get("port", 5020)), srv.get("framer", "socket") + except Exception: # noqa: BLE001 - fall back to the standard profile's endpoint + return 5020, "socket" + + +def _watchdog(port: int, framer: str, interval: float, limit: int, deadline: float) -> None: + time.sleep(max(interval, 5.0)) # let the listener come up + failures = 0 + while True: + try: + fc3_probe.probe(port=port, framer=framer, deadline=deadline) + failures = 0 + except Exception as exc: # noqa: BLE001 + failures += 1 + print(f"watchdog: FC03 probe {failures}/{limit} failed: {type(exc).__name__}: {exc}", + file=sys.stderr, flush=True) + if failures >= limit: + print("watchdog: simulator is not answering in time; exiting so the container restarts", + file=sys.stderr, flush=True) + os._exit(1) + time.sleep(interval) + + +def main() -> None: + interval = float(os.environ.get("SIM_WATCHDOG_INTERVAL", "10")) + if interval > 0: + port, framer = _server_endpoint() + threading.Thread( + target=_watchdog, + args=(port, framer, interval, + int(os.environ.get("SIM_WATCHDOG_FAILURES", "3")), + float(os.environ.get("SIM_WATCHDOG_DEADLINE", "3"))), + name="fc3-watchdog", daemon=True).start() + # debug=False is the point of this file: see the module docstring. + asyncio.run(run_main(sys.argv[1:]), debug=False) + + +if __name__ == "__main__": + main()