test(perf): measure phase-8 WP-4 residual criteria xc-3/4/5/8 (#49)
New ResidualBench tool (eventlog, flatten, audit, audit-contention) and docker/measure-ui-responsiveness.sh. All four residuals pass; results in docs/plans/2026-11-10-wp4-residual-scope-results.md. Findings filed as #122, #123 and #124 (fixed in the following commit).
This commit is contained in:
Joseph Doherty
Executable
+186
@@ -0,0 +1,186 @@
|
||||
#!/usr/bin/env bash
|
||||
# measure-ui-responsiveness.sh — Phase-8 WP-4 [xc-8] "UI workflows remain responsive" (ScadaBridge#49).
|
||||
#
|
||||
# Times Central UI page loads on the docker rig, first idle and then under load, with a real
|
||||
# Chromium (host python Playwright). A "load" is a full navigation measured until the page is
|
||||
# usable: the Blazor circuit is up and no loading indicator (spinner / "Loading…") remains.
|
||||
#
|
||||
# Load phase, all concurrent with the measuring browser:
|
||||
# * OPERATORS extra browser sessions (default 6), each looping the same page list;
|
||||
# * an inbound-API client calling SimpleAlarmStatus on MesAlarmDemo01 at API_RPS (default 20)
|
||||
# — central method script + Route.To() to site-a + audit rows for every call.
|
||||
#
|
||||
# Prereqs: rig up; docker/seed-mes-alarm-demo.sh applied (SimpleAlarmStatus + MesAlarmDemo01);
|
||||
# python3 playwright with chromium; the CLI built. A temporary API key `zztest-ui-load-<ts>`
|
||||
# is created for the run and deleted afterwards.
|
||||
#
|
||||
# Usage: bash docker/measure-ui-responsiveness.sh [--operators N] [--api-rps N] [--repeats N]
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
URL="${SCADABRIDGE_URL:-http://localhost:9000}"
|
||||
OPERATORS=6; API_RPS=20; REPEATS=5
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--operators) OPERATORS="$2"; shift 2 ;;
|
||||
--api-rps) API_RPS="$2"; shift 2 ;;
|
||||
--repeats) REPEATS="$2"; shift 2 ;;
|
||||
*) echo "unknown option $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
CLI=(dotnet run --project src/ZB.MOM.WW.ScadaBridge.CLI --no-build -- --url "$URL"
|
||||
--username "${SCADABRIDGE_USERNAME:-multi-role}" --password "${SCADABRIDGE_PASSWORD:-password}" --format json)
|
||||
python3 -c 'import playwright.sync_api' 2>/dev/null || { echo "python3 playwright missing (pip install playwright && playwright install chromium)"; exit 1; }
|
||||
|
||||
KEY_NAME="zztest-ui-load-$(date +%s)"
|
||||
# `api-key create` prints text, not JSON: "API key created. KeyId: <id>", then the token alone on a line
|
||||
# (the "save this token" banner goes to stderr).
|
||||
KEY_OUT=$("${CLI[@]}" security api-key create --name "$KEY_NAME" --methods SimpleAlarmStatus 2>/dev/null)
|
||||
KEY_ID=$(echo "$KEY_OUT" | sed -n 's/.*KeyId: *\([A-Za-z0-9]*\).*/\1/p' | head -1)
|
||||
TOKEN=$(echo "$KEY_OUT" | awk 'NF==1 && length($1) > 20 {print $1; exit}')
|
||||
[ -n "$TOKEN" ] && [ -n "$KEY_ID" ] || { echo "could not parse api-key create output; delete key '$KEY_NAME' by hand" >&2; exit 1; }
|
||||
cleanup() { "${CLI[@]}" security api-key delete --key-id "$KEY_ID" >/dev/null 2>&1 || echo "WARN: delete API key $KEY_NAME ($KEY_ID) by hand"; }
|
||||
trap cleanup EXIT
|
||||
echo "# temporary API key $KEY_NAME ($KEY_ID)"
|
||||
|
||||
# multiprocessing "spawn" re-imports the main module from a file, so the helper cannot run from stdin.
|
||||
HELPER=$(mktemp "${TMPDIR:-/tmp}/sb-ui-responsiveness.XXXXXX.py")
|
||||
trap 'rm -f "$HELPER"; cleanup' EXIT
|
||||
cat >"$HELPER" <<'PY'
|
||||
import json, multiprocessing as mp, os, statistics, threading, time, urllib.request
|
||||
|
||||
URL = os.environ["URL"]; TOKEN = os.environ["TOKEN"]
|
||||
OPERATORS = int(os.environ["OPERATORS"]); API_RPS = int(os.environ["API_RPS"]); REPEATS = int(os.environ["REPEATS"])
|
||||
|
||||
PAGES = [
|
||||
("Dashboard", "/"),
|
||||
("Health", "/monitoring/health"),
|
||||
("Topology", "/deployment/topology"),
|
||||
("Instances", "/deployment/instances"),
|
||||
("Deployments", "/deployment/deployments"),
|
||||
("Templates", "/design/templates"),
|
||||
("Audit Log", "/audit/log"),
|
||||
("Config audit", "/audit/configuration"),
|
||||
("Site Calls", "/site-calls/report"),
|
||||
("Notifications", "/notifications/report"),
|
||||
("Alarms", "/monitoring/alarms"),
|
||||
("Event Logs", "/monitoring/event-logs"),
|
||||
("Sites", "/admin/sites"),
|
||||
("Flow Debugger", "/deployment/flow-debugger"),
|
||||
]
|
||||
|
||||
READY_JS = """() => {
|
||||
if (window.Blazor === undefined) return false;
|
||||
const busy = document.querySelector('.spinner-border, .spinner-grow, .placeholder-glow, [aria-busy="true"]');
|
||||
if (busy && busy.offsetParent !== null) return false;
|
||||
const main = document.querySelector('main') || document.body;
|
||||
return !/Loading\\u2026|Loading\\.\\.\\./.test(main.innerText);
|
||||
}"""
|
||||
|
||||
def load(page, path):
|
||||
"""Navigate and return ms until the page is usable: the Blazor circuit's websocket has
|
||||
received a frame, and the page then shows no loading indicator for 200 ms straight
|
||||
(an interactive re-render can briefly bring a spinner back after the prerendered HTML)."""
|
||||
t = time.perf_counter()
|
||||
with page.expect_websocket(lambda ws: "_blazor" in ws.url, timeout=60000) as ws_info:
|
||||
page.goto(URL + path, wait_until="domcontentloaded", timeout=60000)
|
||||
ws_info.value.wait_for_event("framereceived", timeout=60000)
|
||||
deadline = t + 60
|
||||
ready_since = None
|
||||
while time.perf_counter() < deadline:
|
||||
now = time.perf_counter()
|
||||
if page.evaluate(READY_JS):
|
||||
ready_since = ready_since or now
|
||||
if now - ready_since >= 0.2:
|
||||
return (ready_since - t) * 1000
|
||||
else:
|
||||
ready_since = None
|
||||
time.sleep(0.03)
|
||||
raise TimeoutError(f"{path} not ready within 60 s")
|
||||
|
||||
def measure(label, repeats):
|
||||
from playwright.sync_api import sync_playwright
|
||||
res = {}
|
||||
with sync_playwright() as p:
|
||||
b = p.chromium.launch()
|
||||
pg = b.new_page()
|
||||
load(pg, "/") # warm the browser + the circuit path once
|
||||
for name, path in PAGES:
|
||||
res[name] = [load(pg, path) for _ in range(repeats)]
|
||||
b.close()
|
||||
return res
|
||||
|
||||
def operator(stop, idx, counter):
|
||||
from playwright.sync_api import sync_playwright
|
||||
with sync_playwright() as p:
|
||||
b = p.chromium.launch()
|
||||
pg = b.new_page()
|
||||
i = idx
|
||||
while not stop.is_set():
|
||||
name, path = PAGES[i % len(PAGES)]
|
||||
try:
|
||||
load(pg, path)
|
||||
with counter.get_lock(): counter.value += 1
|
||||
except Exception:
|
||||
pass
|
||||
i += 1
|
||||
b.close()
|
||||
|
||||
def api_client(stop, stats):
|
||||
body = json.dumps({"MachineCode": "MesAlarmDemo01"}).encode()
|
||||
interval = 1.0 / API_RPS
|
||||
def one():
|
||||
req = urllib.request.Request(URL + "/api/SimpleAlarmStatus", data=body, method="POST",
|
||||
headers={"Content-Type": "application/json", "X-API-Key": TOKEN})
|
||||
t = time.perf_counter()
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=60) as r:
|
||||
r.read(); ok = r.status == 200
|
||||
except Exception:
|
||||
ok = False
|
||||
stats.append(((time.perf_counter() - t) * 1000, ok))
|
||||
nxt = time.perf_counter()
|
||||
while not stop.is_set():
|
||||
threading.Thread(target=one, daemon=True).start()
|
||||
nxt += interval
|
||||
time.sleep(max(0, nxt - time.perf_counter()))
|
||||
|
||||
def pct(xs, p):
|
||||
xs = sorted(xs); return xs[min(len(xs) - 1, max(0, int(round(p / 100 * len(xs) + 0.5)) - 1))]
|
||||
|
||||
def report(idle, loaded):
|
||||
print("| page | idle P50 ms | idle max ms | load P50 ms | load max ms |")
|
||||
print("|---|---|---|---|---|")
|
||||
for name, _ in PAGES:
|
||||
a, b = idle[name], loaded[name]
|
||||
print(f"| {name} | {statistics.median(a):.0f} | {max(a):.0f} | {statistics.median(b):.0f} | {max(b):.0f} |")
|
||||
allA = [x for v in idle.values() for x in v]; allB = [x for v in loaded.values() for x in v]
|
||||
print(f"| **all pages** | P50 {statistics.median(allA):.0f}, P95 {pct(allA, 95):.0f} | max {max(allA):.0f} | "
|
||||
f"P50 {statistics.median(allB):.0f}, P95 {pct(allB, 95):.0f} | max {max(allB):.0f} |")
|
||||
|
||||
if __name__ == "__main__":
|
||||
mp.set_start_method("spawn")
|
||||
print(f"## idle ({REPEATS} loads per page)")
|
||||
idle = measure("idle", REPEATS)
|
||||
|
||||
stop = mp.Event(); counter = mp.Value("i", 0)
|
||||
ops = [mp.Process(target=operator, args=(stop, i * 3, counter)) for i in range(OPERATORS)]
|
||||
for o in ops: o.start()
|
||||
api_stats = []; tstop = threading.Event()
|
||||
api = threading.Thread(target=api_client, args=(tstop, api_stats), daemon=True); api.start()
|
||||
time.sleep(15) # let the background load reach steady state
|
||||
t0 = time.perf_counter(); c0 = counter.value; n0 = len(api_stats)
|
||||
loaded = measure("load", REPEATS)
|
||||
secs = time.perf_counter() - t0
|
||||
tstop.set(); stop.set()
|
||||
for o in ops: o.join(timeout=60)
|
||||
window = api_stats[n0:]
|
||||
lat = [l for l, ok in window]; oks = sum(1 for _, ok in window if ok)
|
||||
print(f"\n## under load: {OPERATORS} operator sessions ({(counter.value - c0) / secs:.1f} page loads/s), "
|
||||
f"inbound API {len(window) / secs:.1f} calls/s offered, {oks}/{len(window)} HTTP 200, "
|
||||
f"API latency P50 {statistics.median(lat):.0f} ms / P95 {pct(lat, 95):.0f} ms / max {max(lat):.0f} ms")
|
||||
print()
|
||||
report(idle, loaded)
|
||||
PY
|
||||
URL="$URL" TOKEN="$TOKEN" OPERATORS="$OPERATORS" API_RPS="$API_RPS" REPEATS="$REPEATS" python3 "$HELPER"
|
||||
Reference in New Issue
Block a user