diff --git a/.github/workflows/v2-ci.yml b/.github/workflows/v2-ci.yml index b65dc26e..6fe0cadf 100644 --- a/.github/workflows/v2-ci.yml +++ b/.github/workflows/v2-ci.yml @@ -114,9 +114,6 @@ jobs: # OpcUaServer.IntegrationTests) was removed as fully redundant — the unit-tests leg # above already runs BOTH projects under the identical filter, fixtureless, and gates # them at the MIN_EXECUTED floor. Its only unique output was a report-only skip tally the unit - # leg's trx already contains. FOLLOW-UP (S-2 option #2): if a fixture-backed integration - # leg is wanted, add it as a NEW job that starts the one public-image fixture as a - # `services:` block (mcr.microsoft.com/iotedge/opc-plc; the modbus/S7/AB/FOCAS sims are - # locally-built `otopcua-*` images not pullable here), sets OPCUA_SIM_ENDPOINT, and gates - # at a MIN_EXECUTED floor so a fixture outage turns the job red — covering something the unit - # leg cannot, rather than re-running the same fixtureless tests. + # leg's trx already contains. The fixture-backed leg now lives in its own + # workflow, v2-integration.yml (#757), and the guarded fleet E2E in v2-e2e.yml — kept out of this file + # so this job's runtime and flakiness are unchanged. diff --git a/.github/workflows/v2-e2e.yml b/.github/workflows/v2-e2e.yml new file mode 100644 index 00000000..de2a6701 --- /dev/null +++ b/.github/workflows/v2-e2e.yml @@ -0,0 +1,73 @@ +# Guarded E2E workflow (#757 item a). UNVERIFIED on a hosted runner until its first run. +# +# Runs the simulator-backed driver integration projects against the shared fixture fleet on +# 10.100.0.35 (the fixtures' own default endpoints: Modbus :5020, S7 :1102, opc-plc :50000). That fleet +# is only reachable from a runner on the right network, so this workflow is: +# * MANUAL or SCHEDULED only — never push/PR, so it can never slow or flake the unit-tests leg; +# * SKIP-CLEAN — a TCP probe runs first and every later step is conditioned on it, so an unreachable +# fleet ends the run green with an explicit "skipped" notice rather than a wall of Assert.Skip; +# * ZERO-MATCH GUARDED — once the fleet is reachable, MIN_EXECUTED turns "everything skipped / filter +# matched nothing" (the failure mode that made the old v2-e2e.yml a silent no-op) into a RED job. +# Note: no test currently carries Category=E2E, so the guard is the executed-count floor over the +# fleet-backed projects rather than a Category=E2E match. +# Self-contained (throwaway-container) coverage lives in v2-integration.yml, not here. + +name: v2-e2e + +on: + workflow_dispatch: {} + schedule: + - cron: "17 3 * * 0" # weekly, Sunday 03:17 UTC (off-peak, off the :00 mark) + +# Own group; same semantics as v2-ci (one active run, an active run is never cancelled). +concurrency: + group: lmxopcua-v2-e2e + cancel-in-progress: false + +env: + DOTNET_NOLOGO: "1" + DOTNET_CLI_TELEMETRY_OPTOUT: "1" + +jobs: + e2e: + runs-on: ubuntu-latest + timeout-minutes: 90 # #718 guard + steps: + - uses: actions/checkout@v4 + - name: probe fixture fleet + id: probe + run: bash scripts/ci/probe-e2e-fixtures.sh + - name: skipped — fixture fleet unreachable + if: steps.probe.outputs.reachable != 'true' + run: echo "::notice::E2E skipped — shared fixture fleet 10.100.0.35 is not reachable from this runner." + - uses: actions/setup-dotnet@v4 + if: steps.probe.outputs.reachable == 'true' + with: + dotnet-version: 10.0.x + - name: dotnet test (fleet-backed projects) + if: steps.probe.outputs.reachable == 'true' + env: + E2E_PROJECTS: >- + Driver.Modbus Driver.S7 Driver.OpcUaClient Driver.OpcUaClient.Browser + run: | + set -euo pipefail + for p in $E2E_PROJECTS; do + dotnet test "tests/Drivers/ZB.MOM.WW.OtOpcUa.$p.IntegrationTests/ZB.MOM.WW.OtOpcUa.$p.IntegrationTests.csproj" \ + --configuration Release -m:1 --settings scripts/ci/ci.runsettings \ + --filter "Category!=LiveIntegration" \ + --logger "trx;LogFileName=$p.trx" --results-directory artifacts/trx \ + --blame-hang --blame-hang-timeout 20m --blame-hang-dump-type none + done + - name: assert not-all-skipped + if: always() && steps.probe.outputs.reachable == 'true' + env: + MIN_EXECUTED: "35" + run: bash scripts/ci/assert-not-all-skipped.sh artifacts/trx/*.trx + - name: upload trx + if: always() && steps.probe.outputs.reachable == 'true' + continue-on-error: true + uses: actions/upload-artifact@v3 + with: + name: e2e-trx + path: artifacts/trx/*.trx + if-no-files-found: warn diff --git a/.github/workflows/v2-integration.yml b/.github/workflows/v2-integration.yml new file mode 100644 index 00000000..4272764b --- /dev/null +++ b/.github/workflows/v2-integration.yml @@ -0,0 +1,97 @@ +# Fixture-backed integration leg (#757 item b). UNVERIFIED on a hosted runner until its first run. +# +# Separate from v2-ci.yml on purpose: the 60-minute whole-solution unit-tests job is untouched, so +# it gets no slower or flakier. This workflow builds ONLY the five fixture-backed *.IntegrationTests +# projects and runs them against THROWAWAY containers that the job itself creates and removes +# (scripts/ci/integration-fixtures.sh): SQL Server 2022, PostgreSQL 17 (seeded), QuestDB and the public +# opc-plc image. Nothing here touches the shared 10.100.0.35 host, so CI does not depend on that box. +# +# Explicit `docker run` steps rather than a `services:` block: whether the Gitea act_runner honours +# `services:` (and makes service ports reachable from the job) is not documented in infra/README.md §10, +# and the Sql/Postgres fixtures need a post-start seed step anyway. The script needs only a reachable +# docker daemon on the runner. +# +# Gate: MIN_EXECUTED turns a fixture outage (every test Assert.Skip-ing) into a RED job instead of a +# silent green, which is the whole point of this leg. Not covered here (no public/pullable image or +# needs credentials/TLS): Modbus/S7/AB/FOCAS/TwinCAT sims, Mqtt, MTConnect, Oracle, Host real-infra mode. + +name: v2-integration + +# Triggers start CONSERVATIVE (manual + nightly) because the shared runner is already heavily loaded +# (#718: load avg ~35-51 on 22 cores during unit-tests) and this leg is unverified on it. Once the first +# hosted runs are green and the runner has headroom, widen to push/PR on master (paths below). +# push: { branches: [master] } +# pull_request: { branches: [master], paths: ["src/Drivers/**", "src/Core/**", "tests/Drivers/**", "scripts/ci/**", ".github/workflows/v2-integration.yml"] } +on: + workflow_dispatch: {} + schedule: + - cron: "41 2 * * *" + +# Own group (not lmxopcua-v2-ci): sharing it would let a pending integration run supersede a pending +# unit-tests run (Gitea supersedes pending runs per group). Same semantics as v2-ci: one active run, +# an active run is never cancelled. +concurrency: + group: lmxopcua-v2-integration + cancel-in-progress: false + +env: + DOTNET_NOLOGO: "1" + DOTNET_CLI_TELEMETRY_OPTOUT: "1" + +jobs: + integration-tests: + runs-on: ubuntu-latest + # #718: bound a stall. Expected runtime is well under an hour (image pulls + 5 small projects); + # the per-test --blame-hang below names a hung test, this cancels the job if even that fails. + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: 10.0.x + - name: docker available + run: docker version --format '{{.Server.Version}}' + - name: start throwaway fixtures + run: bash scripts/ci/integration-fixtures.sh up + - name: dotnet test (fixture-backed integration projects) + env: + # Mirrors the unit-tests leg: one worker, bounded xUnit parallelism, hang killer. + INTEGRATION_PROJECTS: >- + Driver.Sql Driver.Historian.SqlServer Driver.Historian.QuestDb + Driver.OpcUaClient Driver.OpcUaClient.Browser + run: | + set -euo pipefail + for p in $INTEGRATION_PROJECTS; do + dotnet test "tests/Drivers/ZB.MOM.WW.OtOpcUa.$p.IntegrationTests/ZB.MOM.WW.OtOpcUa.$p.IntegrationTests.csproj" \ + --configuration Release -m:1 --settings scripts/ci/ci.runsettings \ + --filter "Category!=E2E&Category!=LiveIntegration" \ + --logger "trx;LogFileName=$p.trx" --results-directory artifacts/trx \ + --blame-hang --blame-hang-timeout 20m --blame-hang-dump-type none + done + # A fixture that never came up Assert.Skip()s every test and would otherwise pass. The floor is set + # from a local run against the same containers (see scripts/ci/README.md); raise it as suites grow. + - name: assert not-all-skipped + if: always() + env: + MIN_EXECUTED: "440" + run: bash scripts/ci/assert-not-all-skipped.sh artifacts/trx/*.trx + - name: fixture logs on failure + if: failure() + continue-on-error: true + run: | + for c in otopcua-ci-mssql otopcua-ci-pg otopcua-ci-questdb otopcua-ci-opcplc; do + echo "=== $c ==="; docker logs --tail 40 "$c" 2>&1 || true + done + - name: stop fixtures + if: always() + continue-on-error: true + run: bash scripts/ci/integration-fixtures.sh down + # v3 on purpose — see v2-ci.yml (act_runner reports as GHES; upload-artifact@v4+ refuses it). + - name: upload trx + if: always() + continue-on-error: true + uses: actions/upload-artifact@v3 + with: + name: integration-trx + path: artifacts/trx/*.trx + if-no-files-found: warn diff --git a/infra/README.md b/infra/README.md index 65b3d6af..c1553b1c 100644 --- a/infra/README.md +++ b/infra/README.md @@ -256,3 +256,11 @@ Things that are not obvious (#606): the verified baseline run 975 (2026-10-13) counted 14,630 executed tests across 62 trx files. The step also prints an individual-skip diagnostic (`Results/UnitTestResult` records with outcome `NotExecuted`); it does not affect the floor. Raise the floor as the suite grows. + +### CI legs beyond `unit-tests` (#757; unverified on a hosted runner until first run) + +`v2-integration.yml` runs the fixture-backed Sql / Historian (SQL Server, QuestDB) / OpcUaClient integration +projects against throwaway containers the job starts itself (`scripts/ci/integration-fixtures.sh`; not the +shared 10.100.0.35 host), floor `MIN_EXECUTED=440`. `v2-e2e.yml` is manual/weekly and skips cleanly unless +10.100.0.35 is reachable from the runner. Each has its own concurrency group; see +[`scripts/ci/README.md`](../scripts/ci/README.md). diff --git a/scripts/ci/README.md b/scripts/ci/README.md index d09638a4..7a3d28c7 100644 --- a/scripts/ci/README.md +++ b/scripts/ci/README.md @@ -56,3 +56,26 @@ Restore, Release build, README drift check, the serialized whole-solution test step, the 12,000-test floor and TRX upload all passed. The floor counted 14,630 executed tests across 62 TRX files. This baseline predates the individual skip-reporting correction; its printed zero skip diagnostic is not skip evidence. + +## Fixture-backed integration and guarded E2E workflows (#757) + +**Both are UNVERIFIED on a hosted Gitea runner until their first run**; they were validated locally only +(YAML parse, the exact `dotnet test` commands and fixture script against local throwaway containers). +Neither touches `v2-ci.yml`; each has its own concurrency group (`lmxopcua-v2-integration`, +`lmxopcua-v2-e2e`, `cancel-in-progress: false`) so a pending run of one cannot supersede a pending +`lmxopcua-v2-ci` run. + +- **`v2-integration.yml`** (push to `master`, path-filtered PRs, manual; `timeout-minutes: 60`). + `integration-fixtures.sh up` starts throwaway SQL Server 2022, PostgreSQL 17 (seeded from `seed-pg.sql`), + QuestDB and opc-plc (`otopcua-ci-*`, non-default host ports), exports the env vars the fixtures read, and + `down` removes them. It uses plain `docker run`, not `services:` (act_runner support is undocumented and + Postgres needs a seed step); the only runner requirement is a docker daemon. Runs the Sql, Historian.SqlServer, + Historian.QuestDb, OpcUaClient and OpcUaClient.Browser `*.IntegrationTests` with `MIN_EXECUTED=440`. + Local run: 468 executed / 8 skipped (Oracle x2, the blackhole `docker pause` gate, one QuestDB stress test, + 4 others), about 4.5 min of test time, QuestDB the longest (~3.3 min). Not covered (no pullable image or + needs credentials): Modbus/S7/AB/FOCAS/TwinCAT sims, Mqtt, MTConnect, Oracle, Host real-infra mode. +- **`v2-e2e.yml`** (manual + weekly cron only; `timeout-minutes: 90`). `probe-e2e-fixtures.sh` TCP-probes + 10.100.0.35 ports 5020/1102/50000; if any is closed every later step is skipped and the run is green with a + notice. Otherwise runs the Modbus, S7, OpcUaClient and OpcUaClient.Browser integration projects against the + shared fleet with `MIN_EXECUTED=35` (local: 42 executed / 38 skipped; Modbus skips the profiles that are not + up). No test carries `Category=E2E` today, so the zero-match guard is this executed floor. diff --git a/scripts/ci/integration-fixtures.sh b/scripts/ci/integration-fixtures.sh new file mode 100755 index 00000000..b230a86c --- /dev/null +++ b/scripts/ci/integration-fixtures.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# +# integration-fixtures.sh — throwaway, self-contained fixtures for the v2-integration workflow (#757). +# +# up start SQL Server, PostgreSQL (seeded), QuestDB and opc-plc as containers named +# otopcua-ci-*, wait until each is usable, and print the env exports the +# *.IntegrationTests fixtures read (also appended to $GITHUB_ENV when set). +# down remove those containers (idempotent; touches nothing else). +# +# Deliberately NOT the shared 10.100.0.35 host: CI must not depend on that box, and these +# containers are created and destroyed inside the job. Host ports are non-default so a +# developer's own fixtures are never clobbered when the script is run locally. +# +# Reaching the containers: on the host (or a host-network job) that is 127.0.0.1; when the job +# itself runs inside a container (/.dockerenv) the published ports live on the docker host, +# i.e. the container's default gateway. Override with FIXTURE_HOST. +set -euo pipefail + +SQL_PORT="${CI_SQL_PORT:-14340}" +PG_PORT="${CI_PG_PORT:-15440}" +QDB_PORT="${CI_QDB_PORT:-18812}" +QDB_HTTP_PORT="${CI_QDB_HTTP_PORT:-19000}" +OPC_PORT="${CI_OPC_PORT:-50100}" +# Throwaway-container-only credentials (never used against anything else). +SA_PASSWORD="${CI_SA_PASSWORD:-Ci_throwaway_Pw1}" +PG_PASSWORD="${CI_PG_PASSWORD:-Ci_throwaway_pg1}" +NAMES=(otopcua-ci-mssql otopcua-ci-pg otopcua-ci-questdb otopcua-ci-opcplc) +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +fixture_host() { + if [ -n "${FIXTURE_HOST:-}" ]; then echo "$FIXTURE_HOST"; return; fi + if [ -f /.dockerenv ] && command -v ip >/dev/null 2>&1; then + ip route | awk '/^default/{print $3; exit}'; return + fi + echo 127.0.0.1 +} + +down() { docker rm -f "${NAMES[@]}" >/dev/null 2>&1 || true; } + +wait_for() { # + local what="$1" t="$2"; shift 2 + local end=$((SECONDS + t)) + until "$@" >/dev/null 2>&1; do + if [ "$SECONDS" -ge "$end" ]; then echo "TIMEOUT waiting for $what" >&2; return 1; fi + sleep 3 + done + echo "ready: $what" +} + +up() { + command -v docker >/dev/null || { echo "docker not available" >&2; exit 1; } + down + docker run -d --name otopcua-ci-mssql -e ACCEPT_EULA=Y -e MSSQL_PID=Developer \ + -e "MSSQL_SA_PASSWORD=$SA_PASSWORD" -p "$SQL_PORT:1433" \ + mcr.microsoft.com/mssql/server:2022-latest >/dev/null + docker run -d --name otopcua-ci-pg -e "POSTGRES_PASSWORD=$PG_PASSWORD" -e POSTGRES_DB=sqlpollfixture \ + -p "$PG_PORT:5432" postgres:17-alpine >/dev/null + docker run -d --name otopcua-ci-questdb -p "$QDB_PORT:8812" -p "$QDB_HTTP_PORT:9000" \ + questdb/questdb:10.0.1 >/dev/null + docker run -d --name otopcua-ci-opcplc -p "$OPC_PORT:$OPC_PORT" \ + mcr.microsoft.com/iotedge/opc-plc:2.14.10 --pn="$OPC_PORT" --ut --aa --alm >/dev/null + + wait_for "postgres" 120 docker exec otopcua-ci-pg pg_isready -U postgres -d sqlpollfixture + # pg_isready can answer during initdb's temp server; require a real query on the final server. + wait_for "postgres query" 60 docker exec -e "PGPASSWORD=$PG_PASSWORD" otopcua-ci-pg \ + psql -h 127.0.0.1 -U postgres -d sqlpollfixture -c 'select 1' + docker exec -i -e "PGPASSWORD=$PG_PASSWORD" otopcua-ci-pg \ + psql -v ON_ERROR_STOP=1 -v "app_password=$PG_PASSWORD" -h 127.0.0.1 -U postgres -d sqlpollfixture \ + < "$ROOT/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.IntegrationTests/Docker/postgres/seed-pg.sql" >/dev/null + echo "seeded: postgres" + + local sqlcmd=/opt/mssql-tools18/bin/sqlcmd + wait_for "sql server" 300 docker exec otopcua-ci-mssql "$sqlcmd" -C -S localhost -U sa -P "$SA_PASSWORD" -Q 'SELECT 1' + wait_for "questdb" 120 curl -fsS "http://$(fixture_host):$QDB_HTTP_PORT/exec?query=select%201" + wait_for "opc-plc" 120 bash -c "exec 3<>/dev/tcp/$(fixture_host)/$OPC_PORT" + + local h; h="$(fixture_host)" + local -a exports=( + "SQL_TEST_ENDPOINT=$h,$SQL_PORT" + "SQL_TEST_USER=sa" + "SQL_TEST_PASSWORD=$SA_PASSWORD" + "SQL_PG_ENDPOINT=$h:$PG_PORT" + "SQL_PG_PASSWORD=$PG_PASSWORD" + "HISTORIAN_SQL_TEST_CONNECTION=Server=$h,$SQL_PORT;User Id=sa;Password=$SA_PASSWORD;TrustServerCertificate=True;Encrypt=False" + "HISTORIAN_QUESTDB_TEST_CONNECTION=Host=$h;Port=$QDB_PORT;Username=admin;Password=quest;Database=qdb;ServerCompatibilityMode=NoTypeLoading" + "OPCUA_SIM_ENDPOINT=opc.tcp://$h:$OPC_PORT" + ) + for e in "${exports[@]}"; do + echo "export \"$e\"" + [ -n "${GITHUB_ENV:-}" ] && echo "$e" >> "$GITHUB_ENV" + done +} + +case "${1:-}" in + up) up ;; + down) down ;; + *) echo "usage: $0 up|down" >&2; exit 2 ;; +esac diff --git a/scripts/ci/probe-e2e-fixtures.sh b/scripts/ci/probe-e2e-fixtures.sh new file mode 100755 index 00000000..4652877e --- /dev/null +++ b/scripts/ci/probe-e2e-fixtures.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# +# probe-e2e-fixtures.sh — reachability gate for the guarded v2-e2e workflow (#757). +# +# TCP-probes the shared fixture host's REQUIRED simulator ports. Prints `reachable=true|false` (and +# appends it to $GITHUB_OUTPUT when set). Exit status is always 0: an unreachable fleet is an expected, +# clean skip (the runner is off-VPN, the host is down), not a failure. Only the probe itself erring +# (bad usage) exits non-zero. +# +# Env: E2E_FIXTURE_HOST (default 10.100.0.35) +# E2E_REQUIRED_PORTS (default "5020 1102 50000" = Modbus, S7, opc-plc) +# E2E_PROBE_TIMEOUT (seconds per port, default 5) +set -uo pipefail +host="${E2E_FIXTURE_HOST:-10.100.0.35}" +ports="${E2E_REQUIRED_PORTS:-5020 1102 50000}" +t="${E2E_PROBE_TIMEOUT:-5}" +ok=true +for p in $ports; do + if timeout "$t" bash -c "exec 3<>/dev/tcp/$host/$p" 2>/dev/null; then + echo "probe: $host:$p open" + else + echo "probe: $host:$p UNREACHABLE"; ok=false + fi +done +echo "reachable=$ok" +[ -n "${GITHUB_OUTPUT:-}" ] && echo "reachable=$ok" >> "$GITHUB_OUTPUT" +exit 0