ci: fixture-backed integration workflow and guarded fleet E2E workflow (lmxopcua#757)

v2-integration.yml runs the Sql, Historian.SqlServer, Historian.QuestDb and OpcUaClient integration projects against throwaway SQL Server/PostgreSQL/QuestDB/opc-plc containers started by scripts/ci/integration-fixtures.sh, with a MIN_EXECUTED=440 floor. v2-e2e.yml is manual/weekly, probes the shared fleet and skips cleanly when unreachable. v2-ci.yml is functionally unchanged. Unverified on a hosted runner until first run. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
2026-11-09 16:53:02 -05:00
parent 0702b31029
commit 94926eb0d7
7 changed files with 329 additions and 6 deletions
@@ -114,9 +114,6 @@ jobs:
  # OpcUaServer.IntegrationTests) was removed as fully redundant — the unit-tests leg
  # above already runs BOTH projects under the identical filter, fixtureless, and gates
  # them at the MIN_EXECUTED floor. Its only unique output was a report-only skip tally the unit
-  # leg's trx already contains. FOLLOW-UP (S-2 option #2): if a fixture-backed integration
-  # leg is wanted, add it as a NEW job that starts the one public-image fixture as a
-  # `services:` block (mcr.microsoft.com/iotedge/opc-plc; the modbus/S7/AB/FOCAS sims are
-  # locally-built `otopcua-*` images not pullable here), sets OPCUA_SIM_ENDPOINT, and gates
-  # at a MIN_EXECUTED floor so a fixture outage turns the job red — covering something the unit
-  # leg cannot, rather than re-running the same fixtureless tests.
+  # leg's trx already contains. The fixture-backed leg now lives in its own
+  # workflow, v2-integration.yml (#757), and the guarded fleet E2E in v2-e2e.yml — kept out of this file
+  # so this job's runtime and flakiness are unchanged.
@@ -0,0 +1,73 @@
+# Guarded E2E workflow (#757 item a). UNVERIFIED on a hosted runner until its first run.
+#
+# Runs the simulator-backed driver integration projects against the shared fixture fleet on
+# 10.100.0.35 (the fixtures' own default endpoints: Modbus :5020, S7 :1102, opc-plc :50000). That fleet
+# is only reachable from a runner on the right network, so this workflow is:
+#   * MANUAL or SCHEDULED only — never push/PR, so it can never slow or flake the unit-tests leg;
+#   * SKIP-CLEAN — a TCP probe runs first and every later step is conditioned on it, so an unreachable
+#     fleet ends the run green with an explicit "skipped" notice rather than a wall of Assert.Skip;
+#   * ZERO-MATCH GUARDED — once the fleet is reachable, MIN_EXECUTED turns "everything skipped / filter
+#     matched nothing" (the failure mode that made the old v2-e2e.yml a silent no-op) into a RED job.
+#     Note: no test currently carries Category=E2E, so the guard is the executed-count floor over the
+#     fleet-backed projects rather than a Category=E2E match.
+# Self-contained (throwaway-container) coverage lives in v2-integration.yml, not here.
+
+name: v2-e2e
+
+on:
+  workflow_dispatch: {}
+  schedule:
+    - cron: "17 3 * * 0"   # weekly, Sunday 03:17 UTC (off-peak, off the :00 mark)
+
+# Own group; same semantics as v2-ci (one active run, an active run is never cancelled).
+concurrency:
+  group: lmxopcua-v2-e2e
+  cancel-in-progress: false
+
+env:
+  DOTNET_NOLOGO: "1"
+  DOTNET_CLI_TELEMETRY_OPTOUT: "1"
+
+jobs:
+  e2e:
+    runs-on: ubuntu-latest
+    timeout-minutes: 90   # #718 guard
+    steps:
+      - uses: actions/checkout@v4
+      - name: probe fixture fleet
+        id: probe
+        run: bash scripts/ci/probe-e2e-fixtures.sh
+      - name: skipped — fixture fleet unreachable
+        if: steps.probe.outputs.reachable != 'true'
+        run: echo "::notice::E2E skipped — shared fixture fleet 10.100.0.35 is not reachable from this runner."
+      - uses: actions/setup-dotnet@v4
+        if: steps.probe.outputs.reachable == 'true'
+        with:
+          dotnet-version: 10.0.x
+      - name: dotnet test (fleet-backed projects)
+        if: steps.probe.outputs.reachable == 'true'
+        env:
+          E2E_PROJECTS: >-
+            Driver.Modbus Driver.S7 Driver.OpcUaClient Driver.OpcUaClient.Browser
+        run: |
+          set -euo pipefail
+          for p in $E2E_PROJECTS; do
+            dotnet test "tests/Drivers/ZB.MOM.WW.OtOpcUa.$p.IntegrationTests/ZB.MOM.WW.OtOpcUa.$p.IntegrationTests.csproj" \
+              --configuration Release -m:1 --settings scripts/ci/ci.runsettings \
+              --filter "Category!=LiveIntegration" \
+              --logger "trx;LogFileName=$p.trx" --results-directory artifacts/trx \
+              --blame-hang --blame-hang-timeout 20m --blame-hang-dump-type none
+          done
+      - name: assert not-all-skipped
+        if: always() && steps.probe.outputs.reachable == 'true'
+        env:
+          MIN_EXECUTED: "35"
+        run: bash scripts/ci/assert-not-all-skipped.sh artifacts/trx/*.trx
+      - name: upload trx
+        if: always() && steps.probe.outputs.reachable == 'true'
+        continue-on-error: true
+        uses: actions/upload-artifact@v3
+        with:
+          name: e2e-trx
+          path: artifacts/trx/*.trx
+          if-no-files-found: warn
@@ -0,0 +1,97 @@
+# Fixture-backed integration leg (#757 item b). UNVERIFIED on a hosted runner until its first run.
+#
+# Separate from v2-ci.yml on purpose: the 60-minute whole-solution unit-tests job is untouched, so
+# it gets no slower or flakier. This workflow builds ONLY the five fixture-backed *.IntegrationTests
+# projects and runs them against THROWAWAY containers that the job itself creates and removes
+# (scripts/ci/integration-fixtures.sh): SQL Server 2022, PostgreSQL 17 (seeded), QuestDB and the public
+# opc-plc image. Nothing here touches the shared 10.100.0.35 host, so CI does not depend on that box.
+#
+# Explicit `docker run` steps rather than a `services:` block: whether the Gitea act_runner honours
+# `services:` (and makes service ports reachable from the job) is not documented in infra/README.md §10,
+# and the Sql/Postgres fixtures need a post-start seed step anyway. The script needs only a reachable
+# docker daemon on the runner.
+#
+# Gate: MIN_EXECUTED turns a fixture outage (every test Assert.Skip-ing) into a RED job instead of a
+# silent green, which is the whole point of this leg. Not covered here (no public/pullable image or
+# needs credentials/TLS): Modbus/S7/AB/FOCAS/TwinCAT sims, Mqtt, MTConnect, Oracle, Host real-infra mode.
+
+name: v2-integration
+
+# Triggers start CONSERVATIVE (manual + nightly) because the shared runner is already heavily loaded
+# (#718: load avg ~35-51 on 22 cores during unit-tests) and this leg is unverified on it. Once the first
+# hosted runs are green and the runner has headroom, widen to push/PR on master (paths below).
+#   push: { branches: [master] }
+#   pull_request: { branches: [master], paths: ["src/Drivers/**", "src/Core/**", "tests/Drivers/**", "scripts/ci/**", ".github/workflows/v2-integration.yml"] }
+on:
+  workflow_dispatch: {}
+  schedule:
+    - cron: "41 2 * * *"
+
+# Own group (not lmxopcua-v2-ci): sharing it would let a pending integration run supersede a pending
+# unit-tests run (Gitea supersedes pending runs per group). Same semantics as v2-ci: one active run,
+# an active run is never cancelled.
+concurrency:
+  group: lmxopcua-v2-integration
+  cancel-in-progress: false
+
+env:
+  DOTNET_NOLOGO: "1"
+  DOTNET_CLI_TELEMETRY_OPTOUT: "1"
+
+jobs:
+  integration-tests:
+    runs-on: ubuntu-latest
+    # #718: bound a stall. Expected runtime is well under an hour (image pulls + 5 small projects);
+    # the per-test --blame-hang below names a hung test, this cancels the job if even that fails.
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: 10.0.x
+      - name: docker available
+        run: docker version --format '{{.Server.Version}}'
+      - name: start throwaway fixtures
+        run: bash scripts/ci/integration-fixtures.sh up
+      - name: dotnet test (fixture-backed integration projects)
+        env:
+          # Mirrors the unit-tests leg: one worker, bounded xUnit parallelism, hang killer.
+          INTEGRATION_PROJECTS: >-
+            Driver.Sql Driver.Historian.SqlServer Driver.Historian.QuestDb
+            Driver.OpcUaClient Driver.OpcUaClient.Browser
+        run: |
+          set -euo pipefail
+          for p in $INTEGRATION_PROJECTS; do
+            dotnet test "tests/Drivers/ZB.MOM.WW.OtOpcUa.$p.IntegrationTests/ZB.MOM.WW.OtOpcUa.$p.IntegrationTests.csproj" \
+              --configuration Release -m:1 --settings scripts/ci/ci.runsettings \
+              --filter "Category!=E2E&Category!=LiveIntegration" \
+              --logger "trx;LogFileName=$p.trx" --results-directory artifacts/trx \
+              --blame-hang --blame-hang-timeout 20m --blame-hang-dump-type none
+          done
+      # A fixture that never came up Assert.Skip()s every test and would otherwise pass. The floor is set
+      # from a local run against the same containers (see scripts/ci/README.md); raise it as suites grow.
+      - name: assert not-all-skipped
+        if: always()
+        env:
+          MIN_EXECUTED: "440"
+        run: bash scripts/ci/assert-not-all-skipped.sh artifacts/trx/*.trx
+      - name: fixture logs on failure
+        if: failure()
+        continue-on-error: true
+        run: |
+          for c in otopcua-ci-mssql otopcua-ci-pg otopcua-ci-questdb otopcua-ci-opcplc; do
+            echo "=== $c ==="; docker logs --tail 40 "$c" 2>&1 || true
+          done
+      - name: stop fixtures
+        if: always()
+        continue-on-error: true
+        run: bash scripts/ci/integration-fixtures.sh down
+      # v3 on purpose — see v2-ci.yml (act_runner reports as GHES; upload-artifact@v4+ refuses it).
+      - name: upload trx
+        if: always()
+        continue-on-error: true
+        uses: actions/upload-artifact@v3
+        with:
+          name: integration-trx
+          path: artifacts/trx/*.trx
+          if-no-files-found: warn