diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index bb1e7826..21c9155b 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -3,19 +3,29 @@ # Scope note: the x86 Worker (src/ZB.MOM.WW.MxGateway.Worker) and Worker.Tests target # .NET Framework 4.8 / x86 and need MXAccess COM installed, so they build ONLY on a Windows host. # They are out of scope for this Linux `portable` job — the `windows-x86` job below builds and -# tests them on windev (10.100.0.48) over SSH (see scripts/ci/run-windev-ci.sh), and the nightly -# `nightly-windev` job additionally runs the full solution build, gateway suite, and live-MXAccess -# smoke, which are scheduled-only so they never gate a push. +# tests them on windev (10.100.0.48) over SSH (see scripts/ci/run-windev-ci.sh). +# +# There is no scheduled run. The deep windev tier (full-slnx build, gateway suite, live-MXAccess +# smoke) is on demand, the same way the clients are packed and published on demand: either +# dispatch this workflow by hand with `windev_mode: live`, or run +# `CI_SHA= scripts/ci/run-windev-ci.sh live` from a machine with SSH access to windev. +# The old `nightly-windev` cron job failed on most nights for host reasons and filed an issue +# each time, so it was removed (2026-11-15). name: ci on: push: pull_request: - schedule: - # 06:00 UTC nightly: x86 Worker build + Worker.Tests, full-slnx build, gateway tests, then the - # live-MXAccess smoke (MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1), serialized on windev's host lock. - # Push/PR runs skip it — the job is gated `if: github.event_name == 'schedule'`. - - cron: '0 6 * * *' + workflow_dispatch: + inputs: + windev_mode: + description: 'windev tier for the windows-x86 job (test = x86 Worker build + Worker.Tests; live = test + full slnx + gateway suite + live-MXAccess smoke)' + type: choice + options: + - test + - build + - live + default: test jobs: portable: @@ -165,12 +175,6 @@ jobs: # actions), and a `runs-on` gate with no matching runner wedges the queue in "queued" forever. # An unreachable windev fails this job RED (loud), never leaves it stuck. A red job can mean the # Windows tier is down rather than the change — see docs/GatewayTesting.md § Continuous Integration. - # - # Skipped on the schedule: nightly-windev's `live` mode already runs this exact build + test as - # its first two tiers, and running both made every scheduled run start two windev sessions in - # the same second, contending for the one CI clone (the 2026-09-22 index.lock failure, run 827, - # mxaccessgw#150) and doubling the nightly's load on a host that was already short of memory. - if: github.event_name != 'schedule' runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -183,47 +187,13 @@ jobs: WINDEV_SSH_KEY: ${{ secrets.WINDEV_SSH_KEY }} WINDEV_SSH_USER: ${{ vars.WINDEV_SSH_USER }} CI_SHA: ${{ github.sha }} + # Push/PR runs have no inputs, so they fall back to `test`; a manual dispatch picks the mode. + WINDEV_MODE: ${{ inputs.windev_mode || 'test' }} # `test` = x86 build + Worker.Tests (~50s measured on windev). Demote to `build` only if the # test step ever exceeds ~10 min; the x86 build alone still guards the net48/CS0246/x86 class. - run: ./scripts/ci/run-windev-ci.sh test - - nightly-windev: - # TST-35: scheduled deep run — Worker, full-slnx, gateway, and live-smoke tiers on windev. - # Gated to the schedule event via `if:` (not an unsatisfiable `runs-on`), so push/PR runs mark it - # skipped, not queued. Never gates a push. On failure it opens a Gitea issue so a red nightly is - # visible even though nobody watches the Actions page. - if: github.event_name == 'schedule' - runs-on: ubuntu-latest - env: - # `github.server_url` is the URL Gitea hands the runner — the docker-network-internal - # `http://gitea:3000`. That is the right base for the issue-creation API call below (the job - # container resolves `gitea` only on that network, and has no LAN egress to the public origin), - # but it is useless as a link a human clicks out of the issue. So browser-facing URLs in the - # issue body use the public origin instead. TST-25 acceptance Check 6 caught this: every - # nightly issue since #126 carried an unreachable `http://gitea:3000/...` run link. - PUBLIC_SERVER_URL: https://gitea.dohertylan.com - steps: - - uses: actions/checkout@v4 - - name: Worker + gateway tests + live-MXAccess smoke on windev - env: - # WINDEV_SSH_KEY is stored base64-encoded (single line) so Gitea's line-oriented secret - # masker redacts it in the step env echo; run-windev-ci.sh decodes it. A raw multiline PEM - # would leak in cleartext here (TST-25 acceptance: no key material in logs). Known-hosts is - # public and comes from the committed scripts/ci/windev.known_hosts pin (no secret needed). - WINDEV_SSH_KEY: ${{ secrets.WINDEV_SSH_KEY }} - WINDEV_SSH_USER: ${{ vars.WINDEV_SSH_USER }} - CI_SHA: ${{ github.sha }} - run: ./scripts/ci/run-windev-ci.sh live - - name: Report a red nightly as a Gitea issue - if: failure() - run: | - curl -sSf -X POST \ - -H "Authorization: token ${{ github.token }}" \ - -H "Content-Type: application/json" \ - "${{ github.server_url }}/api/v1/repos/${{ github.repository }}/issues" \ - -d "{\"title\":\"nightly-windev failed on ${{ github.sha }}\",\"body\":\"The scheduled windev Worker + live-MXAccess run failed: ${PUBLIC_SERVER_URL}/${{ github.repository }}/actions/runs/${{ github.run_id }} . A red nightly may mean the Windows tier is down rather than the change — see docs/GatewayTesting.md (Continuous Integration).\"}" + run: ./scripts/ci/run-windev-ci.sh "$WINDEV_MODE" # NOTE: there is intentionally no native `windows` runner job. act_runner v0.6.1 host-mode on # Windows is broken and Windows containers are impractical for the net48/x86/MXAccess Worker, so the - # Windows tier runs via the SSH-driven `windows-x86` (per-push) and `nightly-windev` (scheduled) - # jobs above. Restore a native job only alongside a Windows runner that actually works. + # Windows tier runs via the SSH-driven `windows-x86` job above (per push/PR, or on demand with a + # chosen mode via workflow_dispatch). Restore a native job only alongside a Windows runner that actually works. diff --git a/docs/GatewayTesting.md b/docs/GatewayTesting.md index efa2e194..60ecf556 100644 --- a/docs/GatewayTesting.md +++ b/docs/GatewayTesting.md @@ -89,10 +89,12 @@ fake-worker tests cannot validate: when the rig does not drive sample-bearing buffered batches on demand. All nine tests are gated by the same `MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1` -opt-in variable. Opt-in does not mean unscheduled: the `nightly-windev` job runs -this suite on windev every night at 06:00 UTC and files a Gitea issue when it goes -red (see [Continuous Integration](#continuous-integration)), so the smoke no longer -depends on someone remembering to set the variable. +opt-in variable. Nothing runs this suite on a schedule; run it on demand on windev with +the `live` tier — dispatch the `ci` workflow with `windev_mode: live`, or run +`CI_SHA= scripts/ci/run-windev-ci.sh live` (see +[Continuous Integration](#continuous-integration)). The scheduled `nightly-windev` job +that used to do this was removed on 2026-11-15 because it failed most nights for host +reasons rather than code changes. Known coverage gap: the suite reaches all six late-added MXAccess **COM** commands but none of the five **control** commands (`Ping`, `GetSessionState`, @@ -531,8 +533,8 @@ dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj ## Running the Gateway Suite on windev The gateway suite (`ZB.MOM.WW.MxGateway.Tests`, net10.0/x64) is not part of the CI -Windows tier — `windows-x86` and `nightly-windev` run only the x86 Worker build and -`Worker.Tests`. It is still run on windev by hand when a change needs Windows +Windows tier on a push — `windows-x86` runs only the x86 Worker build and +`Worker.Tests` unless it is dispatched by hand with `windev_mode: live`. It is still run on windev by hand when a change needs Windows confirmation, and that run has three Windows-specific characteristics worth knowing before results are interpreted. @@ -682,8 +684,7 @@ runtime because the x86 Worker cannot build on Linux: client (IPC-24 deleted the old unconditional churn-revert step, which masked exactly that class). The dev Mac has a Homebrew JDK 17, so `gradle generateProto` can be run there to refresh the Java aggregates when a `.proto` changes. -- **`windows-x86`** (Linux runner, per push/PR; skipped on the schedule, where - `nightly-windev` runs the same tiers) — builds the **x86 / net48 Worker and +- **`windows-x86`** (Linux runner, per push/PR, and on demand via `workflow_dispatch`) — builds the **x86 / net48 Worker and Worker.Tests**, which are Windows-only and out of scope for the Linux jobs. It runs on a Linux runner that always schedules and SSHes to windev (`10.100.0.48`), where `scripts/ci/run-windev-ci.sh` fast-forwards the isolated CI clone `C:\build\mxaccessgw-ci` @@ -694,25 +695,28 @@ runtime because the x86 Worker cannot build on Linux: the tier is down, not the change**: an unreachable windev fails the job (never skips), so if it goes red without a related code change, check the tier — `ssh windev`, read `C:\build\mxaccessgw-ci\ci.log` — before assuming a real test failure. -- **`nightly-windev`** (Linux runner, scheduled `0 6 * * *`) — runs, in order, the x86 Worker - build, `Worker.Tests`, a full `slnx` build (which surfaces Windows-only analyzer diagnostics - such as `xUnit1030`), the net10 gateway suite, and the live-MXAccess smoke - (`MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1`). The checkout and all five tiers share the windev - host lock, and the first nonzero native exit stops the sequence and propagates to the CI job. - Gated `if: github.event_name == 'schedule'`, so it never gates a push. On failure it opens a - Gitea issue via the Actions token, since nobody watches the Actions page. +- **Deep windev tier, on demand** — there is no scheduled run. The `live` mode runs, in order, + the x86 Worker build, `Worker.Tests`, a full `slnx` build (which surfaces Windows-only analyzer + diagnostics such as `xUnit1030`), the net10 gateway suite, and the live-MXAccess smoke + (`MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1`). Run it when a change needs it, the same way the clients + are packed and published on demand: dispatch the `ci` workflow from the Gitea Actions page with + `windev_mode: live`, or run `CI_SHA= scripts/ci/run-windev-ci.sh live` from a machine with + SSH access to windev. The checkout and all five tiers share the windev host lock, and the first + nonzero native exit stops the sequence and propagates back. This replaced the `nightly-windev` + cron job (removed 2026-11-15), which failed most nights for host reasons and filed a Gitea issue + each time. The windev host lock is an exclusive file handle on `C:\build\mxaccessgw-ci.ci-lock`, held by the SSH bootstrap for the whole run. It replaced a `New-Item -ItemType Directory` lock that two runs starting in the same second could both take (PowerShell checks for the directory, then -creates it), which is how the 2026-09-22 nightly collided with the scheduled `windows-x86` run on +creates it), which is how the 2026-09-22 nightly (since removed) collided with the scheduled `windows-x86` run on `.git/index.lock` (mxaccessgw#150). The OS drops the handle when its holder dies, so a killed run leaves no stale lock, and a holder may delete a leftover `index.lock` because no other CI git can be running. Each run also writes a `host (start)` line — and a `host (failure)` line when a tier fails — to `C:\build\mxaccessgw-ci\ci.log` with free commit, physical memory, disk and the number of `testhost` processes, so a host-resource failure can be matched to the host's state. -When a nightly goes red, sort the cause before touching code: a test that fails with an +When a `live` run goes red, sort the cause before touching code: a test that fails with an assertion or test exception is a test or product fault; `OutOfMemoryException` from MSBuild/csc/protoc, `csc.exe exited with code -1073741502` (0xC0000142, a process that could not initialize), a remote exit code with no `ci.log` lines at all, or an SSH `kex_exchange_identification` @@ -743,7 +747,7 @@ second runner. When queue depth (or the missing-cancel reality) makes waiting impractical, verify a specific commit out of band instead of waiting behind the queue: run `CI_SHA= scripts/ci/run-windev-ci.sh ` from a machine with SSH access -to windev (the same script the SSH-driven `windows-x86`/`nightly-windev` jobs use — see +to windev (the same script the SSH-driven `windows-x86` job uses — see `scripts/ci/README.md`), or fall back to the manual windev worktree procedure below. This is the same escape hatch used when the windev tier itself is down — TST-30 generalizes it from "tier down" to "runner contended": either way, a stuck or slow shared runner should not diff --git a/docs/runbooks/DeployUpgradeNssmService.md b/docs/runbooks/DeployUpgradeNssmService.md index 62ae7729..9066f266 100644 --- a/docs/runbooks/DeployUpgradeNssmService.md +++ b/docs/runbooks/DeployUpgradeNssmService.md @@ -228,7 +228,8 @@ days. The SQLite DB must also never be opened with a host-side `sqlite3` while t ## 3. Preflight (no changes yet) 1. Decide the target SHA on `origin/main`; confirm CI is green for it (`windows-x86` job runs the - Worker build + `Worker.Tests` on windev; `nightly-windev` runs the live tier, + Worker build + `Worker.Tests` on windev; run the live tier on demand with + `CI_SHA= scripts/ci/run-windev-ci.sh live` or a `windev_mode: live` dispatch, `scripts/ci/README.md`). 2. Read the current state of the service (read-only): ```text diff --git a/scripts/ci/README.md b/scripts/ci/README.md index df15e932..c5005a11 100644 --- a/scripts/ci/README.md +++ b/scripts/ci/README.md @@ -26,10 +26,17 @@ Runs of SHAs older than that change still use the old directory lock and do not runs; that window closes as branches rebase. each native command is checked immediately and the first nonzero exit aborts the remaining tiers and propagates to CI. -CI jobs (`.gitea/workflows/ci.yml`): `windows-x86` runs `test` per push/PR (not on the schedule, -where it would duplicate the nightly's first two tiers and start a second windev session in the -same second); `nightly-windev` runs `live` on the `0 6 * * *` schedule and opens a Gitea issue on -failure. +CI jobs (`.gitea/workflows/ci.yml`): `windows-x86` runs `test` per push/PR. There is no +scheduled run. Run the `live` tier on demand, like the client pack/publish scripts: + +```bash +# from a machine with SSH access to windev (falls back to the default SSH identity) +WINDEV_SSH_USER= CI_SHA=$(git rev-parse HEAD) scripts/ci/run-windev-ci.sh live +``` + +or dispatch the `ci` workflow from the Gitea Actions page with `windev_mode: live`. The +`nightly-windev` cron job that used to run `live` every night and file a Gitea issue on failure +was removed on 2026-11-15: it failed most nights for host reasons and the issues were noise. ## One-time bring-up (operator — required before this lands on `main`) @@ -53,8 +60,6 @@ first, then merge. 4. **Network precheck** — confirm the Gitea runner container (on the `traefik` docker network, host `10.100.0.35`) can reach `10.100.0.48:22`: a one-off `nc -z -w5 10.100.0.48 22` step. The network was created for gitea name resolution, not LAN egress, so verify. -5. **Actions token** — the `nightly-windev` on-failure step creates a Gitea issue with the - built-in token; confirm Actions tokens have issue-write on this repo. ## Verifying it end-to-end (design's acceptance checks) @@ -63,8 +68,8 @@ first, then merge. red; revert. (Proves exit-code propagation — the whole point.) - Unreachable-host red: point at a bogus port / stop sshd, confirm the job fails fast, not hangs. - Concurrency: push two branches back-to-back, confirm the second remote run waits on the lock. -- Nightly: trigger the schedule path, confirm `live` runs and a forced failure opens an issue. - **Done 2026-08-10** (Check 6). Verified two ways: (a) production — every red nightly since +- Nightly (historical — the job was removed 2026-11-15): trigger the schedule path, confirm + `live` runs and a forced failure opens an issue. **Done 2026-08-10** (Check 6). Verified two ways: (a) production — every red nightly since 2026-07-17 has auto-filed an issue (#126–#139) authored by the `gitea-actions` bot, e.g. run 672 → issue #139, with the built-in token masked to `***` in the job log; (b) a forced-failure probe on the throwaway branch `test/tst25-check6-nightly-issue` (run 677 → issue #140, since closed and diff --git a/scripts/ci/run-windev-ci.sh b/scripts/ci/run-windev-ci.sh index 9faddfba..c29c93ed 100755 --- a/scripts/ci/run-windev-ci.sh +++ b/scripts/ci/run-windev-ci.sh @@ -2,7 +2,7 @@ # # TST-25: Linux side of the Windows/x86 Worker CI tier. # -# Runs in the `windows-x86` / nightly Gitea jobs on a Linux runner (which always schedules, +# Runs in the `windows-x86` Gitea job on a Linux runner, or by hand for an on-demand run (which always schedules, # unlike a broken Windows act_runner). SSHes to windev (10.100.0.48), where it fetches + # checks out the SHA under test in the isolated CI clone C:\build\mxaccessgw-ci and runs # scripts/ci/windev-worker-ci.ps1 there. The remote exit code propagates through @@ -94,7 +94,7 @@ echo "run-windev-ci: mode=$MODE sha=$CI_SHA target=${WINDEV_SSH_USER}@${WINDEV_S # It replaced a `New-Item -ItemType Directory` lock that was NOT mutually exclusive: # PowerShell's directory provider checks for existence and then calls CreateDirectory (which # succeeds on an existing directory), so two bootstraps starting together both "acquired" it. -# A scheduled run starts windows-x86 and nightly-windev in the same second, and on 2026-09-22 +# A scheduled run used to start windows-x86 and nightly-windev in the same second, and on 2026-09-22 # (run 827) both got in, ran git concurrently and the nightly died on # ".git/index.lock: File exists" (mxaccessgw#148/#150). A file handle is also released by the # OS if the holder is killed (ssh drop, host reboot), so a dead run can never leave a stale diff --git a/scripts/ci/windev-worker-ci.ps1 b/scripts/ci/windev-worker-ci.ps1 index 89916c43..5609d744 100644 --- a/scripts/ci/windev-worker-ci.ps1 +++ b/scripts/ci/windev-worker-ci.ps1 @@ -5,7 +5,7 @@ Worker test tier that the removed `windows` CI job used to cover (TST-25). .DESCRIPTION - The Linux `windows-x86` / nightly CI jobs cannot build the x86/net48 Worker, so they + The Linux `windows-x86` CI job (and an on-demand run-windev-ci.sh) cannot build the x86/net48 Worker, so they ssh here and run this script inside the isolated CI clone C:\build\mxaccessgw-ci. The script is deliberately self-contained and versioned with the code under test: the @@ -17,7 +17,7 @@ build x86 Worker build only (net48/CS0246/x86 compile guard — always fast). test build + Worker.Tests (x86). The per-push default. live test + full slnx build + the net10 gateway suite + the live-MXAccess smoke - (MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1). Nightly. + (MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1). On demand. Runs under Windows PowerShell 5.1 (`powershell.exe`), so it avoids PS7-only syntax and checks $LASTEXITCODE after every native command rather than relying on stderr-as-error diff --git a/scripts/ci/windev.known_hosts b/scripts/ci/windev.known_hosts index 2aa651e8..fb0d7219 100644 --- a/scripts/ci/windev.known_hosts +++ b/scripts/ci/windev.known_hosts @@ -1,5 +1,5 @@ # Pinned SSH host keys for the windev CI target (10.100.0.48), used by the -# `windows-x86` / nightly jobs so `StrictHostKeyChecking=yes` can verify the host +# `windows-x86` job (and on-demand runs) so `StrictHostKeyChecking=yes` can verify the host # without an interactive TOFU prompt. Host keys are public data — committing them is # intentional (greppable, versioned) per TST-25's design. #