From c9288a56c0732b0beea6341d90b2018d69dd3e9b Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 9 Nov 2026 16:22:40 -0500 Subject: [PATCH] ci: job timeout and --blame-hang so a stalled test step fails located, not after 3h of silence (lmxopcua#718) Run 959 (job 3552) was silent from 13:39 for 3h12m and was failed by the server reaper with no cause. Pre-#707 runs 935/938/941/942/947/959 all ended at 189-193 min; no run since 76bdca90 exceeds 82 min (healthy ~53). Adds timeout-minutes 120 and blame-hang (20m, no dump) to the whole-solution test step. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV --- .github/workflows/v2-ci.yml | 9 +++++++++ scripts/ci/README.md | 5 +++++ 2 files changed, 14 insertions(+) diff --git a/.github/workflows/v2-ci.yml b/.github/workflows/v2-ci.yml index b9e801e8..b65dc26e 100644 --- a/.github/workflows/v2-ci.yml +++ b/.github/workflows/v2-ci.yml @@ -33,6 +33,11 @@ jobs: unit-tests: runs-on: ubuntu-latest + # #718: run 959 sat silent for 3h12m (no log after 13:39) until the server's own ~3h endless-task + # reaper failed it, with no hung test named. Every healthy run is ~60 min end to end (test step + # ~53 min), so 120 min fails a stall in half the time and cancels the step rather than waiting + # for the server reaper. + timeout-minutes: 120 steps: - uses: actions/checkout@v4 - uses: actions/setup-dotnet@v4 @@ -69,6 +74,9 @@ jobs: # — sized on an idle dev box — failed in a long # tail, a different test per run. Dilation only stretches waits: a presence wait still fails if # the message never comes, and absence windows get longer (stricter). Local runs stay at 1. + # #718: --blame-hang kills a test host whose single test makes no progress for 20 min and + # NAMES that test in the log (no dump: the runner has no createdump handling), so a hang is a + # located failure instead of hours of silence. Healthy tests finish in seconds. env: AKKA_TEST_TIMEFACTOR: "3" run: > @@ -76,6 +84,7 @@ jobs: -m:1 -p:TestTfmsInParallel=false --settings scripts/ci/ci.runsettings --filter "Category!=E2E&Category!=LiveIntegration" --logger "trx" --results-directory artifacts/trx + --blame-hang --blame-hang-timeout 20m --blame-hang-dump-type none # 07/S-2 fail-on-skip: a whole-solution run that executed ZERO tests is a masked outage # (bad filter, broken discovery), not a pass — fail loudly. Runs even on test failure so # the diagnostic is always emitted. diff --git a/scripts/ci/README.md b/scripts/ci/README.md index 18b96fe8..d09638a4 100644 --- a/scripts/ci/README.md +++ b/scripts/ci/README.md @@ -18,6 +18,11 @@ one run is active; an active run is not cancelled, but a newer pending run may supersede an older pending run. The group does not serialize other repositories using the same host. +Stall guards (#718): the job has `timeout-minutes: 120` (healthy runs take about 60 minutes) +and the test step passes `--blame-hang --blame-hang-timeout 20m --blame-hang-dump-type none`, so a +test that stops making progress is killed and named in the log instead of leaving hours of silence +(run 959 stalled 3h12m until the server reaper failed it). + To reproduce the bounded test leg after restore and Release build: ```bash