From 5c8075996f251f86947a3da0f34dfdcb9160f9fc Mon Sep 17 00:00:00 2001
From: Joseph Doherty <dohejw01@gmail.com>
Date: Mon, 13 Jul 2026 10:50:22 -0400
Subject: [PATCH] fix(TST-25): serialize the CI bootstrap fetch/checkout under
 the worktree lock
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Acceptance-check finding (concurrency): the worktree lock in windev-worker-ci.ps1
guarded the build stage, but run-windev-ci.sh's bootstrap did git fetch + git
checkout on the shared C:\build\mxaccessgw-ci clone BEFORE that lock was taken.
Two concurrent runs therefore collided on .git/index.lock at the bootstrap stage
(the second failed 'Another git process seems to be running', exit 1) instead of
the second waiting — defeating the lock's purpose for manual/degraded-mode overlap
or a future second runner. (The single Gitea runner serializes jobs, so CI pushes
never actually overlapped; this is defense-in-depth being restored.)

Fix: the bootstrap now acquires the same mkdir worktree lock around its fetch/
checkout and exports MXGW_CI_LOCK_HELD; windev-worker-ci.ps1 re-uses that lock
(skips re-acquire/release) when the env is set, and still self-locks for a
standalone/manual run. Now the second concurrent run waits at the bootstrap.
---
 scripts/ci/run-windev-ci.sh     | 42 ++++++++++++++++++++++++++-------
 scripts/ci/windev-worker-ci.ps1 | 29 +++++++++++++++--------
 2 files changed, 53 insertions(+), 18 deletions(-)

diff --git a/scripts/ci/run-windev-ci.sh b/scripts/ci/run-windev-ci.sh
index 7d52fff..08b1072 100755
--- a/scripts/ci/run-windev-ci.sh
+++ b/scripts/ci/run-windev-ci.sh
@@ -80,16 +80,42 @@ fi
 echo "run-windev-ci: mode=$MODE sha=$CI_SHA target=${WINDEV_SSH_USER}@${WINDEV_SSH_HOST}"
 
 # Remote PowerShell bootstrap: fetch + checkout enough to load the versioned CI script, then
-# hand off to it. Not Stop around git (stderr progress). windev-worker-ci.ps1 re-fetches and
-# re-checks-out under a lock, so this checkout only needs to load the right script version.
+# hand off to it. Not Stop around git (stderr progress).
+#
+# The bootstrap fetch/checkout mutates the shared clone's git index, so it MUST hold the
+# worktree lock — otherwise two concurrent runs collide on .git/index.lock before either
+# reaches windev-worker-ci.ps1's lock (the exact race the lock exists to prevent). We take the
+# same mkdir lock here (atomic; retry to a 20-min deadline) and export MXGW_CI_LOCK_HELD so the
+# handed-off script re-uses this lock instead of re-acquiring/releasing it. A standalone/manual
+# run of windev-worker-ci.ps1 (degraded mode) has no such env and self-locks as before.
 read -r -d '' BOOTSTRAP <<PS || true
 \$ErrorActionPreference = 'Continue'
-Set-Location '$REMOTE_CLONE'
-git fetch --prune origin
-git checkout --force --detach $CI_SHA
-if (\$LASTEXITCODE -ne 0) { Write-Error "bootstrap checkout failed"; exit \$LASTEXITCODE }
-powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File '$REMOTE_SCRIPT' -Sha '$CI_SHA' -Mode '$MODE'
-exit \$LASTEXITCODE
+\$lockDir = '$REMOTE_CLONE.lock'
+\$deadline = (Get-Date).AddMinutes(20)
+\$haveLock = \$false
+while (-not \$haveLock) {
+    try { New-Item -Path \$lockDir -ItemType Directory -ErrorAction Stop | Out-Null; \$haveLock = \$true }
+    catch {
+        if ((Get-Date) -ge \$deadline) { Write-Error "timed out waiting for worktree lock \$lockDir"; exit 75 }
+        Start-Sleep -Seconds 10
+    }
+}
+\$env:MXGW_CI_LOCK_HELD = '1'
+\$rc = 1
+try {
+    Set-Location '$REMOTE_CLONE'
+    git fetch --prune origin
+    git checkout --force --detach $CI_SHA
+    if (\$LASTEXITCODE -ne 0) { Write-Error "bootstrap checkout failed"; \$rc = \$LASTEXITCODE }
+    else {
+        powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File '$REMOTE_SCRIPT' -Sha '$CI_SHA' -Mode '$MODE'
+        \$rc = \$LASTEXITCODE
+    }
+}
+finally {
+    Remove-Item -Path \$lockDir -Recurse -Force -ErrorAction SilentlyContinue
+}
+exit \$rc
 PS
 
 # PowerShell -EncodedCommand wants UTF-16LE base64 on one line (avoids all ssh/quoting hazards).
diff --git a/scripts/ci/windev-worker-ci.ps1 b/scripts/ci/windev-worker-ci.ps1
index d9391ef..6d6044a 100644
--- a/scripts/ci/windev-worker-ci.ps1
+++ b/scripts/ci/windev-worker-ci.ps1
@@ -68,19 +68,28 @@ function Invoke-Native {
 }
 
 # --- acquire the worktree lock (mkdir is atomic; retry until timeout) ---
+# run-windev-ci.sh's bootstrap already holds this lock around its pre-hand-off fetch/checkout
+# (MXGW_CI_LOCK_HELD=1) so that stage can't race a concurrent run; re-use it rather than
+# re-acquire (and leave releasing to the bootstrap). A standalone/manual run (degraded mode)
+# has no such env and self-locks here as before.
 $deadline = (Get-Date).Add($lockTimeout)
 $haveLock = $false
-Write-Log "Acquiring worktree lock $lockDir (mode=$Mode sha=$Sha)"
-while (-not $haveLock) {
-    try {
-        New-Item -Path $lockDir -ItemType Directory -ErrorAction Stop | Out-Null
-        $haveLock = $true
-    }
-    catch {
-        if ((Get-Date) -ge $deadline) {
-            throw "Timed out after $($lockTimeout.TotalMinutes) min waiting for worktree lock $lockDir. A stuck run may have left it; if no CI job is active, remove it by hand."
+if ($env:MXGW_CI_LOCK_HELD -eq '1') {
+    Write-Log "Worktree lock already held by bootstrap (mode=$Mode sha=$Sha)"
+}
+else {
+    Write-Log "Acquiring worktree lock $lockDir (mode=$Mode sha=$Sha)"
+    while (-not $haveLock) {
+        try {
+            New-Item -Path $lockDir -ItemType Directory -ErrorAction Stop | Out-Null
+            $haveLock = $true
+        }
+        catch {
+            if ((Get-Date) -ge $deadline) {
+                throw "Timed out after $($lockTimeout.TotalMinutes) min waiting for worktree lock $lockDir. A stuck run may have left it; if no CI job is active, remove it by hand."
+            }
+            Start-Sleep -Seconds 10
         }
-        Start-Sleep -Seconds 10
     }
 }