fix(TST-25): serialize the CI bootstrap fetch/checkout under the worktree lock
Acceptance-check finding (concurrency): the worktree lock in windev-worker-ci.ps1 guarded the build stage, but run-windev-ci.sh's bootstrap did git fetch + git checkout on the shared C:\build\mxaccessgw-ci clone BEFORE that lock was taken. Two concurrent runs therefore collided on .git/index.lock at the bootstrap stage (the second failed 'Another git process seems to be running', exit 1) instead of the second waiting — defeating the lock's purpose for manual/degraded-mode overlap or a future second runner. (The single Gitea runner serializes jobs, so CI pushes never actually overlapped; this is defense-in-depth being restored.) Fix: the bootstrap now acquires the same mkdir worktree lock around its fetch/ checkout and exports MXGW_CI_LOCK_HELD; windev-worker-ci.ps1 re-uses that lock (skips re-acquire/release) when the env is set, and still self-locks for a standalone/manual run. Now the second concurrent run waits at the bootstrap.
This commit is contained in:
@@ -80,16 +80,42 @@ fi
|
||||
echo "run-windev-ci: mode=$MODE sha=$CI_SHA target=${WINDEV_SSH_USER}@${WINDEV_SSH_HOST}"
|
||||
|
||||
# Remote PowerShell bootstrap: fetch + checkout enough to load the versioned CI script, then
|
||||
# hand off to it. Not Stop around git (stderr progress). windev-worker-ci.ps1 re-fetches and
|
||||
# re-checks-out under a lock, so this checkout only needs to load the right script version.
|
||||
# hand off to it. Not Stop around git (stderr progress).
|
||||
#
|
||||
# The bootstrap fetch/checkout mutates the shared clone's git index, so it MUST hold the
|
||||
# worktree lock — otherwise two concurrent runs collide on .git/index.lock before either
|
||||
# reaches windev-worker-ci.ps1's lock (the exact race the lock exists to prevent). We take the
|
||||
# same mkdir lock here (atomic; retry to a 20-min deadline) and export MXGW_CI_LOCK_HELD so the
|
||||
# handed-off script re-uses this lock instead of re-acquiring/releasing it. A standalone/manual
|
||||
# run of windev-worker-ci.ps1 (degraded mode) has no such env and self-locks as before.
|
||||
read -r -d '' BOOTSTRAP <<PS || true
|
||||
\$ErrorActionPreference = 'Continue'
|
||||
Set-Location '$REMOTE_CLONE'
|
||||
git fetch --prune origin
|
||||
git checkout --force --detach $CI_SHA
|
||||
if (\$LASTEXITCODE -ne 0) { Write-Error "bootstrap checkout failed"; exit \$LASTEXITCODE }
|
||||
powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File '$REMOTE_SCRIPT' -Sha '$CI_SHA' -Mode '$MODE'
|
||||
exit \$LASTEXITCODE
|
||||
\$lockDir = '$REMOTE_CLONE.lock'
|
||||
\$deadline = (Get-Date).AddMinutes(20)
|
||||
\$haveLock = \$false
|
||||
while (-not \$haveLock) {
|
||||
try { New-Item -Path \$lockDir -ItemType Directory -ErrorAction Stop | Out-Null; \$haveLock = \$true }
|
||||
catch {
|
||||
if ((Get-Date) -ge \$deadline) { Write-Error "timed out waiting for worktree lock \$lockDir"; exit 75 }
|
||||
Start-Sleep -Seconds 10
|
||||
}
|
||||
}
|
||||
\$env:MXGW_CI_LOCK_HELD = '1'
|
||||
\$rc = 1
|
||||
try {
|
||||
Set-Location '$REMOTE_CLONE'
|
||||
git fetch --prune origin
|
||||
git checkout --force --detach $CI_SHA
|
||||
if (\$LASTEXITCODE -ne 0) { Write-Error "bootstrap checkout failed"; \$rc = \$LASTEXITCODE }
|
||||
else {
|
||||
powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File '$REMOTE_SCRIPT' -Sha '$CI_SHA' -Mode '$MODE'
|
||||
\$rc = \$LASTEXITCODE
|
||||
}
|
||||
}
|
||||
finally {
|
||||
Remove-Item -Path \$lockDir -Recurse -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
exit \$rc
|
||||
PS
|
||||
|
||||
# PowerShell -EncodedCommand wants UTF-16LE base64 on one line (avoids all ssh/quoting hazards).
|
||||
|
||||
@@ -68,19 +68,28 @@ function Invoke-Native {
|
||||
}
|
||||
|
||||
# --- acquire the worktree lock (mkdir is atomic; retry until timeout) ---
|
||||
# run-windev-ci.sh's bootstrap already holds this lock around its pre-hand-off fetch/checkout
|
||||
# (MXGW_CI_LOCK_HELD=1) so that stage can't race a concurrent run; re-use it rather than
|
||||
# re-acquire (and leave releasing to the bootstrap). A standalone/manual run (degraded mode)
|
||||
# has no such env and self-locks here as before.
|
||||
$deadline = (Get-Date).Add($lockTimeout)
|
||||
$haveLock = $false
|
||||
Write-Log "Acquiring worktree lock $lockDir (mode=$Mode sha=$Sha)"
|
||||
while (-not $haveLock) {
|
||||
try {
|
||||
New-Item -Path $lockDir -ItemType Directory -ErrorAction Stop | Out-Null
|
||||
$haveLock = $true
|
||||
}
|
||||
catch {
|
||||
if ((Get-Date) -ge $deadline) {
|
||||
throw "Timed out after $($lockTimeout.TotalMinutes) min waiting for worktree lock $lockDir. A stuck run may have left it; if no CI job is active, remove it by hand."
|
||||
if ($env:MXGW_CI_LOCK_HELD -eq '1') {
|
||||
Write-Log "Worktree lock already held by bootstrap (mode=$Mode sha=$Sha)"
|
||||
}
|
||||
else {
|
||||
Write-Log "Acquiring worktree lock $lockDir (mode=$Mode sha=$Sha)"
|
||||
while (-not $haveLock) {
|
||||
try {
|
||||
New-Item -Path $lockDir -ItemType Directory -ErrorAction Stop | Out-Null
|
||||
$haveLock = $true
|
||||
}
|
||||
catch {
|
||||
if ((Get-Date) -ge $deadline) {
|
||||
throw "Timed out after $($lockTimeout.TotalMinutes) min waiting for worktree lock $lockDir. A stuck run may have left it; if no CI job is active, remove it by hand."
|
||||
}
|
||||
Start-Sleep -Seconds 10
|
||||
}
|
||||
Start-Sleep -Seconds 10
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user