fix(TST-25): serialize the CI bootstrap fetch/checkout under the worktree lock
Acceptance-check finding (concurrency): the worktree lock in windev-worker-ci.ps1 guarded the build stage, but run-windev-ci.sh's bootstrap did git fetch + git checkout on the shared C:\build\mxaccessgw-ci clone BEFORE that lock was taken. Two concurrent runs therefore collided on .git/index.lock at the bootstrap stage (the second failed 'Another git process seems to be running', exit 1) instead of the second waiting — defeating the lock's purpose for manual/degraded-mode overlap or a future second runner. (The single Gitea runner serializes jobs, so CI pushes never actually overlapped; this is defense-in-depth being restored.) Fix: the bootstrap now acquires the same mkdir worktree lock around its fetch/ checkout and exports MXGW_CI_LOCK_HELD; windev-worker-ci.ps1 re-uses that lock (skips re-acquire/release) when the env is set, and still self-locks for a standalone/manual run. Now the second concurrent run waits at the bootstrap.
This commit is contained in:
@@ -80,16 +80,42 @@ fi
|
|||||||
echo "run-windev-ci: mode=$MODE sha=$CI_SHA target=${WINDEV_SSH_USER}@${WINDEV_SSH_HOST}"
|
echo "run-windev-ci: mode=$MODE sha=$CI_SHA target=${WINDEV_SSH_USER}@${WINDEV_SSH_HOST}"
|
||||||
|
|
||||||
# Remote PowerShell bootstrap: fetch + checkout enough to load the versioned CI script, then
|
# Remote PowerShell bootstrap: fetch + checkout enough to load the versioned CI script, then
|
||||||
# hand off to it. Not Stop around git (stderr progress). windev-worker-ci.ps1 re-fetches and
|
# hand off to it. Not Stop around git (stderr progress).
|
||||||
# re-checks-out under a lock, so this checkout only needs to load the right script version.
|
#
|
||||||
|
# The bootstrap fetch/checkout mutates the shared clone's git index, so it MUST hold the
|
||||||
|
# worktree lock — otherwise two concurrent runs collide on .git/index.lock before either
|
||||||
|
# reaches windev-worker-ci.ps1's lock (the exact race the lock exists to prevent). We take the
|
||||||
|
# same mkdir lock here (atomic; retry to a 20-min deadline) and export MXGW_CI_LOCK_HELD so the
|
||||||
|
# handed-off script re-uses this lock instead of re-acquiring/releasing it. A standalone/manual
|
||||||
|
# run of windev-worker-ci.ps1 (degraded mode) has no such env and self-locks as before.
|
||||||
read -r -d '' BOOTSTRAP <<PS || true
|
read -r -d '' BOOTSTRAP <<PS || true
|
||||||
\$ErrorActionPreference = 'Continue'
|
\$ErrorActionPreference = 'Continue'
|
||||||
Set-Location '$REMOTE_CLONE'
|
\$lockDir = '$REMOTE_CLONE.lock'
|
||||||
git fetch --prune origin
|
\$deadline = (Get-Date).AddMinutes(20)
|
||||||
git checkout --force --detach $CI_SHA
|
\$haveLock = \$false
|
||||||
if (\$LASTEXITCODE -ne 0) { Write-Error "bootstrap checkout failed"; exit \$LASTEXITCODE }
|
while (-not \$haveLock) {
|
||||||
powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File '$REMOTE_SCRIPT' -Sha '$CI_SHA' -Mode '$MODE'
|
try { New-Item -Path \$lockDir -ItemType Directory -ErrorAction Stop | Out-Null; \$haveLock = \$true }
|
||||||
exit \$LASTEXITCODE
|
catch {
|
||||||
|
if ((Get-Date) -ge \$deadline) { Write-Error "timed out waiting for worktree lock \$lockDir"; exit 75 }
|
||||||
|
Start-Sleep -Seconds 10
|
||||||
|
}
|
||||||
|
}
|
||||||
|
\$env:MXGW_CI_LOCK_HELD = '1'
|
||||||
|
\$rc = 1
|
||||||
|
try {
|
||||||
|
Set-Location '$REMOTE_CLONE'
|
||||||
|
git fetch --prune origin
|
||||||
|
git checkout --force --detach $CI_SHA
|
||||||
|
if (\$LASTEXITCODE -ne 0) { Write-Error "bootstrap checkout failed"; \$rc = \$LASTEXITCODE }
|
||||||
|
else {
|
||||||
|
powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File '$REMOTE_SCRIPT' -Sha '$CI_SHA' -Mode '$MODE'
|
||||||
|
\$rc = \$LASTEXITCODE
|
||||||
|
}
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
Remove-Item -Path \$lockDir -Recurse -Force -ErrorAction SilentlyContinue
|
||||||
|
}
|
||||||
|
exit \$rc
|
||||||
PS
|
PS
|
||||||
|
|
||||||
# PowerShell -EncodedCommand wants UTF-16LE base64 on one line (avoids all ssh/quoting hazards).
|
# PowerShell -EncodedCommand wants UTF-16LE base64 on one line (avoids all ssh/quoting hazards).
|
||||||
|
|||||||
@@ -68,19 +68,28 @@ function Invoke-Native {
|
|||||||
}
|
}
|
||||||
|
|
||||||
# --- acquire the worktree lock (mkdir is atomic; retry until timeout) ---
|
# --- acquire the worktree lock (mkdir is atomic; retry until timeout) ---
|
||||||
|
# run-windev-ci.sh's bootstrap already holds this lock around its pre-hand-off fetch/checkout
|
||||||
|
# (MXGW_CI_LOCK_HELD=1) so that stage can't race a concurrent run; re-use it rather than
|
||||||
|
# re-acquire (and leave releasing to the bootstrap). A standalone/manual run (degraded mode)
|
||||||
|
# has no such env and self-locks here as before.
|
||||||
$deadline = (Get-Date).Add($lockTimeout)
|
$deadline = (Get-Date).Add($lockTimeout)
|
||||||
$haveLock = $false
|
$haveLock = $false
|
||||||
Write-Log "Acquiring worktree lock $lockDir (mode=$Mode sha=$Sha)"
|
if ($env:MXGW_CI_LOCK_HELD -eq '1') {
|
||||||
while (-not $haveLock) {
|
Write-Log "Worktree lock already held by bootstrap (mode=$Mode sha=$Sha)"
|
||||||
try {
|
}
|
||||||
New-Item -Path $lockDir -ItemType Directory -ErrorAction Stop | Out-Null
|
else {
|
||||||
$haveLock = $true
|
Write-Log "Acquiring worktree lock $lockDir (mode=$Mode sha=$Sha)"
|
||||||
}
|
while (-not $haveLock) {
|
||||||
catch {
|
try {
|
||||||
if ((Get-Date) -ge $deadline) {
|
New-Item -Path $lockDir -ItemType Directory -ErrorAction Stop | Out-Null
|
||||||
throw "Timed out after $($lockTimeout.TotalMinutes) min waiting for worktree lock $lockDir. A stuck run may have left it; if no CI job is active, remove it by hand."
|
$haveLock = $true
|
||||||
|
}
|
||||||
|
catch {
|
||||||
|
if ((Get-Date) -ge $deadline) {
|
||||||
|
throw "Timed out after $($lockTimeout.TotalMinutes) min waiting for worktree lock $lockDir. A stuck run may have left it; if no CI job is active, remove it by hand."
|
||||||
|
}
|
||||||
|
Start-Sleep -Seconds 10
|
||||||
}
|
}
|
||||||
Start-Sleep -Seconds 10
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user