Files
mxaccessgw/scripts/ci/windev-worker-ci.ps1
T
Joseph Doherty e74918ccfb
ci / portable (push) Failing after 10m10s
ci / java (push) Failing after 20m12s
ci / windows-x86 (push) Failing after 22m40s
ci: drop the nightly-windev schedule; run the deep windev tier on demand
The scheduled nightly-windev job failed most nights for host reasons and
filed a Gitea issue each time. Remove the cron trigger and the job, and add
a workflow_dispatch trigger whose windev_mode input (test/build/live) drives
the windows-x86 job, so the live tier runs on demand - from the Actions page
or via scripts/ci/run-windev-ci.sh live - the same way clients are packed and
published on demand. Docs updated to match.

Co-Authored-By: Claude <noreply@anthropic.com>
2026-11-15 09:42:13 -05:00

196 lines
8.6 KiB
PowerShell

<#
.SYNOPSIS
Windows/x86 Worker CI stage — runs ON windev (10.100.0.48), invoked over SSH by
scripts/ci/run-windev-ci.sh at the exact SHA under test. Restores the x86/net48
Worker test tier that the removed `windows` CI job used to cover (TST-25).
.DESCRIPTION
The Linux `windows-x86` CI job (and an on-demand run-windev-ci.sh) cannot build the x86/net48 Worker, so they
ssh here and run this script inside the isolated CI clone C:\build\mxaccessgw-ci.
The script is deliberately self-contained and versioned with the code under test: the
SSH bootstrap only fetches + checks out the SHA (enough to load *this* file), then this
script re-fetches and re-checks-out $Sha INSIDE the worktree lock so the checkout+build
are fully serialized against a concurrent run and can never stomp each other's tree.
Modes (cumulative):
build x86 Worker build only (net48/CS0246/x86 compile guard — always fast).
test build + Worker.Tests (x86). The per-push default.
live test + full slnx build + the net10 gateway suite + the live-MXAccess smoke
(MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1). On demand.
Runs under Windows PowerShell 5.1 (`powershell.exe`), so it avoids PS7-only syntax and
checks $LASTEXITCODE after every native command rather than relying on stderr-as-error
(git writes progress to stderr — the known windev gotcha).
Exits nonzero on the first failure so the SSH exit code propagates to the CI job.
.PARAMETER Sha
The commit SHA under test. HEAD is force-checked-out to it and verified.
.PARAMETER Mode
build | test | live.
.PARAMETER RepoRoot
The CI clone root. Defaults to C:\build\mxaccessgw-ci.
#>
[CmdletBinding()]
param(
[Parameter(Mandatory = $true)][string]$Sha,
[Parameter(Mandatory = $true)][ValidateSet('build', 'test', 'live')][string]$Mode,
[string]$RepoRoot = 'C:\build\mxaccessgw-ci'
)
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
$lockFile = "$RepoRoot.ci-lock"
$logFile = Join-Path $RepoRoot 'ci.log'
$lockTimeout = [TimeSpan]::FromMinutes(20)
function Write-Log {
param([string]$Message)
$line = ('[{0}] {1}' -f (Get-Date -Format 'yyyy-MM-dd HH:mm:ss'), $Message)
Write-Host $line
# Best-effort file log for the "is the tier down?" diagnosis in docs/GatewayTesting.md.
try { Add-Content -Path $logFile -Value $line -ErrorAction SilentlyContinue } catch { }
}
# Run a native command and throw on nonzero exit. Native stderr (git progress) does NOT
# set $LASTEXITCODE, so it is not treated as failure — only a real nonzero exit is.
function Invoke-Native {
param([Parameter(Mandatory = $true)][string]$What, [Parameter(Mandatory = $true)][scriptblock]$Command)
Write-Log "RUN: $What"
& $Command
if ($LASTEXITCODE -ne 0) {
throw "$What failed with exit code $LASTEXITCODE"
}
}
# Logs the host's memory/commit/disk headroom and any leftover test processes, so a
# host-resource failure (OOM in csc/MSBuild/protoc, 0xC0000142 process-start failures, a
# PowerShell that dies before logging) can be correlated with the host's state at the time
# rather than reconstructed afterwards from the event log (mxaccessgw#150). Never fatal.
function Write-HostResources {
param([string]$When)
try {
$os = Get-CimInstance -ClassName Win32_OperatingSystem
$drive = Get-PSDrive -Name ($RepoRoot.Substring(0, 1)) -ErrorAction Stop
$testhosts = @(Get-Process -Name 'testhost*' -ErrorAction SilentlyContinue).Count
Write-Log ('host ({0}): commit free {1:N1} of {2:N1} GB, physical free {3:N1} of {4:N1} GB, {5}: free {6:N1} GB, testhost processes {7}, up since {8:yyyy-MM-dd HH:mm}' -f
$When,
($os.FreeVirtualMemory / 1MB), ($os.TotalVirtualMemorySize / 1MB),
($os.FreePhysicalMemory / 1MB), ($os.TotalVisibleMemorySize / 1MB),
$drive.Name, ($drive.Free / 1GB), $testhosts, $os.LastBootUpTime)
}
catch {
Write-Log "host ($When): resource probe failed: $($_.Exception.Message)"
}
}
# --- acquire the worktree lock ---
# The lock is an exclusive (FileShare.None) handle on "$RepoRoot.ci-lock". It replaced a
# `New-Item -ItemType Directory` lock that was not mutually exclusive (PowerShell checks for
# the directory and then calls CreateDirectory, which succeeds on an existing one), so two
# runs starting together both got in and collided on .git/index.lock (mxaccessgw#148/#150).
# The OS releases the handle if the holder is killed, so a dead run cannot leave it stale.
# run-windev-ci.sh's bootstrap already holds this lock around its pre-hand-off fetch/checkout
# (MXGW_CI_LOCK_HELD=1); re-use it rather than re-acquire (and leave releasing to the
# bootstrap). A standalone/manual run (degraded mode) has no such env and self-locks here.
$deadline = (Get-Date).Add($lockTimeout)
$lock = $null
if ($env:MXGW_CI_LOCK_HELD -eq '1') {
Write-Log "Worktree lock already held by bootstrap (mode=$Mode sha=$Sha)"
}
else {
Write-Log "Acquiring worktree lock $lockFile (mode=$Mode sha=$Sha)"
while ($null -eq $lock) {
try {
$lock = [System.IO.File]::Open($lockFile, 'OpenOrCreate', 'ReadWrite', 'None')
}
catch {
if ((Get-Date) -ge $deadline) {
throw "Timed out after $($lockTimeout.TotalMinutes) min waiting for worktree lock $lockFile ($($_.Exception.Message)). Another CI run is holding it."
}
Start-Sleep -Seconds 10
}
}
}
Write-HostResources -When 'start'
try {
Push-Location $RepoRoot
try {
# Holding the lock proves no other CI git command is running in this clone, so a
# leftover index.lock can only be debris from a killed git; it would fail the checkout.
if (Test-Path '.git\index.lock') {
Write-Log 'Removing stale .git\index.lock left by a killed git'
Remove-Item -Force '.git\index.lock'
}
# Re-fetch + re-checkout under the lock so the tree is the requested SHA even when the
# bootstrap's checkout was skipped (standalone run) or left residue.
Invoke-Native 'git fetch origin' { git fetch --prune origin }
Invoke-Native "git checkout --force $Sha" { git checkout --force --detach $Sha }
# Drop any stray tracked/untracked residue from a prior run so the build is clean.
Invoke-Native 'git reset --hard' { git reset --hard $Sha }
Invoke-Native 'git clean -fdx (build outputs)' { git clean -fdx -e ci.log }
$head = (& git rev-parse HEAD).Trim()
if ($head -ne $Sha) {
throw "Worktree HEAD $head does not match requested SHA $Sha after checkout."
}
Write-Log "Worktree at $head"
$workerProj = 'src/ZB.MOM.WW.MxGateway.Worker/ZB.MOM.WW.MxGateway.Worker.csproj'
$workerTests = 'src/ZB.MOM.WW.MxGateway.Worker.Tests/ZB.MOM.WW.MxGateway.Worker.Tests.csproj'
$gatewayTests = 'src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj'
$integrationTests = 'src/ZB.MOM.WW.MxGateway.IntegrationTests/ZB.MOM.WW.MxGateway.IntegrationTests.csproj'
# build: always. The x86/net48 compile is the floor guard (catches CS0246 on
# unregenerated Generated/, x86-only breaks) even when the test step is demoted.
Invoke-Native 'x86 Worker build' { dotnet build $workerProj -c Release -p:Platform=x86 }
if ($Mode -eq 'test' -or $Mode -eq 'live') {
Invoke-Native 'x86 Worker.Tests' { dotnet test $workerTests -c Release -p:Platform=x86 }
}
if ($Mode -eq 'live') {
# Full-solution build surfaces Windows-only analyzer diagnostics (xUnit1030 etc.)
# that never fire on the macOS/Linux NonWindows build.
Invoke-Native 'full slnx build' { dotnet build src/ZB.MOM.WW.MxGateway.slnx -c Release }
Invoke-Native 'gateway tests on Windows' {
dotnet test $gatewayTests -c Release --no-build
}
$env:MXGATEWAY_RUN_LIVE_MXACCESS_TESTS = '1'
try {
Invoke-Native 'live-MXAccess smoke' {
dotnet test $integrationTests -c Release --filter 'FullyQualifiedName~WorkerLiveMxAccessSmokeTests'
}
}
finally {
Remove-Item Env:\MXGATEWAY_RUN_LIVE_MXACCESS_TESTS -ErrorAction SilentlyContinue
}
}
Write-Log "SUCCESS mode=$Mode sha=$Sha"
}
finally {
Pop-Location
}
}
catch {
Write-Log "FAILURE mode=$Mode sha=$Sha : $($_.Exception.Message)"
Write-HostResources -When 'failure'
throw
}
finally {
if ($null -ne $lock) {
$lock.Dispose()
Write-Log "Released worktree lock $lockFile"
}
}