e74918ccfb
The scheduled nightly-windev job failed most nights for host reasons and filed a Gitea issue each time. Remove the cron trigger and the job, and add a workflow_dispatch trigger whose windev_mode input (test/build/live) drives the windows-x86 job, so the live tier runs on demand - from the Actions page or via scripts/ci/run-windev-ci.sh live - the same way clients are packed and published on demand. Docs updated to match. Co-Authored-By: Claude <noreply@anthropic.com>
196 lines
8.6 KiB
PowerShell
196 lines
8.6 KiB
PowerShell
<#
|
|
.SYNOPSIS
|
|
Windows/x86 Worker CI stage — runs ON windev (10.100.0.48), invoked over SSH by
|
|
scripts/ci/run-windev-ci.sh at the exact SHA under test. Restores the x86/net48
|
|
Worker test tier that the removed `windows` CI job used to cover (TST-25).
|
|
|
|
.DESCRIPTION
|
|
The Linux `windows-x86` CI job (and an on-demand run-windev-ci.sh) cannot build the x86/net48 Worker, so they
|
|
ssh here and run this script inside the isolated CI clone C:\build\mxaccessgw-ci.
|
|
|
|
The script is deliberately self-contained and versioned with the code under test: the
|
|
SSH bootstrap only fetches + checks out the SHA (enough to load *this* file), then this
|
|
script re-fetches and re-checks-out $Sha INSIDE the worktree lock so the checkout+build
|
|
are fully serialized against a concurrent run and can never stomp each other's tree.
|
|
|
|
Modes (cumulative):
|
|
build x86 Worker build only (net48/CS0246/x86 compile guard — always fast).
|
|
test build + Worker.Tests (x86). The per-push default.
|
|
live test + full slnx build + the net10 gateway suite + the live-MXAccess smoke
|
|
(MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1). On demand.
|
|
|
|
Runs under Windows PowerShell 5.1 (`powershell.exe`), so it avoids PS7-only syntax and
|
|
checks $LASTEXITCODE after every native command rather than relying on stderr-as-error
|
|
(git writes progress to stderr — the known windev gotcha).
|
|
|
|
Exits nonzero on the first failure so the SSH exit code propagates to the CI job.
|
|
|
|
.PARAMETER Sha
|
|
The commit SHA under test. HEAD is force-checked-out to it and verified.
|
|
|
|
.PARAMETER Mode
|
|
build | test | live.
|
|
|
|
.PARAMETER RepoRoot
|
|
The CI clone root. Defaults to C:\build\mxaccessgw-ci.
|
|
#>
|
|
[CmdletBinding()]
|
|
param(
|
|
[Parameter(Mandatory = $true)][string]$Sha,
|
|
[Parameter(Mandatory = $true)][ValidateSet('build', 'test', 'live')][string]$Mode,
|
|
[string]$RepoRoot = 'C:\build\mxaccessgw-ci'
|
|
)
|
|
|
|
Set-StrictMode -Version Latest
|
|
$ErrorActionPreference = 'Stop'
|
|
|
|
$lockFile = "$RepoRoot.ci-lock"
|
|
$logFile = Join-Path $RepoRoot 'ci.log'
|
|
$lockTimeout = [TimeSpan]::FromMinutes(20)
|
|
|
|
function Write-Log {
|
|
param([string]$Message)
|
|
$line = ('[{0}] {1}' -f (Get-Date -Format 'yyyy-MM-dd HH:mm:ss'), $Message)
|
|
Write-Host $line
|
|
# Best-effort file log for the "is the tier down?" diagnosis in docs/GatewayTesting.md.
|
|
try { Add-Content -Path $logFile -Value $line -ErrorAction SilentlyContinue } catch { }
|
|
}
|
|
|
|
# Run a native command and throw on nonzero exit. Native stderr (git progress) does NOT
|
|
# set $LASTEXITCODE, so it is not treated as failure — only a real nonzero exit is.
|
|
function Invoke-Native {
|
|
param([Parameter(Mandatory = $true)][string]$What, [Parameter(Mandatory = $true)][scriptblock]$Command)
|
|
Write-Log "RUN: $What"
|
|
& $Command
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "$What failed with exit code $LASTEXITCODE"
|
|
}
|
|
}
|
|
|
|
# Logs the host's memory/commit/disk headroom and any leftover test processes, so a
|
|
# host-resource failure (OOM in csc/MSBuild/protoc, 0xC0000142 process-start failures, a
|
|
# PowerShell that dies before logging) can be correlated with the host's state at the time
|
|
# rather than reconstructed afterwards from the event log (mxaccessgw#150). Never fatal.
|
|
function Write-HostResources {
|
|
param([string]$When)
|
|
try {
|
|
$os = Get-CimInstance -ClassName Win32_OperatingSystem
|
|
$drive = Get-PSDrive -Name ($RepoRoot.Substring(0, 1)) -ErrorAction Stop
|
|
$testhosts = @(Get-Process -Name 'testhost*' -ErrorAction SilentlyContinue).Count
|
|
Write-Log ('host ({0}): commit free {1:N1} of {2:N1} GB, physical free {3:N1} of {4:N1} GB, {5}: free {6:N1} GB, testhost processes {7}, up since {8:yyyy-MM-dd HH:mm}' -f
|
|
$When,
|
|
($os.FreeVirtualMemory / 1MB), ($os.TotalVirtualMemorySize / 1MB),
|
|
($os.FreePhysicalMemory / 1MB), ($os.TotalVisibleMemorySize / 1MB),
|
|
$drive.Name, ($drive.Free / 1GB), $testhosts, $os.LastBootUpTime)
|
|
}
|
|
catch {
|
|
Write-Log "host ($When): resource probe failed: $($_.Exception.Message)"
|
|
}
|
|
}
|
|
|
|
# --- acquire the worktree lock ---
|
|
# The lock is an exclusive (FileShare.None) handle on "$RepoRoot.ci-lock". It replaced a
|
|
# `New-Item -ItemType Directory` lock that was not mutually exclusive (PowerShell checks for
|
|
# the directory and then calls CreateDirectory, which succeeds on an existing one), so two
|
|
# runs starting together both got in and collided on .git/index.lock (mxaccessgw#148/#150).
|
|
# The OS releases the handle if the holder is killed, so a dead run cannot leave it stale.
|
|
# run-windev-ci.sh's bootstrap already holds this lock around its pre-hand-off fetch/checkout
|
|
# (MXGW_CI_LOCK_HELD=1); re-use it rather than re-acquire (and leave releasing to the
|
|
# bootstrap). A standalone/manual run (degraded mode) has no such env and self-locks here.
|
|
$deadline = (Get-Date).Add($lockTimeout)
|
|
$lock = $null
|
|
if ($env:MXGW_CI_LOCK_HELD -eq '1') {
|
|
Write-Log "Worktree lock already held by bootstrap (mode=$Mode sha=$Sha)"
|
|
}
|
|
else {
|
|
Write-Log "Acquiring worktree lock $lockFile (mode=$Mode sha=$Sha)"
|
|
while ($null -eq $lock) {
|
|
try {
|
|
$lock = [System.IO.File]::Open($lockFile, 'OpenOrCreate', 'ReadWrite', 'None')
|
|
}
|
|
catch {
|
|
if ((Get-Date) -ge $deadline) {
|
|
throw "Timed out after $($lockTimeout.TotalMinutes) min waiting for worktree lock $lockFile ($($_.Exception.Message)). Another CI run is holding it."
|
|
}
|
|
Start-Sleep -Seconds 10
|
|
}
|
|
}
|
|
}
|
|
|
|
Write-HostResources -When 'start'
|
|
|
|
try {
|
|
Push-Location $RepoRoot
|
|
try {
|
|
# Holding the lock proves no other CI git command is running in this clone, so a
|
|
# leftover index.lock can only be debris from a killed git; it would fail the checkout.
|
|
if (Test-Path '.git\index.lock') {
|
|
Write-Log 'Removing stale .git\index.lock left by a killed git'
|
|
Remove-Item -Force '.git\index.lock'
|
|
}
|
|
|
|
# Re-fetch + re-checkout under the lock so the tree is the requested SHA even when the
|
|
# bootstrap's checkout was skipped (standalone run) or left residue.
|
|
Invoke-Native 'git fetch origin' { git fetch --prune origin }
|
|
Invoke-Native "git checkout --force $Sha" { git checkout --force --detach $Sha }
|
|
# Drop any stray tracked/untracked residue from a prior run so the build is clean.
|
|
Invoke-Native 'git reset --hard' { git reset --hard $Sha }
|
|
Invoke-Native 'git clean -fdx (build outputs)' { git clean -fdx -e ci.log }
|
|
|
|
$head = (& git rev-parse HEAD).Trim()
|
|
if ($head -ne $Sha) {
|
|
throw "Worktree HEAD $head does not match requested SHA $Sha after checkout."
|
|
}
|
|
Write-Log "Worktree at $head"
|
|
|
|
$workerProj = 'src/ZB.MOM.WW.MxGateway.Worker/ZB.MOM.WW.MxGateway.Worker.csproj'
|
|
$workerTests = 'src/ZB.MOM.WW.MxGateway.Worker.Tests/ZB.MOM.WW.MxGateway.Worker.Tests.csproj'
|
|
$gatewayTests = 'src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj'
|
|
$integrationTests = 'src/ZB.MOM.WW.MxGateway.IntegrationTests/ZB.MOM.WW.MxGateway.IntegrationTests.csproj'
|
|
|
|
# build: always. The x86/net48 compile is the floor guard (catches CS0246 on
|
|
# unregenerated Generated/, x86-only breaks) even when the test step is demoted.
|
|
Invoke-Native 'x86 Worker build' { dotnet build $workerProj -c Release -p:Platform=x86 }
|
|
|
|
if ($Mode -eq 'test' -or $Mode -eq 'live') {
|
|
Invoke-Native 'x86 Worker.Tests' { dotnet test $workerTests -c Release -p:Platform=x86 }
|
|
}
|
|
|
|
if ($Mode -eq 'live') {
|
|
# Full-solution build surfaces Windows-only analyzer diagnostics (xUnit1030 etc.)
|
|
# that never fire on the macOS/Linux NonWindows build.
|
|
Invoke-Native 'full slnx build' { dotnet build src/ZB.MOM.WW.MxGateway.slnx -c Release }
|
|
|
|
Invoke-Native 'gateway tests on Windows' {
|
|
dotnet test $gatewayTests -c Release --no-build
|
|
}
|
|
|
|
$env:MXGATEWAY_RUN_LIVE_MXACCESS_TESTS = '1'
|
|
try {
|
|
Invoke-Native 'live-MXAccess smoke' {
|
|
dotnet test $integrationTests -c Release --filter 'FullyQualifiedName~WorkerLiveMxAccessSmokeTests'
|
|
}
|
|
}
|
|
finally {
|
|
Remove-Item Env:\MXGATEWAY_RUN_LIVE_MXACCESS_TESTS -ErrorAction SilentlyContinue
|
|
}
|
|
}
|
|
|
|
Write-Log "SUCCESS mode=$Mode sha=$Sha"
|
|
}
|
|
finally {
|
|
Pop-Location
|
|
}
|
|
}
|
|
catch {
|
|
Write-Log "FAILURE mode=$Mode sha=$Sha : $($_.Exception.Message)"
|
|
Write-HostResources -When 'failure'
|
|
throw
|
|
}
|
|
finally {
|
|
if ($null -ne $lock) {
|
|
$lock.Dispose()
|
|
Write-Log "Released worktree lock $lockFile"
|
|
}
|
|
}
|