fix(archreview): CI pipeline + codegen freshness guards (IPC-01/09/19/20, TST-03)
- IPC-01: regenerate the stale client descriptor set and add ClientProtoInputTests (semantic, protoc-free) so a missing contract symbol fails the build. - IPC-20: make publish-client-proto-inputs.ps1 -Check source_code_info-normalized so it is protoc-version tolerant. - IPC-19: document + guard the "Generated/ must be committed for net48" rule (docs/Contracts.md, csproj comment, check-codegen.ps1). - IPC-09: harden the per-client generate-proto scripts (PATH resolution + pinned grpcio/protobuf/java version asserts) so regeneration is reproducible. - TST-03: add .gitea/workflows/ci.yml (portable/java/windows/live jobs) running the build, tests, client checks, and the codegen guards. - Also: check-codegen.ps1 Check 3 guards the CLI-02 vendored Rust protos against drift. archreview: IPC-01/09/19/20 Done, TST-03 In review (pipeline authored + validated, not yet run on a Gitea runner). Verified on macOS: NonWindows build clean, ClientProtoInputTests 5/5, -Check exit 0.
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env pwsh
|
||||
# Codegen freshness guard for CI (IPC-01, IPC-19, IPC-20, CLI-02).
|
||||
#
|
||||
# Three checks, all Linux/macOS-runnable (no Server build, no x86 worker):
|
||||
# 1. Published client descriptor set matches the current .proto sources (delegates to
|
||||
# publish-client-proto-inputs.ps1 -Check, which normalizes source_code_info so it is
|
||||
# protoc-version tolerant).
|
||||
# 2. The committed C# under Contracts/Generated matches a fresh regeneration. Grpc.Tools is
|
||||
# pinned in the contracts csproj, so a clean checkout regenerates byte-identical output; a
|
||||
# non-empty git diff means a .proto was edited without regenerating and committing Generated/
|
||||
# (which breaks the net48 worker build with CS0246 — see docs/Contracts.md).
|
||||
# 3. The Rust crate's vendored protos (clients/rust/protos/*.proto — build inputs that make the
|
||||
# crate buildable outside the repo, CLI-02) are byte-identical to the canonical Contracts
|
||||
# protos. A drift means a .proto was edited without refreshing the vendored copies, which would
|
||||
# publish a stale wire contract to crate consumers while the in-repo build stays correct.
|
||||
#
|
||||
# The x86 Worker + Worker.Tests are Windows-only and are guarded by the Windows CI job, not here.
|
||||
|
||||
[CmdletBinding()]
|
||||
param()
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$repoRoot = Resolve-Path (Join-Path $PSScriptRoot '..')
|
||||
$generatedDir = Join-Path $repoRoot 'src/ZB.MOM.WW.MxGateway.Contracts/Generated'
|
||||
$contractsProject = Join-Path $repoRoot 'src/ZB.MOM.WW.MxGateway.Contracts/ZB.MOM.WW.MxGateway.Contracts.csproj'
|
||||
$failures = New-Object System.Collections.Generic.List[string]
|
||||
|
||||
Write-Host '== Check 1/2: client descriptor set freshness =='
|
||||
try {
|
||||
& (Join-Path $PSScriptRoot 'publish-client-proto-inputs.ps1') -Check
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
$failures.Add('Client descriptor set is stale (publish-client-proto-inputs.ps1 -Check failed).')
|
||||
}
|
||||
}
|
||||
catch {
|
||||
$failures.Add("Descriptor freshness check failed: $($_.Exception.Message)")
|
||||
}
|
||||
|
||||
Write-Host ''
|
||||
Write-Host '== Check 2/2: Contracts/Generated matches a fresh regeneration =='
|
||||
try {
|
||||
# Force a full regeneration: Grpc.Tools skips regen when the committed .cs look up to date, so
|
||||
# remove them first (the documented "del Generated/*.cs to force regen" trick).
|
||||
if (Test-Path $generatedDir) {
|
||||
Get-ChildItem -Path $generatedDir -Filter '*.cs' -Recurse -File | Remove-Item -Force
|
||||
}
|
||||
|
||||
& dotnet build $contractsProject -c Release --nologo | Out-Host
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
$failures.Add('Contracts project failed to build during codegen check.')
|
||||
}
|
||||
|
||||
$diff = (& git -C $repoRoot status --porcelain -- 'src/ZB.MOM.WW.MxGateway.Contracts/Generated').Trim()
|
||||
if (-not [string]::IsNullOrEmpty($diff)) {
|
||||
Write-Host $diff
|
||||
$failures.Add('Contracts/Generated differs from a fresh regeneration. Run `dotnet build` on the contracts project and commit Generated/ (required for the net48 worker build).')
|
||||
}
|
||||
}
|
||||
catch {
|
||||
$failures.Add("Generated codegen check failed: $($_.Exception.Message)")
|
||||
}
|
||||
|
||||
Write-Host ''
|
||||
Write-Host '== Check 3/3: Rust vendored protos match canonical Contracts protos =='
|
||||
try {
|
||||
$canonicalProtoDir = Join-Path $repoRoot 'src/ZB.MOM.WW.MxGateway.Contracts/Protos'
|
||||
$vendoredProtoDir = Join-Path $repoRoot 'clients/rust/protos'
|
||||
foreach ($vendored in Get-ChildItem -Path $vendoredProtoDir -Filter '*.proto' -File) {
|
||||
$canonical = Join-Path $canonicalProtoDir $vendored.Name
|
||||
if (-not (Test-Path $canonical)) {
|
||||
$failures.Add("Vendored Rust proto has no canonical counterpart: $($vendored.Name).")
|
||||
continue
|
||||
}
|
||||
$vendoredHash = (Get-FileHash -Algorithm SHA256 $vendored.FullName).Hash
|
||||
$canonicalHash = (Get-FileHash -Algorithm SHA256 $canonical).Hash
|
||||
if ($vendoredHash -ne $canonicalHash) {
|
||||
$failures.Add("Rust vendored proto drifted from canonical: clients/rust/protos/$($vendored.Name). Refresh it from src/ZB.MOM.WW.MxGateway.Contracts/Protos/$($vendored.Name).")
|
||||
}
|
||||
}
|
||||
}
|
||||
catch {
|
||||
$failures.Add("Rust vendored proto check failed: $($_.Exception.Message)")
|
||||
}
|
||||
|
||||
Write-Host ''
|
||||
if ($failures.Count -gt 0) {
|
||||
Write-Host 'Codegen freshness check FAILED:' -ForegroundColor Red
|
||||
foreach ($failure in $failures) {
|
||||
Write-Host " - $failure" -ForegroundColor Red
|
||||
}
|
||||
exit 1
|
||||
}
|
||||
|
||||
Write-Host 'Codegen freshness check passed.' -ForegroundColor Green
|
||||
@@ -6,23 +6,46 @@ param(
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
# Pinned protoc toolchain version used to generate the committed client descriptor set.
|
||||
# See docs/ToolchainLinks.md. Generating (committing) the canonical artifact must use this
|
||||
# exact version so the checked-in bytes are reproducible. The -Check comparison is made
|
||||
# tolerant of protoc-version encoding drift (chiefly source_code_info, see IPC-20) by
|
||||
# normalizing both the committed and the freshly built descriptor through the *same* protoc
|
||||
# with source info stripped, so it does not false-fail across protoc releases.
|
||||
$PinnedProtocVersion = "34.1"
|
||||
|
||||
$repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..")
|
||||
$protoRoot = Join-Path $repoRoot "src/ZB.MOM.WW.MxGateway.Contracts/Protos"
|
||||
$manifestPath = Join-Path $repoRoot "clients/proto/proto-inputs.json"
|
||||
$descriptorPath = Join-Path $repoRoot "clients/proto/descriptors/mxaccessgw-client-v1.protoset"
|
||||
$protoFiles = @("mxaccess_gateway.proto", "mxaccess_worker.proto", "galaxy_repository.proto")
|
||||
|
||||
function Resolve-Protoc {
|
||||
$pathCommand = Get-Command "protoc.exe" -ErrorAction SilentlyContinue
|
||||
if ($null -ne $pathCommand) {
|
||||
return $pathCommand.Source
|
||||
# Prefer protoc on PATH (protoc on Linux/macOS, protoc.exe on Windows), then fall back to
|
||||
# the documented winget install location on Windows.
|
||||
foreach ($name in @("protoc", "protoc.exe")) {
|
||||
$pathCommand = Get-Command $name -ErrorAction SilentlyContinue
|
||||
if ($null -ne $pathCommand) {
|
||||
return $pathCommand.Source
|
||||
}
|
||||
}
|
||||
|
||||
$documentedPath = Join-Path $env:LOCALAPPDATA "Microsoft/WinGet/Packages/Google.Protobuf_Microsoft.Winget.Source_8wekyb3d8bbwe/bin/protoc.exe"
|
||||
if (Test-Path $documentedPath) {
|
||||
return $documentedPath
|
||||
if ($env:LOCALAPPDATA) {
|
||||
$documentedPath = Join-Path $env:LOCALAPPDATA "Microsoft/WinGet/Packages/Google.Protobuf_Microsoft.Winget.Source_8wekyb3d8bbwe/bin/protoc.exe"
|
||||
if (Test-Path $documentedPath) {
|
||||
return $documentedPath
|
||||
}
|
||||
}
|
||||
|
||||
throw "Could not find protoc.exe. See docs/ToolchainLinks.md for the documented protobuf toolchain path."
|
||||
throw "Could not find protoc on PATH. See docs/ToolchainLinks.md for the documented protobuf toolchain (protoc $PinnedProtocVersion)."
|
||||
}
|
||||
|
||||
function Get-ProtocVersion {
|
||||
param([string]$Protoc)
|
||||
|
||||
# `protoc --version` prints e.g. "libprotoc 34.1".
|
||||
$raw = (& $Protoc --version) | Select-Object -First 1
|
||||
return ($raw -replace '^libprotoc\s+', '').Trim()
|
||||
}
|
||||
|
||||
function Ensure-Directory {
|
||||
@@ -33,7 +56,29 @@ function Ensure-Directory {
|
||||
}
|
||||
}
|
||||
|
||||
function Compare-FileBytes {
|
||||
function New-SourceInfoFreeDescriptor {
|
||||
# Re-emits an existing descriptor set with source_code_info stripped, by round-tripping it
|
||||
# through protoc with --descriptor_set_in and *without* --include_source_info. Because both
|
||||
# the committed and freshly built descriptors are normalized through the same protoc binary,
|
||||
# any protoc-version-specific encoding cancels out and the byte comparison is stable.
|
||||
param(
|
||||
[string]$Protoc,
|
||||
[string]$InputDescriptor,
|
||||
[string]$OutputDescriptor
|
||||
)
|
||||
|
||||
& $Protoc `
|
||||
"--descriptor_set_in=$InputDescriptor" `
|
||||
"--include_imports" `
|
||||
"--descriptor_set_out=$OutputDescriptor" `
|
||||
@protoFiles
|
||||
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "protoc normalization (--descriptor_set_in) failed with exit code $LASTEXITCODE."
|
||||
}
|
||||
}
|
||||
|
||||
function Test-FileBytesEqual {
|
||||
param(
|
||||
[string]$ExpectedPath,
|
||||
[string]$ActualPath
|
||||
@@ -66,31 +111,67 @@ foreach ($output in $manifest.generatedOutputs.PSObject.Properties.Value) {
|
||||
}
|
||||
|
||||
$protoc = Resolve-Protoc
|
||||
$outputPath = $descriptorPath
|
||||
if ($Check) {
|
||||
$outputPath = Join-Path ([System.IO.Path]::GetTempPath()) ("mxaccessgw-client-v1-" + [System.Guid]::NewGuid().ToString("N") + ".protoset")
|
||||
}
|
||||
$protocVersion = Get-ProtocVersion $protoc
|
||||
|
||||
if ($Check) {
|
||||
# Version drift no longer produces a false "stale" failure: the comparison is normalized
|
||||
# below. A mismatch is surfaced as a warning so it is visible without blocking the gate.
|
||||
if ($protocVersion -ne $PinnedProtocVersion) {
|
||||
Write-Warning "protoc version '$protocVersion' differs from the pinned '$PinnedProtocVersion'. The freshness comparison is normalized and tolerant of this, but regeneration must use the pinned version (see docs/ClientProtoGeneration.md)."
|
||||
}
|
||||
|
||||
$freshDescriptor = Join-Path ([System.IO.Path]::GetTempPath()) ("mxaccessgw-client-v1-fresh-" + [System.Guid]::NewGuid().ToString("N") + ".protoset")
|
||||
$committedNormalized = Join-Path ([System.IO.Path]::GetTempPath()) ("mxaccessgw-client-v1-committed-" + [System.Guid]::NewGuid().ToString("N") + ".protoset")
|
||||
|
||||
try {
|
||||
# Fresh descriptor from the current .proto sources, source info omitted (already normalized).
|
||||
& $protoc `
|
||||
"--proto_path=$protoRoot" `
|
||||
"--include_imports" `
|
||||
"--descriptor_set_out=$freshDescriptor" `
|
||||
@protoFiles
|
||||
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "protoc failed with exit code $LASTEXITCODE."
|
||||
}
|
||||
|
||||
if (-not (Test-Path $descriptorPath)) {
|
||||
throw "Committed descriptor '$descriptorPath' does not exist. Run scripts/publish-client-proto-inputs.ps1 to create it."
|
||||
}
|
||||
|
||||
# Normalize the committed descriptor (which carries source info) through the same protoc.
|
||||
New-SourceInfoFreeDescriptor -Protoc $protoc -InputDescriptor $descriptorPath -OutputDescriptor $committedNormalized
|
||||
|
||||
if (-not (Test-FileBytesEqual -ExpectedPath $committedNormalized -ActualPath $freshDescriptor)) {
|
||||
throw "Client proto descriptor is stale. Run scripts/publish-client-proto-inputs.ps1 with the pinned protoc ($PinnedProtocVersion) and commit the updated descriptor."
|
||||
}
|
||||
|
||||
Write-Host "Client proto descriptor is up to date."
|
||||
}
|
||||
finally {
|
||||
foreach ($temp in @($freshDescriptor, $committedNormalized)) {
|
||||
if (Test-Path $temp) {
|
||||
Remove-Item -LiteralPath $temp
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
# Generating the canonical committed artifact must use the pinned protoc so the bytes are reproducible.
|
||||
if ($protocVersion -ne $PinnedProtocVersion) {
|
||||
throw "protoc version '$protocVersion' does not match the pinned toolchain version '$PinnedProtocVersion'. Install the pinned protoc (see docs/ToolchainLinks.md) before regenerating the committed descriptor, or run with -Check to only verify freshness."
|
||||
}
|
||||
|
||||
try {
|
||||
& $protoc `
|
||||
"--proto_path=$protoRoot" `
|
||||
"--include_imports" `
|
||||
"--include_source_info" `
|
||||
"--descriptor_set_out=$outputPath" `
|
||||
"mxaccess_gateway.proto" `
|
||||
"mxaccess_worker.proto" `
|
||||
"galaxy_repository.proto"
|
||||
"--descriptor_set_out=$descriptorPath" `
|
||||
@protoFiles
|
||||
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "protoc failed with exit code $LASTEXITCODE."
|
||||
}
|
||||
|
||||
if ($Check -and -not (Compare-FileBytes -ExpectedPath $descriptorPath -ActualPath $outputPath)) {
|
||||
throw "Client proto descriptor is stale. Run scripts/publish-client-proto-inputs.ps1 and commit the updated descriptor."
|
||||
}
|
||||
}
|
||||
finally {
|
||||
if ($Check -and (Test-Path $outputPath)) {
|
||||
Remove-Item -LiteralPath $outputPath
|
||||
}
|
||||
Write-Host "Wrote client proto descriptor to $descriptorPath (protoc $protocVersion)."
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user