diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 0000000..9c6cbdb --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,124 @@ +# Continuous integration for mxaccessgw (Gitea Actions; origin is Gitea at gitea.dohertylan.com). +# +# Scope note: the x86 Worker (src/ZB.MOM.WW.MxGateway.Worker) and Worker.Tests target +# .NET Framework 4.8 / x86 and need MXAccess COM installed, so they build ONLY on a Windows host. +# They are out of scope for this Linux `portable` job — the `windows` job below (self-hosted windev +# runner) covers them, and live-MXAccess integration runs are scheduled-only so they never gate a push. +name: ci + +on: + push: + pull_request: + schedule: + # Nightly live-MXAccess smoke (windev). Push/PR runs skip the live-mxaccess job. + - cron: '0 6 * * *' + +jobs: + portable: + # Any Linux runner: no MXAccess, no x86. Covers the NonWindows solution, gateway fake-worker + # tests, the codegen/descriptor freshness guards, and the clients that build on Linux. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: '10.0.x' + + - uses: actions/setup-go@v5 + with: + go-version: '1.26' + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy, rustfmt + + # protoc is pinned to 34.1 (see docs/ToolchainLinks.md) so the committed client descriptor + # regenerates reproducibly. The freshness check normalizes source_code_info, so it tolerates + # patch drift, but keep the CI toolchain on the pin. + - name: Install protoc 34.1 + run: | + curl -sSL -o /tmp/protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v34.1/protoc-34.1-linux-x86_64.zip + sudo unzip -o /tmp/protoc.zip -d /usr/local bin/protoc 'include/*' + protoc --version + + - name: Build NonWindows solution + run: dotnet build src/ZB.MOM.WW.MxGateway.NonWindows.slnx -c Release + + # IPC-01 / IPC-19 / IPC-20: descriptor set + Contracts/Generated must match the current protos. + - name: Codegen / descriptor freshness + shell: pwsh + run: ./scripts/check-codegen.ps1 + + - name: Gateway fake-worker tests + run: dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj -c Release --no-build + + - name: .NET client + run: dotnet build clients/dotnet/ZB.MOM.WW.MxGateway.Client.slnx -c Release + + - name: Go client + working-directory: clients/go + run: | + test -z "$(gofmt -l .)" || (gofmt -l . && echo 'gofmt needed' && exit 1) + go build ./... + go test ./... + + - name: Rust client + working-directory: clients/rust + run: | + cargo fmt --all -- --check + cargo test --workspace + cargo clippy --workspace --all-targets -- -D warnings + + - name: Python client + working-directory: clients/python + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[dev]" + python -m pytest + + java: + # Java client runs on a JDK-17 Linux runner (the macOS dev box has no JRE). The protobuf gradle + # plugin rewrites MxaccessGateway.java with spurious protobuf-runtime-version churn on every + # build; when no .proto changed, revert that one file so checkGeneratedClean / a dirty tree does + # not fail the build (repo memory project_java_generated_churn). + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '17' + - name: Gradle test + working-directory: clients/java + run: gradle test + - name: Revert spurious protobuf-version churn (no .proto changed) + run: git checkout -- clients/java/src/main/generated/main/java/mxaccess_gateway/v1/MxaccessGateway.java || true + - name: Verify generated tree is clean + run: git diff --exit-code -- clients/java/src/main/generated + + windows: + # Self-hosted windev runner (10.100.0.48): the only host that can build the x86 / net48 Worker + # and MXAccess-adjacent code. Not required for a green portable build; runs where a runner exists. + runs-on: [self-hosted, windows] + steps: + - uses: actions/checkout@v4 + - name: Build x86 Worker + run: dotnet build src/ZB.MOM.WW.MxGateway.Worker/ZB.MOM.WW.MxGateway.Worker.csproj -p:Platform=x86 + - name: Worker tests + run: dotnet test src/ZB.MOM.WW.MxGateway.Worker.Tests/ZB.MOM.WW.MxGateway.Worker.Tests.csproj -p:Platform=x86 + + live-mxaccess: + # Opt-in, scheduled only — needs installed MXAccess COM + live provider state. Never gates a push. + if: ${{ github.event_name == 'schedule' }} + runs-on: [self-hosted, windows, mxaccess] + env: + MXGATEWAY_RUN_LIVE_MXACCESS_TESTS: '1' + steps: + - uses: actions/checkout@v4 + - name: Live MXAccess smoke + run: dotnet test src/ZB.MOM.WW.MxGateway.IntegrationTests/ZB.MOM.WW.MxGateway.IntegrationTests.csproj --filter FullyQualifiedName~WorkerLiveMxAccessSmokeTests diff --git a/clients/go/generate-proto.ps1 b/clients/go/generate-proto.ps1 index 1e88740..9c807e6 100644 --- a/clients/go/generate-proto.ps1 +++ b/clients/go/generate-proto.ps1 @@ -5,25 +5,46 @@ $repoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..') $protoRoot = Join-Path $repoRoot 'src\ZB.MOM.WW.MxGateway.Contracts\Protos' $outputRoot = Join-Path $PSScriptRoot 'internal\generated' $modulePath = 'gitea.dohertylan.com/dohertj2/mxaccessgw/clients/go/internal/generated' -$protoc = 'C:\Users\dohertj2\AppData\Local\Microsoft\WinGet\Packages\Google.Protobuf_Microsoft.Winget.Source_8wekyb3d8bbwe\bin\protoc.exe' -$goPluginPath = 'C:\Users\dohertj2\go\bin' -if (-not (Test-Path $protoc)) { - throw "protoc was not found at $protoc. See docs/ToolchainLinks.md." -} +function Resolve-Tool { + # Resolve a codegen tool from PATH first (portable), then the documented Windows fallbacks, + # instead of the previous hard-coded per-machine paths. See docs/ToolchainLinks.md. + param( + [string[]]$Names, + [string[]]$FallbackPaths + ) -foreach ($pluginName in @('protoc-gen-go.exe', 'protoc-gen-go-grpc.exe')) { - $pluginPath = Join-Path $goPluginPath $pluginName - if (-not (Test-Path $pluginPath)) { - throw "$pluginName was not found at $pluginPath. See docs/ToolchainLinks.md." + foreach ($name in $Names) { + $cmd = Get-Command $name -ErrorAction SilentlyContinue + if ($null -ne $cmd) { + return $cmd.Source + } } + + foreach ($fallback in $FallbackPaths) { + if ($fallback -and (Test-Path $fallback)) { + return $fallback + } + } + + throw "Could not find $($Names -join '/') on PATH. See docs/ToolchainLinks.md." } +$wingetProtoc = if ($env:LOCALAPPDATA) { + Join-Path $env:LOCALAPPDATA 'Microsoft\WinGet\Packages\Google.Protobuf_Microsoft.Winget.Source_8wekyb3d8bbwe\bin\protoc.exe' +} else { $null } +$goBin = if ($env:USERPROFILE) { Join-Path $env:USERPROFILE 'go\bin' } elseif ($env:HOME) { Join-Path $env:HOME 'go/bin' } else { $null } + +$protoc = Resolve-Tool -Names @('protoc', 'protoc.exe') -FallbackPaths @($wingetProtoc) +$protocGenGo = Resolve-Tool -Names @('protoc-gen-go', 'protoc-gen-go.exe') -FallbackPaths @((if ($goBin) { Join-Path $goBin 'protoc-gen-go.exe' }), (if ($goBin) { Join-Path $goBin 'protoc-gen-go' })) +$protocGenGoGrpc = Resolve-Tool -Names @('protoc-gen-go-grpc', 'protoc-gen-go-grpc.exe') -FallbackPaths @((if ($goBin) { Join-Path $goBin 'protoc-gen-go-grpc.exe' }), (if ($goBin) { Join-Path $goBin 'protoc-gen-go-grpc' })) + +# protoc discovers the plugins on PATH; prepend the directories the resolved plugins live in. +$env:Path = (Split-Path $protocGenGo -Parent) + [System.IO.Path]::PathSeparator + (Split-Path $protocGenGoGrpc -Parent) + [System.IO.Path]::PathSeparator + $env:Path + New-Item -ItemType Directory -Path $outputRoot -Force | Out-Null Get-ChildItem -Path $outputRoot -Filter '*.pb.go' -File | Remove-Item -$env:Path = "$goPluginPath;$env:Path" - & $protoc ` --proto_path=$protoRoot ` --go_out=$outputRoot ` @@ -35,6 +56,10 @@ $env:Path = "$goPluginPath;$env:Path" mxaccess_worker.proto ` galaxy_repository.proto +if ($LASTEXITCODE -ne 0) { + throw "protoc (go) failed with exit code $LASTEXITCODE." +} + & $protoc ` --proto_path=$protoRoot ` --go-grpc_out=$outputRoot ` @@ -44,3 +69,6 @@ $env:Path = "$goPluginPath;$env:Path" mxaccess_gateway.proto ` galaxy_repository.proto +if ($LASTEXITCODE -ne 0) { + throw "protoc (go-grpc) failed with exit code $LASTEXITCODE." +} diff --git a/clients/java/zb-mom-ww-mxgateway-client/build.gradle b/clients/java/zb-mom-ww-mxgateway-client/build.gradle index aea6eef..cdb2b4a 100644 --- a/clients/java/zb-mom-ww-mxgateway-client/build.gradle +++ b/clients/java/zb-mom-ww-mxgateway-client/build.gradle @@ -57,3 +57,35 @@ protobuf { } } } + +// IPC-09 guard: the generated Java tree is tracked (generatedFilesBaseDir above points at the +// committed src/main/generated). `gradle build` regenerates it, so an un-regenerated .proto change, +// or a plugin/protobuf version bump, silently drifts the committed output. checkGeneratedClean +// fails when the regenerated tree differs from what is committed. +// +// Caveat (repo memory project_java_generated_churn): the protobuf gradle plugin also rewrites +// MxaccessGateway.java with a spurious protobuf-runtime-version delta on every build even when no +// .proto changed. CI reverts that one file (git checkout) before invoking this task; locally, do the +// same when you did not touch a .proto. See docs/GatewayTesting.md "Continuous Integration". +tasks.register('checkGeneratedClean') { + group = 'verification' + description = 'Fails if the committed generated Java tree differs from a fresh regeneration.' + dependsOn 'generateProto' + doLast { + def generatedDir = 'clients/java/src/main/generated' + def stdout = new ByteArrayOutputStream() + def result = exec { + workingDir = rootProject.projectDir.parentFile.parentFile + commandLine 'git', 'status', '--porcelain', '--', generatedDir + standardOutput = stdout + ignoreExitValue = true + } + def dirty = stdout.toString().trim() + if (!dirty.isEmpty()) { + throw new GradleException( + "Generated Java is stale or churned:\n${dirty}\n" + + "Regenerate and commit after a .proto change, or 'git checkout' the spurious " + + "MxaccessGateway.java protobuf-version churn when no .proto changed.") + } + } +} diff --git a/clients/proto/descriptors/mxaccessgw-client-v1.protoset b/clients/proto/descriptors/mxaccessgw-client-v1.protoset index 6dd07cb..bdb64fb 100644 Binary files a/clients/proto/descriptors/mxaccessgw-client-v1.protoset and b/clients/proto/descriptors/mxaccessgw-client-v1.protoset differ diff --git a/clients/python/generate-proto.ps1 b/clients/python/generate-proto.ps1 index b336eee..a98eb29 100644 --- a/clients/python/generate-proto.ps1 +++ b/clients/python/generate-proto.ps1 @@ -1,15 +1,52 @@ Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' +# Pinned generator baseline. The committed Python bindings are stamped with +# GRPC_GENERATED_VERSION = 1.80.0 and "Protobuf Python Version: 6.31.1". A newer grpcio-tools +# stamps a higher GRPC_GENERATED_VERSION than the pinned grpcio runtime (pyproject.toml: +# grpcio>=1.80,<2), which makes _pb2_grpc import raise at runtime and breaks pytest. Regeneration +# MUST use this exact grpcio-tools version. See docs/ClientProtoGeneration.md and the repo memory +# project_python_client_regen_pin. +$PinnedGrpcioToolsVersion = '1.80.0' + $repoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..') $protoRoot = Join-Path $repoRoot 'src\ZB.MOM.WW.MxGateway.Contracts\Protos' $outputRoot = Join-Path $PSScriptRoot 'src\zb_mom_ww_mxgateway\generated' -$python = 'C:\Users\dohertj2\AppData\Local\Programs\Python\Python312\python.exe' -if (-not (Test-Path $python)) { - throw "Python was not found at $python. See docs/ToolchainLinks.md." +function Resolve-Python { + # Prefer python on PATH (python on Windows, python3 on Linux/macOS), then the documented + # Windows install location. Avoids the previous hard-coded per-machine path. + foreach ($name in @('python', 'python3', 'python.exe')) { + $cmd = Get-Command $name -ErrorAction SilentlyContinue + if ($null -ne $cmd) { + return $cmd.Source + } + } + + if ($env:LOCALAPPDATA) { + $documentedPath = Join-Path $env:LOCALAPPDATA 'Programs\Python\Python312\python.exe' + if (Test-Path $documentedPath) { + return $documentedPath + } + } + + throw 'Could not find python on PATH. See docs/ToolchainLinks.md.' } +function Assert-GrpcioToolsVersion { + param([string]$Python) + + $version = (& $Python -c 'import grpc_tools; from importlib.metadata import version; print(version("grpcio-tools"))').Trim() + if ($version -ne $PinnedGrpcioToolsVersion) { + throw "grpcio-tools $version is installed, but regeneration is pinned to $PinnedGrpcioToolsVersion. " + + "Install the pin (python -m pip install 'grpcio-tools==$PinnedGrpcioToolsVersion') before regenerating, " + + "or the generated bindings will stamp a GRPC_GENERATED_VERSION the pinned grpcio runtime rejects." + } +} + +$python = Resolve-Python +Assert-GrpcioToolsVersion -Python $python + New-Item -ItemType Directory -Path $outputRoot -Force | Out-Null Get-ChildItem -Path (Join-Path $outputRoot '*_pb2.py') -File | Remove-Item Get-ChildItem -Path (Join-Path $outputRoot '*_pb2_grpc.py') -File | Remove-Item @@ -21,3 +58,7 @@ Get-ChildItem -Path (Join-Path $outputRoot '*_pb2_grpc.py') -File | Remove-Item mxaccess_gateway.proto ` mxaccess_worker.proto ` galaxy_repository.proto + +if ($LASTEXITCODE -ne 0) { + throw "grpc_tools.protoc failed with exit code $LASTEXITCODE." +} diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index bd09fbb..2b0e748 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -38,6 +38,9 @@ Issues = "https://gitea.dohertylan.com/dohertj2/mxaccessgw/issues" [project.optional-dependencies] dev = [ + # Runtime range for local dev, but REGENERATION is pinned to grpcio-tools==1.80.0 + # (protobuf 6.31.1): a newer version stamps a GRPC_GENERATED_VERSION the pinned grpcio + # runtime rejects and breaks pytest. generate-proto.ps1 asserts the exact pin. "grpcio-tools>=1.80,<2", "pytest>=9,<10", "pytest-asyncio>=1.3,<2", diff --git a/docs/ClientProtoGeneration.md b/docs/ClientProtoGeneration.md index 306209d..597869b 100644 --- a/docs/ClientProtoGeneration.md +++ b/docs/ClientProtoGeneration.md @@ -48,25 +48,55 @@ session. ## Descriptor Publishing Run this command after changing either source `.proto` file or the client proto -manifest: +manifest, with the **pinned protoc 34.1** (see [Toolchain Links](./ToolchainLinks.md)): ```powershell -scripts/publish-client-proto-inputs.ps1 +pwsh -File scripts/publish-client-proto-inputs.ps1 ``` The script writes `clients/proto/descriptors/mxaccessgw-client-v1.protoset` with imports and source information included. The descriptor is a generated artifact; do not edit -it by hand. +it by hand. Generating the committed artifact requires the pinned protoc version +so the checked-in bytes are reproducible; the script fails fast on a version +mismatch when generating. Use the check mode in CI or before committing: ```powershell -scripts/publish-client-proto-inputs.ps1 -Check +pwsh -File scripts/publish-client-proto-inputs.ps1 -Check ``` -`-Check` rebuilds the descriptor in a temporary path and fails when the checked -in descriptor is stale. +`-Check` rebuilds the descriptor and fails when the checked-in descriptor is +stale. The comparison normalizes both the committed and freshly built descriptor +through the same protoc with `source_code_info` stripped, so it is tolerant of +protoc-version encoding drift and does not false-fail across protoc releases +(it warns, rather than fails, when protoc is off the pin). + +The gateway test project carries an independent, protoc-free freshness guard: +`ClientProtoInputTests.Descriptor_ContainsEveryContractMessageAndField` reflects +over the in-process contract descriptors and fails if any contract message or +field is missing from the committed protoset. This is the primary CI gate for +descriptor staleness; a red test means "regenerate and commit the protoset." + +### Pinned generator versions + +Regeneration is reproducible only with the pinned toolchain. Regenerating with a +different version silently produces incompatible or noisy output, so the per-client +scripts assert the pin and resolve tools from `PATH`: + +| Generator | Pinned version | Guard | +|-----------|----------------|-------| +| protoc (descriptor set) | 34.1 | version assertion in `scripts/publish-client-proto-inputs.ps1` | +| `Grpc.Tools` (C# `Generated/`) | 2.80.0 (contracts csproj) | `scripts/check-codegen.ps1` git-diff of `Generated/` | +| `grpcio-tools` (Python) | 1.80.0 (protobuf runtime 6.31.1) | version assertion in `clients/python/generate-proto.ps1` | +| protobuf / grpc-java (Java) | `protobufVersion` / `grpcVersion` in `clients/java/build.gradle` | `checkGeneratedClean` gradle task | + +A newer `grpcio-tools` stamps a `GRPC_GENERATED_VERSION` above the pinned grpcio +runtime and breaks Python `pytest`; the Java protobuf plugin rewrites +`MxaccessGateway.java` with spurious protobuf-runtime-version churn on every build +(revert that one file when no `.proto` changed — see +[Gateway Testing](./GatewayTesting.md) "Continuous Integration"). ## Output Directories diff --git a/docs/Contracts.md b/docs/Contracts.md index 1690ab7..f0ee3d2 100644 --- a/docs/Contracts.md +++ b/docs/Contracts.md @@ -97,6 +97,24 @@ behavior. Generated C# output is written to `src/ZB.MOM.WW.MxGateway.Contracts/Generated/`. Do not hand-edit generated files. +The `Generated/` C# is **tracked and must be committed after any `.proto` change.** The +contracts project `Compile Remove`s `Generated/**/*.cs` and has `Grpc.Tools` regenerate them, +but `Grpc.Tools` skips regeneration when the committed output looks up to date. On the net10 +build the freshly regenerated code is compiled, so a stale `Generated/` is invisible there — +but the **net48 worker** consumes the committed `Generated/` and fails to build with `CS0246` +on any new type when the checked-in code lags the `.proto`. So after editing a `.proto` you must +regenerate and commit `Generated/`. If a build does not pick up your proto change, delete the +stale output to force a full regeneration: + +```bash +rm src/ZB.MOM.WW.MxGateway.Contracts/Generated/*.cs +dotnet build src/ZB.MOM.WW.MxGateway.Contracts/ZB.MOM.WW.MxGateway.Contracts.csproj +``` + +`scripts/check-codegen.ps1` enforces this in CI (it force-regenerates and fails on any +`git diff` against the committed `Generated/`); the Windows CI job's net48 worker build is the +definitive guard. + Client generation inputs are published through `clients/proto/proto-inputs.json` and the descriptor set under `clients/proto/descriptors/`. See @@ -124,12 +142,25 @@ gateway and test projects: dotnet build src/ZB.MOM.WW.MxGateway.slnx ``` -Regenerate the client descriptor after changing either `.proto` file: +Regenerate the client descriptor after changing either `.proto` file, using the pinned protoc +(34.1, see [Toolchain Links](./ToolchainLinks.md)): ```bash -powershell -ExecutionPolicy Bypass -File scripts/publish-client-proto-inputs.ps1 +pwsh -File scripts/publish-client-proto-inputs.ps1 ``` +Freshness is guarded two ways so a skipped regeneration cannot ship silently: + +- `ClientProtoInputTests.Descriptor_ContainsEveryContractMessageAndField` (gateway test project) + reflects over the in-process contract descriptors and fails if any message or field is missing + from the committed protoset. It is semantic (symbol presence), needs no protoc, and runs in the + Linux CI. A red test means "regenerate and commit the protoset." +- `pwsh -File scripts/publish-client-proto-inputs.ps1 -Check` rebuilds the descriptor and compares + it to the committed one. The comparison normalizes both sides through the same protoc with + `source_code_info` stripped, so it does not false-fail across protoc releases. +- `pwsh scripts/check-codegen.ps1` runs both the descriptor check and the `Generated/`-clean check + together; it is the single guard the CI pipeline invokes. + ## Related Documentation - [Client Proto Generation](./ClientProtoGeneration.md) diff --git a/docs/GatewayTesting.md b/docs/GatewayTesting.md index bdb04ba..6743272 100644 --- a/docs/GatewayTesting.md +++ b/docs/GatewayTesting.md @@ -408,6 +408,34 @@ Run the gateway test project after shared gateway test infrastructure changes: dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj ``` +## Continuous Integration + +CI runs on Gitea Actions (`.gitea/workflows/ci.yml`; origin is Gitea at +`gitea.dohertylan.com`) on every push and pull request. The pipeline is split by +runtime because the x86 Worker cannot build on Linux: + +- **`portable`** (Linux runner) — builds `src/ZB.MOM.WW.MxGateway.NonWindows.slnx`, + runs the codegen/descriptor freshness guard (`scripts/check-codegen.ps1`), runs the + gateway fake-worker tests, and builds/tests the clients that run on Linux: .NET client + build, Go (`gofmt` + `go build` + `go test`), Rust (`cargo fmt --check` + `cargo test` + + `cargo clippy -D warnings`), and Python (`pytest`). +- **`java`** (Linux, JDK 17) — `gradle test`. The protobuf gradle plugin rewrites + `MxaccessGateway.java` with spurious protobuf-runtime-version churn on every build, so + when no `.proto` changed the job reverts that one file (`git checkout`) before asserting + the generated tree is clean. The dev Mac has no JRE, so Java verification is CI-only. +- **`windows`** (self-hosted windev runner) — the only host that builds the **x86 / + net48 Worker and Worker.Tests**; these are Windows-only and out of scope for the Linux + jobs. This job also proves the net48 build against the committed `Generated/` (the + definitive guard for the "regenerate and commit `Generated/`" rule). +- **`live-mxaccess`** (self-hosted, MXAccess-installed) — scheduled only + (`MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1`); never gates a push. + +The freshness guard `scripts/check-codegen.ps1` fails the build when the committed +client descriptor set or the C# `Generated/` no longer matches the current `.proto` +sources — the codegen drift class this repo has hit repeatedly (stale client +descriptors, net48 `CS0246` on unregenerated protos). See +[Client Proto Generation](./ClientProtoGeneration.md) and [Contracts](./Contracts.md). + ## Related Documentation - [Cross-Language Smoke Matrix](./CrossLanguageSmokeMatrix.md) diff --git a/scripts/check-codegen.ps1 b/scripts/check-codegen.ps1 new file mode 100644 index 0000000..62fe57f --- /dev/null +++ b/scripts/check-codegen.ps1 @@ -0,0 +1,96 @@ +#!/usr/bin/env pwsh +# Codegen freshness guard for CI (IPC-01, IPC-19, IPC-20, CLI-02). +# +# Three checks, all Linux/macOS-runnable (no Server build, no x86 worker): +# 1. Published client descriptor set matches the current .proto sources (delegates to +# publish-client-proto-inputs.ps1 -Check, which normalizes source_code_info so it is +# protoc-version tolerant). +# 2. The committed C# under Contracts/Generated matches a fresh regeneration. Grpc.Tools is +# pinned in the contracts csproj, so a clean checkout regenerates byte-identical output; a +# non-empty git diff means a .proto was edited without regenerating and committing Generated/ +# (which breaks the net48 worker build with CS0246 — see docs/Contracts.md). +# 3. The Rust crate's vendored protos (clients/rust/protos/*.proto — build inputs that make the +# crate buildable outside the repo, CLI-02) are byte-identical to the canonical Contracts +# protos. A drift means a .proto was edited without refreshing the vendored copies, which would +# publish a stale wire contract to crate consumers while the in-repo build stays correct. +# +# The x86 Worker + Worker.Tests are Windows-only and are guarded by the Windows CI job, not here. + +[CmdletBinding()] +param() + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$repoRoot = Resolve-Path (Join-Path $PSScriptRoot '..') +$generatedDir = Join-Path $repoRoot 'src/ZB.MOM.WW.MxGateway.Contracts/Generated' +$contractsProject = Join-Path $repoRoot 'src/ZB.MOM.WW.MxGateway.Contracts/ZB.MOM.WW.MxGateway.Contracts.csproj' +$failures = New-Object System.Collections.Generic.List[string] + +Write-Host '== Check 1/2: client descriptor set freshness ==' +try { + & (Join-Path $PSScriptRoot 'publish-client-proto-inputs.ps1') -Check + if ($LASTEXITCODE -ne 0) { + $failures.Add('Client descriptor set is stale (publish-client-proto-inputs.ps1 -Check failed).') + } +} +catch { + $failures.Add("Descriptor freshness check failed: $($_.Exception.Message)") +} + +Write-Host '' +Write-Host '== Check 2/2: Contracts/Generated matches a fresh regeneration ==' +try { + # Force a full regeneration: Grpc.Tools skips regen when the committed .cs look up to date, so + # remove them first (the documented "del Generated/*.cs to force regen" trick). + if (Test-Path $generatedDir) { + Get-ChildItem -Path $generatedDir -Filter '*.cs' -Recurse -File | Remove-Item -Force + } + + & dotnet build $contractsProject -c Release --nologo | Out-Host + if ($LASTEXITCODE -ne 0) { + $failures.Add('Contracts project failed to build during codegen check.') + } + + $diff = (& git -C $repoRoot status --porcelain -- 'src/ZB.MOM.WW.MxGateway.Contracts/Generated').Trim() + if (-not [string]::IsNullOrEmpty($diff)) { + Write-Host $diff + $failures.Add('Contracts/Generated differs from a fresh regeneration. Run `dotnet build` on the contracts project and commit Generated/ (required for the net48 worker build).') + } +} +catch { + $failures.Add("Generated codegen check failed: $($_.Exception.Message)") +} + +Write-Host '' +Write-Host '== Check 3/3: Rust vendored protos match canonical Contracts protos ==' +try { + $canonicalProtoDir = Join-Path $repoRoot 'src/ZB.MOM.WW.MxGateway.Contracts/Protos' + $vendoredProtoDir = Join-Path $repoRoot 'clients/rust/protos' + foreach ($vendored in Get-ChildItem -Path $vendoredProtoDir -Filter '*.proto' -File) { + $canonical = Join-Path $canonicalProtoDir $vendored.Name + if (-not (Test-Path $canonical)) { + $failures.Add("Vendored Rust proto has no canonical counterpart: $($vendored.Name).") + continue + } + $vendoredHash = (Get-FileHash -Algorithm SHA256 $vendored.FullName).Hash + $canonicalHash = (Get-FileHash -Algorithm SHA256 $canonical).Hash + if ($vendoredHash -ne $canonicalHash) { + $failures.Add("Rust vendored proto drifted from canonical: clients/rust/protos/$($vendored.Name). Refresh it from src/ZB.MOM.WW.MxGateway.Contracts/Protos/$($vendored.Name).") + } + } +} +catch { + $failures.Add("Rust vendored proto check failed: $($_.Exception.Message)") +} + +Write-Host '' +if ($failures.Count -gt 0) { + Write-Host 'Codegen freshness check FAILED:' -ForegroundColor Red + foreach ($failure in $failures) { + Write-Host " - $failure" -ForegroundColor Red + } + exit 1 +} + +Write-Host 'Codegen freshness check passed.' -ForegroundColor Green diff --git a/scripts/publish-client-proto-inputs.ps1 b/scripts/publish-client-proto-inputs.ps1 index 737ad7c..29deb5c 100644 --- a/scripts/publish-client-proto-inputs.ps1 +++ b/scripts/publish-client-proto-inputs.ps1 @@ -6,23 +6,46 @@ param( Set-StrictMode -Version Latest $ErrorActionPreference = "Stop" +# Pinned protoc toolchain version used to generate the committed client descriptor set. +# See docs/ToolchainLinks.md. Generating (committing) the canonical artifact must use this +# exact version so the checked-in bytes are reproducible. The -Check comparison is made +# tolerant of protoc-version encoding drift (chiefly source_code_info, see IPC-20) by +# normalizing both the committed and the freshly built descriptor through the *same* protoc +# with source info stripped, so it does not false-fail across protoc releases. +$PinnedProtocVersion = "34.1" + $repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..") $protoRoot = Join-Path $repoRoot "src/ZB.MOM.WW.MxGateway.Contracts/Protos" $manifestPath = Join-Path $repoRoot "clients/proto/proto-inputs.json" $descriptorPath = Join-Path $repoRoot "clients/proto/descriptors/mxaccessgw-client-v1.protoset" +$protoFiles = @("mxaccess_gateway.proto", "mxaccess_worker.proto", "galaxy_repository.proto") function Resolve-Protoc { - $pathCommand = Get-Command "protoc.exe" -ErrorAction SilentlyContinue - if ($null -ne $pathCommand) { - return $pathCommand.Source + # Prefer protoc on PATH (protoc on Linux/macOS, protoc.exe on Windows), then fall back to + # the documented winget install location on Windows. + foreach ($name in @("protoc", "protoc.exe")) { + $pathCommand = Get-Command $name -ErrorAction SilentlyContinue + if ($null -ne $pathCommand) { + return $pathCommand.Source + } } - $documentedPath = Join-Path $env:LOCALAPPDATA "Microsoft/WinGet/Packages/Google.Protobuf_Microsoft.Winget.Source_8wekyb3d8bbwe/bin/protoc.exe" - if (Test-Path $documentedPath) { - return $documentedPath + if ($env:LOCALAPPDATA) { + $documentedPath = Join-Path $env:LOCALAPPDATA "Microsoft/WinGet/Packages/Google.Protobuf_Microsoft.Winget.Source_8wekyb3d8bbwe/bin/protoc.exe" + if (Test-Path $documentedPath) { + return $documentedPath + } } - throw "Could not find protoc.exe. See docs/ToolchainLinks.md for the documented protobuf toolchain path." + throw "Could not find protoc on PATH. See docs/ToolchainLinks.md for the documented protobuf toolchain (protoc $PinnedProtocVersion)." +} + +function Get-ProtocVersion { + param([string]$Protoc) + + # `protoc --version` prints e.g. "libprotoc 34.1". + $raw = (& $Protoc --version) | Select-Object -First 1 + return ($raw -replace '^libprotoc\s+', '').Trim() } function Ensure-Directory { @@ -33,7 +56,29 @@ function Ensure-Directory { } } -function Compare-FileBytes { +function New-SourceInfoFreeDescriptor { + # Re-emits an existing descriptor set with source_code_info stripped, by round-tripping it + # through protoc with --descriptor_set_in and *without* --include_source_info. Because both + # the committed and freshly built descriptors are normalized through the same protoc binary, + # any protoc-version-specific encoding cancels out and the byte comparison is stable. + param( + [string]$Protoc, + [string]$InputDescriptor, + [string]$OutputDescriptor + ) + + & $Protoc ` + "--descriptor_set_in=$InputDescriptor" ` + "--include_imports" ` + "--descriptor_set_out=$OutputDescriptor" ` + @protoFiles + + if ($LASTEXITCODE -ne 0) { + throw "protoc normalization (--descriptor_set_in) failed with exit code $LASTEXITCODE." + } +} + +function Test-FileBytesEqual { param( [string]$ExpectedPath, [string]$ActualPath @@ -66,31 +111,67 @@ foreach ($output in $manifest.generatedOutputs.PSObject.Properties.Value) { } $protoc = Resolve-Protoc -$outputPath = $descriptorPath -if ($Check) { - $outputPath = Join-Path ([System.IO.Path]::GetTempPath()) ("mxaccessgw-client-v1-" + [System.Guid]::NewGuid().ToString("N") + ".protoset") -} +$protocVersion = Get-ProtocVersion $protoc + +if ($Check) { + # Version drift no longer produces a false "stale" failure: the comparison is normalized + # below. A mismatch is surfaced as a warning so it is visible without blocking the gate. + if ($protocVersion -ne $PinnedProtocVersion) { + Write-Warning "protoc version '$protocVersion' differs from the pinned '$PinnedProtocVersion'. The freshness comparison is normalized and tolerant of this, but regeneration must use the pinned version (see docs/ClientProtoGeneration.md)." + } + + $freshDescriptor = Join-Path ([System.IO.Path]::GetTempPath()) ("mxaccessgw-client-v1-fresh-" + [System.Guid]::NewGuid().ToString("N") + ".protoset") + $committedNormalized = Join-Path ([System.IO.Path]::GetTempPath()) ("mxaccessgw-client-v1-committed-" + [System.Guid]::NewGuid().ToString("N") + ".protoset") + + try { + # Fresh descriptor from the current .proto sources, source info omitted (already normalized). + & $protoc ` + "--proto_path=$protoRoot" ` + "--include_imports" ` + "--descriptor_set_out=$freshDescriptor" ` + @protoFiles + + if ($LASTEXITCODE -ne 0) { + throw "protoc failed with exit code $LASTEXITCODE." + } + + if (-not (Test-Path $descriptorPath)) { + throw "Committed descriptor '$descriptorPath' does not exist. Run scripts/publish-client-proto-inputs.ps1 to create it." + } + + # Normalize the committed descriptor (which carries source info) through the same protoc. + New-SourceInfoFreeDescriptor -Protoc $protoc -InputDescriptor $descriptorPath -OutputDescriptor $committedNormalized + + if (-not (Test-FileBytesEqual -ExpectedPath $committedNormalized -ActualPath $freshDescriptor)) { + throw "Client proto descriptor is stale. Run scripts/publish-client-proto-inputs.ps1 with the pinned protoc ($PinnedProtocVersion) and commit the updated descriptor." + } + + Write-Host "Client proto descriptor is up to date." + } + finally { + foreach ($temp in @($freshDescriptor, $committedNormalized)) { + if (Test-Path $temp) { + Remove-Item -LiteralPath $temp + } + } + } +} +else { + # Generating the canonical committed artifact must use the pinned protoc so the bytes are reproducible. + if ($protocVersion -ne $PinnedProtocVersion) { + throw "protoc version '$protocVersion' does not match the pinned toolchain version '$PinnedProtocVersion'. Install the pinned protoc (see docs/ToolchainLinks.md) before regenerating the committed descriptor, or run with -Check to only verify freshness." + } -try { & $protoc ` "--proto_path=$protoRoot" ` "--include_imports" ` "--include_source_info" ` - "--descriptor_set_out=$outputPath" ` - "mxaccess_gateway.proto" ` - "mxaccess_worker.proto" ` - "galaxy_repository.proto" + "--descriptor_set_out=$descriptorPath" ` + @protoFiles if ($LASTEXITCODE -ne 0) { throw "protoc failed with exit code $LASTEXITCODE." } - if ($Check -and -not (Compare-FileBytes -ExpectedPath $descriptorPath -ActualPath $outputPath)) { - throw "Client proto descriptor is stale. Run scripts/publish-client-proto-inputs.ps1 and commit the updated descriptor." - } -} -finally { - if ($Check -and (Test-Path $outputPath)) { - Remove-Item -LiteralPath $outputPath - } + Write-Host "Wrote client proto descriptor to $descriptorPath (protoc $protocVersion)." } diff --git a/src/ZB.MOM.WW.MxGateway.Contracts/ZB.MOM.WW.MxGateway.Contracts.csproj b/src/ZB.MOM.WW.MxGateway.Contracts/ZB.MOM.WW.MxGateway.Contracts.csproj index b23971b..8cfb3fe 100644 --- a/src/ZB.MOM.WW.MxGateway.Contracts/ZB.MOM.WW.MxGateway.Contracts.csproj +++ b/src/ZB.MOM.WW.MxGateway.Contracts/ZB.MOM.WW.MxGateway.Contracts.csproj @@ -23,6 +23,13 @@ + diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Contracts/ClientProtoInputTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Contracts/ClientProtoInputTests.cs index 4848871..981b54f 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Contracts/ClientProtoInputTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Contracts/ClientProtoInputTests.cs @@ -1,5 +1,6 @@ using System.Text.Json; using Google.Protobuf; +using Google.Protobuf.Reflection; using ZB.MOM.WW.MxGateway.Contracts; using ZB.MOM.WW.MxGateway.Contracts.Proto; @@ -7,6 +8,102 @@ namespace ZB.MOM.WW.MxGateway.Tests.Contracts; public sealed class ClientProtoInputTests { + /// + /// Guards the published client descriptor set against silent staleness (IPC-01). Every message + /// and field compiled into the in-process contract (which the build regenerates from the current + /// .proto sources) must appear in the committed protoset. A missing symbol means the + /// descriptor was not regenerated after a proto change; run + /// scripts/publish-client-proto-inputs.ps1 and commit the refreshed protoset. + /// The check is semantic (symbol presence) rather than byte-wise, so it is independent of protoc + /// version and does not require protoc on the test runner. + /// + [Fact] + public void Descriptor_ContainsEveryContractMessageAndField() + { + DirectoryInfo repositoryRoot = FindRepositoryRoot(); + string descriptorPath = Path.Combine( + repositoryRoot.FullName, + "clients", + "proto", + "descriptors", + "mxaccessgw-client-v1.protoset"); + + Assert.True(File.Exists(descriptorPath), $"Expected descriptor set '{descriptorPath}' to exist."); + + FileDescriptorSet descriptorSet = FileDescriptorSet.Parser.ParseFrom(File.ReadAllBytes(descriptorPath)); + + HashSet publishedMessages = new(StringComparer.Ordinal); + HashSet publishedFields = new(StringComparer.Ordinal); + foreach (FileDescriptorProto file in descriptorSet.File) + { + foreach (DescriptorProto message in file.MessageType) + { + CollectPublishedSymbols(file.Package, message, publishedMessages, publishedFields); + } + } + + List missing = []; + foreach (FileDescriptor file in new[] { MxaccessGatewayReflection.Descriptor, MxaccessWorkerReflection.Descriptor }) + { + foreach (MessageDescriptor message in file.MessageTypes) + { + CollectMissingContractSymbols(message, publishedMessages, publishedFields, missing); + } + } + + Assert.True( + missing.Count == 0, + "Published client descriptor is stale; regenerate with scripts/publish-client-proto-inputs.ps1 and commit. " + + "Missing symbols: " + + string.Join(", ", missing)); + } + + private static void CollectPublishedSymbols( + string package, + DescriptorProto message, + HashSet messages, + HashSet fields) + { + string fullName = string.IsNullOrEmpty(package) ? message.Name : package + "." + message.Name; + messages.Add(fullName); + + foreach (FieldDescriptorProto field in message.Field) + { + fields.Add(fullName + "/" + field.Name); + } + + foreach (DescriptorProto nested in message.NestedType) + { + CollectPublishedSymbols(fullName, nested, messages, fields); + } + } + + private static void CollectMissingContractSymbols( + MessageDescriptor message, + HashSet publishedMessages, + HashSet publishedFields, + List missing) + { + if (!publishedMessages.Contains(message.FullName)) + { + missing.Add(message.FullName); + } + + foreach (FieldDescriptor field in message.Fields.InDeclarationOrder()) + { + string key = message.FullName + "/" + field.Name; + if (!publishedFields.Contains(key)) + { + missing.Add(key); + } + } + + foreach (MessageDescriptor nested in message.NestedTypes) + { + CollectMissingContractSymbols(nested, publishedMessages, publishedFields, missing); + } + } + /// Verifies that the proto inputs manifest declares current protocol versions and existing source files. [Fact] public void Manifest_DeclaresCurrentProtocolVersionsAndExistingInputs()