fix: bind private temp storage for read-only late-slope runs

This commit is contained in:
Joseph Doherty
2026-11-10 18:18:11 -05:00
parent 351edb0a86
commit 681e1c1a9b
5 changed files with 58 additions and 8 deletions
@@ -51,8 +51,12 @@ python3 "$bundle/tools/launch.py" --bundle "$bundle" \
Root owns launch after review of the exact bundle receipt and controls. The
bundled launcher uses the pinned Unix endpoint, immutable image, network-none
container, read-only root filesystem and read-only application and tools mounts.
A fresh private task directory holds results and fixture files. The two
descriptor controls must execute successfully before the canonical test starts. The launcher verifies
A fresh private task directory holds results and fixture files. The launcher
creates its temporary subdirectory with mode `0700` and binds the same storage
at `/tmp` for NuGet's named mutex. `TMPDIR` continues to point at `/task/tmp`.
The container checks both paths identify the same private directory before
starting VSTest. The two descriptor controls must execute successfully before
the canonical test starts. The launcher verifies
the actual test process's apphost, mapped CLR and CLR hash. Its outer deadline
is 7,440 seconds, with cleanup limited to the uniquely labeled container.
@@ -81,6 +81,7 @@ def container_command(root, output, bundle, run_id):
"--name", f"opcua-late-slope-{run_id}",
"--label", f"{DOCKER_LABEL}={run_id}", "-e", "OTOPCUA_SOAK_BUILD_SHA=" + bundle["sourceRevision"],
"-v", f"{root / 'app'}:/app:ro", "-v", f"{root / 'tools'}:/tools:ro", "-v", f"{output}:/task",
"-v", f"{output / 'tmp'}:/tmp:rw",
bundle["image"], "bash", "/tools/container.sh"]
@@ -17,7 +17,9 @@ set -euo pipefail
umask 077
app=ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests
test_class="$app.Sdk2LifecycleSoakTests"
mkdir -p /task/tmp /task/data /task/cli /task/results
[[ -d /task/tmp && ! -L /task/tmp && -d /tmp && ! -L /tmp ]]
[[ $(stat -c '%a' /task/tmp) == 700 && $(stat -c '%d:%i' /task/tmp) == "$(stat -c '%d:%i' /tmp)" ]]
mkdir -p /task/data /task/cli /task/results
export TMPDIR=/task/tmp XDG_DATA_HOME=/task/data DOTNET_CLI_HOME=/task/cli
export OTOPCUA_SDK2_SOAK=1 OTOPCUA_SDK2_SOAK_MODE=late-slope OTOPCUA_SDK2_SOAK_SECONDS=7200
export OTOPCUA_SDK2_SOAK_OUTPUT=/task/resources.jsonl OPCUA_CERTIFICATE_LEAK_TRACKING=1
@@ -5,6 +5,7 @@ import argparse
import json
from pathlib import Path
import re
import stat
import subprocess
import sys
import time
@@ -46,12 +47,24 @@ def verify_local_image(bundle):
require(image_identity(bundle["image"], context) == bundle["imageIdentity"], "Wrong cached image identity")
def create_task_output(output, bundle_root):
require(not output.is_symlink(), "Run output cannot be a symbolic link")
output = output.resolve()
require(not output.is_relative_to(bundle_root), "Run output must be outside the immutable bundle")
output.mkdir(mode=0o700, parents=True, exist_ok=False)
temporary = output / "tmp"
temporary.mkdir(mode=0o700, exist_ok=False)
for directory in (output, temporary):
mode = directory.lstat().st_mode
require(stat.S_ISDIR(mode) and stat.S_IMODE(mode) == 0o700, f"Expected private directory mode 0700 for {directory}")
return output
def launch(root, bundle, bundle_hash, output):
require(Path(__file__).resolve() == root / "tools/launch.py", "Invoke the reviewed bundled launcher")
require(not output.is_relative_to(root), "Run output must be outside the immutable bundle")
output = create_task_output(output, root)
run_id = uuid.uuid4().hex
invocation = container_command(root, output, bundle, run_id)
output.mkdir(mode=0o700, parents=True, exist_ok=False)
provenance = {"kind": "launch", "protocol": PROTOCOL, "acceptance": False, "runId": run_id,
"bundleReceiptSha256": bundle_hash, "sourceRevision": bundle["sourceRevision"], "image": bundle["image"],
"coreClrSha256": bundle["files"]["app/libcoreclr.so"], "command": invocation,
@@ -120,7 +133,7 @@ def main():
"receiptSha256": options.receipt_sha256, "verified": True,
"acceptance": False, "workloadLaunched": False}, indent=2))
return 0
return launch(root, bundle, options.receipt_sha256, options.output.resolve())
return launch(root, bundle, options.receipt_sha256, options.output)
except (OSError, KeyError, TypeError, ValueError, subprocess.SubprocessError) as error:
print(f"Late-slope launcher rejected the request: {error}", file=sys.stderr)
return 1
@@ -4,6 +4,7 @@
import json
import os
from pathlib import Path
import stat
import subprocess
import sys
import tempfile
@@ -16,6 +17,7 @@ sys.dont_write_bytecode = True
import analyze
import bundle
import launch
REVISION = "a" * 40
@@ -284,14 +286,21 @@ class ReceiptControls(unittest.TestCase):
self.report()
def test_recorded_command_must_use_the_same_local_isolation(self):
for isolation in ("network", "read-only rootfs"):
private_mount = f"{self.run / 'tmp'}:/tmp:rw"
self.assertIn(private_mount, self.provenance["command"])
for isolation in ("network", "read-only rootfs", "missing /tmp bind", "foreign /tmp bind"):
with self.subTest(isolation=isolation):
self.build_evidence()
invocation = self.provenance["command"].copy()
if isolation == "network":
invocation[invocation.index("--network") + 1] = "host"
else:
elif isolation == "read-only rootfs":
invocation.remove("--read-only")
elif isolation == "missing /tmp bind":
index = invocation.index(private_mount)
del invocation[index - 1:index + 1]
else:
invocation[invocation.index(private_mount)] = f"{self.root / 'foreign'}:/tmp:rw"
self.provenance["command"] = self.receipt["command"] = invocation
write_json(self.run / "launch.json", self.provenance)
self.refresh_receipt()
@@ -300,6 +309,27 @@ class ReceiptControls(unittest.TestCase):
class BundleControls(unittest.TestCase):
def test_task_output_and_tmp_are_fresh_private_directories(self):
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary).resolve()
output = launch.create_task_output(root / "run", root / "bundle")
for directory in (output, output / "tmp"):
mode = directory.lstat().st_mode
self.assertTrue(stat.S_ISDIR(mode))
self.assertEqual(stat.S_IMODE(mode), 0o700)
with self.assertRaises(FileExistsError):
launch.create_task_output(output, root / "bundle")
file = root / "existing-file"
file.touch()
with self.assertRaises(FileExistsError):
launch.create_task_output(file, root / "bundle")
link = root / "linked-output"
destination = root / "missing-target"
link.symlink_to(destination, target_is_directory=True)
with self.assertRaisesRegex(ValueError, "symbolic link"):
launch.create_task_output(link, root / "bundle")
self.assertFalse(destination.exists())
def test_mapped_runtime_requires_the_mounted_inode_and_live_file(self):
script = Path(__file__).with_name("container.sh")
with tempfile.TemporaryDirectory() as temporary: