feat(upm): implement Rust user package manager

Adds the `user.rust` UPM to workshop-engine for managing Rust
toolchains,
components, cross-compilation targets, and cargo operations:

- `rustup default <toolchain>` / `rustup component add` / `rustup target
  add`
- `cargo fetch` for dependency pre-fetch with optional `--locked`
- `cargo install` for global crate tools with registry support

Includes 275-line test suite (`tests/upm/test_rust.py`) covering the
full
lifecycle with optional network tests.
This commit is contained in:
Catty Steve
2026-04-28 12:20:05 +08:00
parent cf317b55c6
commit fd18b547e5
14 changed files with 825 additions and 197 deletions
+2
View File
@@ -18,3 +18,5 @@ tests/**/__pycache__/
tests/.pytest_cache/
tests/llm/.cache/
*.pyc
.staging
.pytest_cache
+171 -55
View File
@@ -4,97 +4,213 @@ set -euo pipefail
# HoneyBiscuitWorkshop System Test Runner
#
# Usage:
# ./runner.sh # Arch baker tests (hbw-system-test:latest)
# ./runner.sh --image hbw-test-pm:test-apt # Debian engine PM tests
# ./runner.sh -k test_install # Arch baker, filtered
MODE="auto"
TEST_IMAGE=""
TEST_DIR="tests/baker"
PYTEST_ARGS=""
while [[ $# -gt 0 ]]; do
case "$1" in
--image) TEST_IMAGE="$2"; TEST_DIR="tests/engine"; shift 2 ;;
manual) MODE="manual"; shift ;;
auto) MODE="auto"; shift ;;
--) shift; PYTEST_ARGS="$@"; break ;;
*) PYTEST_ARGS="$PYTEST_ARGS $1"; shift ;;
esac
done
# Default image: Arch-based baker test image
: ${TEST_IMAGE:="hbw-system-test:latest"}
# ./runner.sh # baker tests (default)
# ./runner.sh -c upm:rust # Rust UPM tests
# ./runner.sh -c pm:pacman # pacman PM engine tests
# ./runner.sh -c pm:apt -k test_mirror # apt mirror test only
# ./runner.sh -c upm:rust --network # include network tests
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
CONTAINER_NAME="hbw-system-test-container"
STAGING_DIR="/tmp/hbw-system-test"
STAGING_DIR="$SCRIPT_DIR/.staging"
# ── defaults ────────────────────────────────────────────────────────
MODE="auto"
CATEGORY="baker" # baker | upm:<name> | pm:<name>
TEST_DIR=""
TEST_IMAGE=""
PYTEST_ARGS=()
NETWORK=0
BAKER_LOG=1
# ── parse args ──────────────────────────────────────────────────────
show_help() {
cat << 'HELP'
Usage: ./runner.sh [OPTIONS]
Categories (-c):
baker Baker system tests [default]
upm:rust Rust UPM tests
pm:pacman Pacman PM engine tests
pm:apt APT PM engine tests
pm:dnf DNF PM engine tests
pm:apk APK PM engine tests
Options:
-c, --category CAT Test category (see above)
--network Allow network-dependent tests (toolchain/crate downloads)
-k FILTER pytest -k expression
--timeout SECONDS Per-test timeout (default 120)
--log / --no-log Stream baker output to terminal [default: --log]
-h, --help This message
HELP
exit 0
}
while [[ $# -gt 0 ]]; do
case "$1" in
-c|--category)
CATEGORY="$2"; shift 2 ;;
--network)
NETWORK=1; shift ;;
-k)
PYTEST_ARGS+=(-k "$2"); shift 2 ;;
--timeout)
PYTEST_ARGS+=(--timeout "$2"); shift 2 ;;
--log) BAKER_LOG=1; shift ;;
--no-log) BAKER_LOG=0; shift ;;
manual) MODE="manual"; shift ;;
auto) MODE="auto"; shift ;;
-h|--help)
show_help ;;
--)
shift; PYTEST_ARGS+=("$@"); break ;;
*)
PYTEST_ARGS+=("$1"); shift ;;
esac
done
# ── resolve category → test dir + image ─────────────────────────────
case "$CATEGORY" in
baker)
TEST_DIR="tests/baker"
TEST_IMAGE="hbw-system-test:latest"
;;
upm:rust)
TEST_DIR="tests/upm"
TEST_IMAGE="hbw-system-test:latest"
;;
pm:pacman|pm:arch)
TEST_DIR="tests/engine"
TEST_IMAGE="hbw-test-pm:test-arch"
;;
pm:apt)
TEST_DIR="tests/engine"
TEST_IMAGE="hbw-test-pm:test-apt"
;;
pm:dnf)
TEST_DIR="tests/engine"
TEST_IMAGE="hbw-test-pm:test-dnf"
;;
pm:apk)
TEST_DIR="tests/engine"
TEST_IMAGE="hbw-test-pm:test-apk"
;;
*)
echo "Unknown category: $CATEGORY" >&2
echo "Run './runner.sh --help' for usage." >&2
exit 1
;;
esac
# ── env vars ──────────────────────────────────────────────────────
DOCKER_ENV=()
DOCKER_ENV+=(-e RUST_LOG=${RUST_LOG:-info})
DOCKER_ENV+=(-e BAKER_LOG=${BAKER_LOG:-1})
# Load llm.env if present (for LLM-based test evaluation)
if [ -f "$SCRIPT_DIR/llm.env" ]; then
while IFS='=' read -r key value; do
[ -z "$key" ] && continue
[[ "$key" =~ ^# ]] && continue
DOCKER_ENV+=(-e "$key=$value")
done < "$SCRIPT_DIR/llm.env"
fi
if [ "$NETWORK" -eq 1 ]; then
DOCKER_ENV+=(-e RUST_UPM_NETWORK=1)
fi
# ── cleanup ─────────────────────────────────────────────────────────
cleanup() {
docker rm -f "$CONTAINER_NAME" 2>/dev/null || true
[ "$TEST_DIR" = "tests/baker" ] && rm -rf "$STAGING_DIR"
}
trap cleanup EXIT
# ── build baker binary ──────────────────────────────────────────────
build_binary() {
local target="x86_64-unknown-linux-musl"
local binary="$SCRIPT_DIR/workshop-baker/target/$target/debug/workshop-baker"
if [ ! -f "$binary" ]; then
echo "[runner] Building static (musl) baker binary..."
cd "$SCRIPT_DIR/workshop-baker"
cargo build -p workshop-baker --target "$target" 2>&1 | tail -n 1
fi
}
# ── staging ─────────────────────────────────────────────────────────
prepare_staging() {
# Only needed for baker tests (hbw-system-test:latest mounts /workshop)
if [ "$TEST_DIR" = "tests/baker" ]; then
if [ "$TEST_DIR" != "tests/engine" ]; then
mkdir -p "$STAGING_DIR"
cp "$SCRIPT_DIR/workshop-baker/target/x86_64-unknown-linux-musl/debug/workshop-baker" "$STAGING_DIR/"
chmod +x "$STAGING_DIR/workshop-baker"
fi
}
run_container() {
local tag="${TEST_IMAGE##*:}"
local filter=""
case "$tag" in
test-arch) filter="-k pacman" ;;
test-apt) filter="-k apt" ;;
test-dnf) filter="-k dnf" ;;
test-apk) filter="-k apk" ;;
# ── pm filter ───────────────────────────────────────────────────────
pm_filter() {
case "${TEST_IMAGE##*:}" in
test-arch) echo "-k pacman" ;;
test-apt) echo "-k apt" ;;
test-dnf) echo "-k dnf" ;;
test-apk) echo "-k apk" ;;
*) echo "" ;;
esac
}
echo "[runner] Starting: $TEST_IMAGE (tests=$TEST_DIR${filter:+, filter=$filter})"
mkdir -p "$SCRIPT_DIR/tests/results"
# ── run ─────────────────────────────────────────────────────────────
local mounts="-v $SCRIPT_DIR:/src:ro"
local entrypoint=""
if [ "$TEST_DIR" = "tests/baker" ]; then
mounts="$mounts -v $STAGING_DIR:/workshop:ro"
entrypoint='--entrypoint ""'
run_container() {
local filter=""
if [ "$TEST_DIR" = "tests/engine" ]; then
filter="$(pm_filter)"
fi
local cmd="sh -c \"python3 -m venv /tmp/v && . /tmp/v/bin/activate && \
pip install pytest pytest-timeout -q && \
exec python -m pytest $TEST_DIR $PYTEST_ARGS $filter\""
echo "============================================"
echo " HBW System Test Runner"
echo " Category: $TEST_DIR Image: $TEST_IMAGE"
echo "============================================"
mkdir -p "$SCRIPT_DIR/tests/results"
local mounts=(-v "$SCRIPT_DIR:/src:ro")
if [ "$TEST_DIR" != "tests/engine" ]; then
mounts+=(-v "$STAGING_DIR:/workshop") # writable: baker writes temp scripts here
fi
local pytest_cmd="python3 -m venv /tmp/v && . /tmp/v/bin/activate && \
pip install -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple \
-r $TEST_DIR/../requirements.txt -q && \
exec python -m pytest -v -s $TEST_DIR ${PYTEST_ARGS[*]} $filter"
if [ "$MODE" = "manual" ]; then
docker run -it --rm --name "$CONTAINER_NAME" --privileged $entrypoint \
$mounts "$TEST_IMAGE"
docker run -it --rm --name "$CONTAINER_NAME" --privileged \
${DOCKER_ENV[@]} "${mounts[@]}" "$TEST_IMAGE"
else
docker run --rm --name "$CONTAINER_NAME" --privileged $entrypoint \
$mounts -w /src \
"$TEST_IMAGE" "$cmd"
# Baker/UPM images (hbw-system-test) have ./entrypoint.sh which
# doesn't exist when /src is the workdir → override to empty.
# PM images (hbw-test-pm:*) have /bin/sh -c → CMD passed directly.
if [ "$TEST_DIR" = "tests/engine" ]; then
docker run --rm --name "$CONTAINER_NAME" --privileged \
${DOCKER_ENV[@]} "${mounts[@]}" -w /src \
"$TEST_IMAGE" "$pytest_cmd"
else
docker run --rm --name "$CONTAINER_NAME" --privileged \
--entrypoint "" ${DOCKER_ENV[@]} "${mounts[@]}" -w /src \
"$TEST_IMAGE" /bin/sh -c "$pytest_cmd"
fi
fi
}
echo "============================================"
echo " HBW System Test Runner"
echo " Image: $TEST_IMAGE"
echo " Tests: $TEST_DIR"
echo "============================================"
# ── main ────────────────────────────────────────────────────────────
build_binary
prepare_staging
run_container
+6 -30
View File
@@ -1,16 +1,14 @@
import os
import pytest
FIXTURES = os.path.join(os.path.dirname(__file__), "fixtures", "bake")
FIXTURES = os.path.join(os.path.dirname(os.path.dirname(__file__)), "fixtures", "bake")
@pytest.mark.bake
class TestBakeSimpleScript:
def test_simple_script_runs(self, run_baker, bake_workspace, work_dir):
prebake_cfg = os.path.join(
os.path.dirname(__file__), "fixtures", "prebake", "minimal.yml"
os.path.dirname(os.path.dirname(__file__)), "fixtures", "prebake", "minimal.yml"
)
script = os.path.join(FIXTURES, "simple.sh")
bake_base = os.path.join(FIXTURES, "bake_base.sh")
@@ -27,7 +25,7 @@ class TestBakeSimpleScript:
def test_simple_script_sets_env_vars(self, run_baker, bake_workspace, work_dir):
prebake_cfg = os.path.join(
os.path.dirname(__file__), "fixtures", "prebake", "minimal.yml"
os.path.dirname(os.path.dirname(__file__)), "fixtures", "prebake", "minimal.yml"
)
script = os.path.join(FIXTURES, "simple.sh")
bake_base = os.path.join(FIXTURES, "bake_base.sh")
@@ -42,13 +40,12 @@ class TestBakeSimpleScript:
assert result["success"]
assert "test-pipeline" in result["stdout"]
@pytest.mark.bake
class TestBakePipeline:
def test_pipeline_functions_execute(self, run_baker, bake_workspace, work_dir):
prebake_cfg = os.path.join(
os.path.dirname(__file__), "fixtures", "prebake", "minimal.yml"
os.path.dirname(os.path.dirname(__file__)), "fixtures", "prebake", "minimal.yml"
)
script = os.path.join(FIXTURES, "pipeline.sh")
bake_base = os.path.join(FIXTURES, "bake_base.sh")
@@ -66,7 +63,7 @@ class TestBakePipeline:
def test_pipeline_step_outputs_correct(self, run_baker, bake_workspace, work_dir):
prebake_cfg = os.path.join(
os.path.dirname(__file__), "fixtures", "prebake", "minimal.yml"
os.path.dirname(os.path.dirname(__file__)), "fixtures", "prebake", "minimal.yml"
)
script = os.path.join(FIXTURES, "pipeline.sh")
bake_base = os.path.join(FIXTURES, "bake_base.sh")
@@ -84,13 +81,12 @@ class TestBakePipeline:
with open("/tmp/bake-test-output/step2.txt") as f:
assert f.read().strip() == "step2-done"
@pytest.mark.bake
class TestBakeAfterDependencies:
def test_after_ordering(self, run_baker, bake_workspace, work_dir):
prebake_cfg = os.path.join(
os.path.dirname(__file__), "fixtures", "prebake", "minimal.yml"
os.path.dirname(os.path.dirname(__file__)), "fixtures", "prebake", "minimal.yml"
)
script = os.path.join(FIXTURES, "with_after.sh")
bake_base = os.path.join(FIXTURES, "bake_base.sh")
@@ -107,23 +103,3 @@ class TestBakeAfterDependencies:
assert os.path.exists("/tmp/bake-after-test/source.txt")
assert os.path.exists("/tmp/bake-after-test/build.txt")
@pytest.mark.bake
class TestBakeStatusFiles:
def test_status_written_after_bake(self, run_baker, bake_workspace, work_dir):
prebake_cfg = os.path.join(
os.path.dirname(__file__), "fixtures", "prebake", "minimal.yml"
)
script = os.path.join(FIXTURES, "simple.sh")
bake_base = os.path.join(FIXTURES, "bake_base.sh")
result = run_baker(
"bake", script,
"--bake-base", bake_base,
"--prebake", prebake_cfg,
cwd=work_dir,
timeout=60,
)
assert result["success"]
assert os.path.exists("/tmp/.hbwstatus")
+3 -18
View File
@@ -4,9 +4,7 @@ import pytest
from probes import file
from llm import judge_log, LLMClient
FIXTURES = os.path.join(os.path.dirname(__file__), "fixtures", "finalize")
FIXTURES = os.path.join(os.path.dirname(os.path.dirname(__file__)), "fixtures", "finalize")
@pytest.mark.finalize
class TestFinalizeMinimal:
@@ -16,7 +14,6 @@ class TestFinalizeMinimal:
result = run_baker("finalize", config, cwd=work_dir, timeout=60)
assert result["success"], f"Minimal finalize failed: {result['stderr']}"
@pytest.mark.finalize
class TestFinalizeHooks:
@@ -32,17 +29,6 @@ class TestFinalizeHooks:
assert result["success"]
assert "finalize late hook" in result["stdout"] or "finalize late hook" in result["stderr"]
@pytest.mark.finalize
class TestFinalizeStatus:
def test_finalize_writes_status(self, run_baker, work_dir):
config = os.path.join(FIXTURES, "minimal.yml")
result = run_baker("finalize", config, cwd=work_dir, timeout=60)
assert result["success"]
assert os.path.exists("/tmp/.hbwstatus")
@pytest.mark.finalize
@pytest.mark.llm
class TestFinalizeWithLLM:
@@ -64,14 +50,13 @@ class TestFinalizeWithLLM:
)
assert judgment.passed, f"LLM judgment failed: {judgment.reason}"
@pytest.mark.finalize
class TestFinalizePlugins:
@pytest.mark.xfail(reason="Plugin download requires network/git access", strict=False)
@pytest.mark.skip(reason="Plugin download test not yet implemented")
def test_plugin_download(self, run_baker, work_dir):
pass
@pytest.mark.xfail(reason="Artifact stage not yet implemented", strict=False)
@pytest.mark.skip(reason="Artifact stage not yet implemented")
def test_artifact_collection(self, run_baker, work_dir):
pass
+3 -62
View File
@@ -4,9 +4,7 @@ import pytest
from probes import user, file, package
FIXTURES = os.path.join(os.path.dirname(__file__), "fixtures", "prebake")
FIXTURES = os.path.join(os.path.dirname(os.path.dirname(__file__)), "fixtures", "prebake")
@pytest.mark.prebake
class TestPrebakeConfigValidation:
@@ -29,7 +27,6 @@ class TestPrebakeConfigValidation:
result = run_baker("prebake", config, cwd=work_dir, timeout=60)
assert result["success"], f"Minimal prebake failed: {result['stderr']}"
@pytest.mark.prebake
class TestPrebakeBootstrap:
@@ -89,7 +86,6 @@ class TestPrebakeBootstrap:
assert result["success"], f"Custom bootstrap failed: {result['stderr']}"
assert file.dir_exists("/custom-workspace")
@pytest.mark.prebake
class TestPrebakeDryRun:
@@ -107,7 +103,6 @@ class TestPrebakeDryRun:
combined = result["stdout"] + result["stderr"]
assert "DRY_RUN" in combined or "dry_run" in combined.lower()
@pytest.mark.prebake
class TestPrebakeHooks:
@@ -143,17 +138,15 @@ class TestPrebakeHooks:
result = run_baker("prebake", config, cwd=work_dir, timeout=60)
assert result["success"], f"prebake with no hooks failed: {result['stderr']}"
@pytest.mark.prebake
class TestPrebakeDepsSystem:
@pytest.mark.xfail(reason="Package installation may fail in test environment", strict=False)
def test_pacman_packages_installed(self, run_baker, work_dir):
config = os.path.join(FIXTURES, "with_pacman.yml")
result = run_baker("prebake", config, cwd=work_dir, timeout=120)
assert result["success"], f"prebake failed: {result['stderr']}"
assert package.package_installed("curl")
assert package.package_installed("wget")
assert package.binary_exists("/usr/bin/curl")
assert package.binary_exists("/usr/bin/wget")
def test_pacman_mirror_change(self, run_baker, work_dir):
mirror_config = os.path.join(work_dir, "mirror.yml")
@@ -173,7 +166,6 @@ dependencies:
result = run_baker("prebake", mirror_config, cwd=work_dir, timeout=120)
assert result["success"], f"prebake with mirror failed: {result['stderr']}"
@pytest.mark.prebake
class TestPrebakeEnvVars:
@@ -199,7 +191,6 @@ hooks:
combined = result["stdout"] + result["stderr"]
assert "injected_value" in combined
@pytest.mark.prebake
class TestPrebakeSecurity:
@@ -209,53 +200,3 @@ class TestPrebakeSecurity:
result = run_baker("prebake", config, cwd=work_dir, timeout=60)
assert result["success"], f"prebake with security failed: {result['stderr']}"
@pytest.mark.prebake
class TestPrebakeStatus:
def test_status_files_written(self, run_baker, work_dir):
config = os.path.join(FIXTURES, "bootstrap_only.yml")
result = run_baker("prebake", config, cwd=work_dir, timeout=60)
assert result["success"]
assert os.path.exists("/tmp/.hbwstatus")
def test_status_contains_bootstrap_stage(self, run_baker, work_dir):
config = os.path.join(FIXTURES, "bootstrap_only.yml")
result = run_baker("prebake", config, cwd=work_dir, timeout=60)
assert result["success"]
with open("/tmp/.hbwstatus", "r") as f:
lines = f.readlines()
assert len(lines) > 0
bootstrap_lines = [l for l in lines if json.loads(l).get("name") == "bootstrap"]
assert len(bootstrap_lines) > 0, "No bootstrap stage found in status file"
data = json.loads(bootstrap_lines[-1])
assert data.get("phase") == "Prebake"
assert data.get("result") == "Success"
def test_status_contains_duration(self, run_baker, work_dir):
config = os.path.join(FIXTURES, "bootstrap_only.yml")
result = run_baker("prebake", config, cwd=work_dir, timeout=60)
assert result["success"]
with open("/tmp/.hbwstatus", "r") as f:
data = json.loads(f.readline())
assert "duration_ms" in data
assert isinstance(data["duration_ms"], int)
def test_status_contains_timestamps(self, run_baker, work_dir):
config = os.path.join(FIXTURES, "bootstrap_only.yml")
result = run_baker("prebake", config, cwd=work_dir, timeout=60)
assert result["success"]
with open("/tmp/.hbwstatus", "r") as f:
data = json.loads(f.readline())
assert "started_at" in data
assert "finished_at" in data
def test_status_written_for_each_stage(self, run_baker, work_dir):
config = os.path.join(FIXTURES, "with_pacman.yml")
result = run_baker("prebake", config, cwd=work_dir, timeout=120)
assert result["success"], f"prebake failed: {result['stderr']}"
with open("/tmp/.hbwstatus", "r") as f:
lines = f.readlines()
names = [json.loads(line).get("name") for line in lines]
assert "bootstrap" in names
assert "depssystem" in names
+41 -25
View File
@@ -77,36 +77,52 @@ def run_cmd():
"""
def _run(args, cwd=None, env=None, timeout=120):
"""Run a command and capture output.
Args:
args: Command as list of strings.
cwd: Working directory.
env: Environment variables (merged with os.environ).
timeout: Timeout in seconds.
Returns:
dict with keys: stdout, stderr, returncode, success
"""
full_env = {**os.environ}
if env:
full_env.update(env)
show_logs = os.environ.get("BAKER_LOG", "0") == "1"
try:
result = subprocess.run(
args,
cwd=cwd,
env=full_env,
capture_output=True,
text=True,
timeout=timeout,
)
return {
"stdout": result.stdout,
"stderr": result.stderr,
"returncode": result.returncode,
"success": result.returncode == 0,
}
if show_logs:
# Stream output to terminal in real-time while capturing
import sys, select
p = subprocess.Popen(
args, cwd=cwd, env=full_env,
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
text=True,
)
stdout_chunks, stderr_chunks = [], []
try:
p.wait(timeout=timeout)
except subprocess.TimeoutExpired:
p.kill()
p.wait()
return {"stdout": "", "stderr": f"Command timed out after {timeout}s",
"returncode": -1, "success": False}
for line in p.stdout:
sys.stderr.write("[baker] " + line)
stdout_chunks.append(line)
for line in p.stderr:
sys.stderr.write("[baker:err] " + line)
stderr_chunks.append(line)
return {
"stdout": "".join(stdout_chunks),
"stderr": "".join(stderr_chunks),
"returncode": p.returncode,
"success": p.returncode == 0,
}
else:
result = subprocess.run(
args, cwd=cwd, env=full_env,
capture_output=True, text=True, timeout=timeout,
)
return {
"stdout": result.stdout,
"stderr": result.stderr,
"returncode": result.returncode,
"success": result.returncode == 0,
}
except subprocess.TimeoutExpired:
return {
"stdout": "",
+1
View File
@@ -9,6 +9,7 @@ bootstrap:
dependencies:
system:
pacman:
mirror: "https://mirrors.tuna.tsinghua.edu.cn/archlinux/$repo/os/$arch"
packages:
- "curl"
- "wget"
+1 -3
View File
@@ -15,9 +15,7 @@ class LLMCache:
def __init__(self, cache_dir: str | None = None):
if cache_dir is None:
import os
cache_dir = os.path.join(os.path.dirname(__file__), ".cache")
cache_dir = "/tmp/llm-cache"
self.cache_dir = Path(cache_dir)
self.cache_dir.mkdir(parents=True, exist_ok=True)
View File
+275
View File
@@ -0,0 +1,275 @@
"""Rust UPM system tests driven by bare workshop-baker prebake.
These tests exercise the ``user.rust`` UPM through its full lifecycle:
1. sysdep injection (rustup via pacman, repology_endpoint: none)
2. rustup toolchain / component / target installation
3. cargo fetch (project dependency pre-fetch)
4. cargo install (global crate tools)
Observability
* ``RUST_LOG=info`` is injected into every baker invocation.
* On failure the *full* stdout + stderr is printed (no truncation).
* An LLM judge (``tests/llm/judge.py``) diagnoses failures with a
human-readable reason, cached per-log-hash to save tokens.
Network-dependent tests (toolchain download) are skipped by default.
Set ``RUST_UPM_NETWORK=1`` to run them.
"""
import os
import subprocess
import pytest
from probes import binary_exists
# ── helpers ─────────────────────────────────────────────────────────
RUST_LOG = os.environ.get("RUST_LOG", "info")
def _yml(user_section):
"""Build a minimal prebake.yml focused on the rust UPM."""
return """\
version: "1.0"
environment:
builder: "baremetal"
bootstrap:
user: "vulcan"
security:
drop_after: never
envvars:
prebake:
RUSTUP_DIST_SERVER: "https://mirrors.tuna.tsinghua.edu.cn/rustup"
dependencies:
config:
repology_endpoint: none
system:
pacman:
packages:
user:
rust:
""" + _indent(user_section, 6)
def _indent(text, spaces):
pad = " " * spaces
return "\n".join(pad + line if line.strip() else line
for line in text.splitlines())
def _cargo_toml(dir_path):
path = os.path.join(dir_path, "Cargo.toml")
with open(path, "w") as f:
f.write('[package]\nname = "test"\nversion = "0.1.0"\nedition = "2021"\n')
return path
def _query(cmd, timeout=60):
try:
r = subprocess.run(cmd, capture_output=True, text=True,
timeout=timeout, env={**os.environ, "HOME": "/root"})
return {"success": r.returncode == 0, "stdout": r.stdout,
"stderr": r.stderr}
except Exception as e:
return {"success": False, "stdout": "", "stderr": str(e)}
def _run_prebake(run_baker, config_path, timeout, operation, expected):
"""Run prebake with RUST_LOG, return (result, full_log)."""
env_vars = {"RUST_LOG": RUST_LOG}
r = run_baker("prebake", config_path, timeout=timeout, env=env_vars)
full_log = "[stdout]\n{}\n[stderr]\n{}".format(r["stdout"], r["stderr"])
return r, full_log
def _check(r, full_log, operation, expected):
"""Assert success; on failure call LLM judge and show full output."""
if r["success"]:
return
diagnosis = ""
try:
from llm import judge_log
j = judge_log(operation, expected, full_log)
diagnosis = "\n LLM diagnosis: [{}] {}".format(j.verdict, j.reason)
except Exception:
pass
pytest.fail(
"{} failed ({} chars):\n{}\n{}".format(
operation, len(full_log), full_log, diagnosis)
)
# ── system dependency resolution ────────────────────────────────────
def test_repology_none_config(run_baker, work_dir):
cfg = os.path.join(work_dir, "rust.yml")
with open(cfg, "w") as f:
f.write(_yml("""\
cargo: {}
"""))
r, log = _run_prebake(run_baker, cfg, timeout=120,
operation="UPM sysdep: repology_endpoint=none",
expected="rustup gets installed via pacman")
_check(r, log, "repology-none sysdep resolution",
"pacman installs rustup, empty cargo config")
assert binary_exists("/usr/bin/rustup"), "rustup not installed via sysdep"
# ── rustup: toolchain & components ──────────────────────────────────
@pytest.mark.skipif(
"RUST_UPM_NETWORK" not in os.environ,
reason="set RUST_UPM_NETWORK=1 for tests that download toolchain/crates"
)
def test_toolchain_and_component(run_baker, work_dir):
cfg = os.path.join(work_dir, "rust.yml")
with open(cfg, "w") as f:
f.write(_yml("""\
toolchain: "stable"
components: ["rustfmt"]
"""))
r, log = _run_prebake(run_baker, cfg, timeout=600,
operation="rustup default stable + component add rustfmt",
expected="stable toolchain active, rustfmt installed")
_check(r, log, "toolchain + rustfmt installation",
"rustup default stable succeeds; rustfmt available")
assert binary_exists("/usr/bin/rustup"), "rustup not installed"
r2 = _query(["rustup", "which", "rustc"])
assert r2["success"], "rustc not found via rustup which"
assert os.path.isfile(r2["stdout"].strip()), "rustc binary missing"
# ── cross-compilation targets ───────────────────────────────────────
@pytest.mark.skipif(
"RUST_UPM_NETWORK" not in os.environ,
reason="set RUST_UPM_NETWORK=1 for tests that download toolchain/crates"
)
def test_cross_compile_target(run_baker, work_dir):
cfg = os.path.join(work_dir, "rust.yml")
with open(cfg, "w") as f:
f.write(_yml("""\
toolchain: "stable"
targets: ["wasm32-unknown-unknown"]
"""))
r, log = _run_prebake(run_baker, cfg, timeout=600,
operation="rustup target add wasm32-unknown-unknown",
expected="wasm32 target appears in installed list")
_check(r, log, "cross-compilation target installation",
"wasm32-unknown-unknown is installed")
r2 = _query(["rustup", "target", "list", "--installed"])
assert r2["success"], "rustup target list failed"
assert "wasm32-unknown-unknown" in r2["stdout"], "wasm32 target missing"
# ── cargo fetch ─────────────────────────────────────────────────────
@pytest.mark.skipif(
"RUST_UPM_NETWORK" not in os.environ,
reason="set RUST_UPM_NETWORK=1 for tests that download toolchain/crates"
)
def test_cargo_fetch(run_baker, work_dir):
manifest = _cargo_toml(work_dir)
cfg = os.path.join(work_dir, "rust.yml")
with open(cfg, "w") as f:
f.write(_yml("""\
toolchain: "stable"
cargo:
fetch: true
locked: true
manifests: ["{manifest}"]
""".format(manifest=manifest)))
r, log = _run_prebake(run_baker, cfg, timeout=600,
operation="cargo fetch on minimal Cargo.toml",
expected="~/.cargo/registry/cache populated")
_check(r, log, "cargo fetch dependency pre-fetch",
"cargo fetch succeeds, registry cache non-empty")
cache = os.path.expanduser("~/.cargo/registry/cache")
assert os.path.isdir(cache), "registry/cache missing"
assert len(os.listdir(cache)) > 0, "registry/cache empty after fetch"
# ── cargo install ───────────────────────────────────────────────────
@pytest.mark.skipif(
"RUST_UPM_NETWORK" not in os.environ,
reason="set RUST_UPM_NETWORK=1 for tests that download toolchain/crates"
)
def test_cargo_install(run_baker, work_dir):
cfg = os.path.join(work_dir, "rust.yml")
with open(cfg, "w") as f:
f.write(_yml("""\
toolchain: "stable"
cargo:
packages: ["cargo-audit"]
locked: true
"""))
r, log = _run_prebake(run_baker, cfg, timeout=1200,
operation="cargo install cargo-audit",
expected="cargo-audit binary in ~/.cargo/bin/")
_check(r, log, "cargo install crate",
"cargo-audit compiled and installed")
assert binary_exists(
os.path.expanduser("~/.cargo/bin/cargo-audit")
), "cargo-audit not installed"
# ── full flow ───────────────────────────────────────────────────────
@pytest.mark.skipif(
"RUST_UPM_NETWORK" not in os.environ,
reason="set RUST_UPM_NETWORK=1 for tests that download toolchain/crates"
)
def test_full_flow(run_baker, work_dir):
manifest = _cargo_toml(work_dir)
cfg = os.path.join(work_dir, "rust.yml")
with open(cfg, "w") as f:
f.write(_yml("""\
toolchain: "stable"
components: ["rustfmt"]
targets: ["wasm32-unknown-unknown"]
cargo:
fetch: true
locked: true
manifests: ["{manifest}"]
packages: ["cargo-watch"]
""".format(manifest=manifest)))
r, log = _run_prebake(run_baker, cfg, timeout=1200,
operation="full Rust UPM flow",
expected="all stages succeed")
_check(r, log, "full Rust UPM end-to-end",
"toolchain + cross target + cargo fetch + cargo install all pass")
assert binary_exists("/usr/bin/rustup")
r2 = _query(["rustup", "which", "rustc"])
assert r2["success"]
r3 = _query(["rustup", "target", "list", "--installed"])
assert "wasm32-unknown-unknown" in r3["stdout"]
cache = os.path.expanduser("~/.cargo/registry/cache")
assert os.path.isdir(cache) and len(os.listdir(cache)) > 0
assert binary_exists(os.path.expanduser("~/.cargo/bin/cargo-watch"))
+2 -2
View File
@@ -57,7 +57,7 @@ pub async fn hook(
ctx: &ExecutionContext,
event_tx: EventSender,
) -> Result<(), ExecutionError> {
dbg!(ctx);
// dbg!(ctx);
let hook = match hook {
Some(h) => h,
None => {
@@ -79,7 +79,7 @@ pub async fn hook(
let result = engine
.execute_command_with_delta(&hook_cmd.command, ctx, &event_tx, &deltactx)
.await;
dbg!(&result);
// dbg!(&result);
if let Err(e) = result {
log::error!("Hook {} failed: {:?}", index, e);
return Err(e);
+2 -1
View File
@@ -20,13 +20,14 @@ pub struct ExecutionContext {
pub standalone: bool,
}
#[derive(Debug, Clone)]
#[derive(Debug, Clone, Default)]
pub struct DeltaExecutionContext {
pub working_dir: Option<PathBuf>,
pub env_vars: HashMap<String, Option<String>>,
pub timeout: Option<Duration>,
}
impl Default for ExecutionContext {
fn default() -> Self {
Self {
+2 -1
View File
@@ -2,6 +2,7 @@ use crate::types::EventSender;
use async_trait::async_trait;
use serde_yaml::Value;
mod custom;
mod rust;
use crate::error::DependencyError;
use crate::types::ExecutionContext;
@@ -44,7 +45,7 @@ pub trait UserPackageManager {
/// Returns a list of all available user package managers.
pub fn all_managers() -> Vec<Box<dyn UserPackageManager>> {
vec![Box::new(custom::Custom)]
vec![Box::new(rust::Rust), Box::new(custom::Custom)]
}
/// Selects a user package manager by name.
+316
View File
@@ -0,0 +1,316 @@
use async_trait::async_trait;
use serde::Deserialize;
use std::path::PathBuf;
use crate::error::DependencyError;
use crate::types::{DeltaExecutionContext, Engine, EventSender, ExecutionContext};
use crate::upm::UPMSysDeps;
use super::UserPackageManager;
/// Default manifest paths used when `manifests` is not specified
/// in the `[rust.cargo]` sub-config.
///
/// The bootstrap stage creates `/workspace` as the standard project
/// root. Most Docker-based builds have a single `Cargo.toml` here.
fn default_manifests() -> Vec<PathBuf> {
vec![PathBuf::from("/workspace/Cargo.toml")]
}
#[derive(Clone)]
pub struct Rust;
// ── RustConfig (root) ─────────────────────────────────────────────
/// Top-level configuration for the `rust` UPM.
///
/// ```yaml
/// rust:
/// toolchain: "nightly-2024-01-01"
/// components: ["clippy", "rustfmt"]
/// targets: ["wasm32-unknown-unknown"]
/// cargo:
/// fetch: true
/// locked: true
/// manifests: ["/workspace/Cargo.toml"]
/// packages: ["cargo-audit"]
/// ```
#[derive(Debug, Clone, Deserialize)]
struct RustConfig {
/// Rust toolchain to activate via `rustup default`.
/// If unset, the existing default toolchain is left alone.
#[serde(default)]
toolchain: Option<String>,
/// Rustup components to install (e.g. `clippy`, `rustfmt`).
#[serde(default)]
components: Vec<String>,
/// Compilation targets for cross-compilation.
/// Each entry runs `rustup target add <target>`.
#[serde(default)]
targets: Vec<String>,
/// Cargo sub-configuration (fetch, install, manifests, …).
/// Optional — omit if you only need rustup-level setup.
#[serde(default)]
cargo: CargoSubConfig,
}
// ── Cargo sub-config ───────────────────────────────────────────────
/// Nested cargo configuration inside `rust`.
#[derive(Debug, Clone, Deserialize, Default)]
struct CargoSubConfig {
/// Crate names to install via `cargo install`.
#[serde(default)]
packages: Vec<String>,
/// Alternative registry URL (passed as `--registry`).
#[serde(default)]
registry: Option<String>,
/// Additional flags forwarded to `cargo install`
/// (e.g. `--force`, `--version 0.1.0`).
#[serde(default)]
args: Vec<String>,
/// One or more `Cargo.toml` paths for `cargo fetch`.
/// Each path spawns a separate `cargo fetch --manifest-path <path>`.
#[serde(default = "default_manifests")]
manifests: Vec<PathBuf>,
/// When `true`, run `cargo fetch` before installing packages.
#[serde(default)]
fetch: bool,
/// Require an up-to-date `Cargo.lock` (passes `--locked`).
#[serde(default)]
locked: bool,
}
// ── UserPackageManager impl ────────────────────────────────────────
#[async_trait]
impl UserPackageManager for Rust {
fn name(&self) -> &'static str {
"rust"
}
fn system_dependency(&self, _config: &serde_yaml::Value) -> UPMSysDeps {
UPMSysDeps::new(vec!["rustup".to_string()], false)
}
async fn main(
&self,
config: &serde_yaml::Value,
ctx: &ExecutionContext,
event_tx: &EventSender,
) -> Result<(), DependencyError> {
let cfg: RustConfig = serde_yaml::from_value(config.clone()).map_err(|e| {
log::error!("Failed to parse user.rust config: {}", e);
DependencyError::ParseError()
})?;
let engine = Engine::new();
let mut commands: Vec<String> = Vec::new();
// ── 1. rustup default <toolchain> ─────────────────────────
if let Some(ref tc) = cfg.toolchain {
commands.push(format!("rustup default '{}'", tc));
}
// ── 2. rustup component add <comp…> ──────────────────────
if !cfg.components.is_empty() {
commands.push(format!(
"rustup component add {}",
cfg.components.join(" ")
));
}
// ── 3. rustup target add <target…> ───────────────────────
if !cfg.targets.is_empty() {
commands.push(format!(
"rustup target add {}",
cfg.targets.join(" ")
));
}
// ── 4. cargo fetch ───────────────────────────────────────
//
// `cargo fetch` without --target only fetches host-target
// dependencies. For cross-compilation we must fetch once
// per (manifest, target) pair.
let cargo = &cfg.cargo;
if cargo.fetch {
let mut base = String::from("cargo fetch");
if cargo.locked {
base.push_str(" --locked");
}
for manifest in &cargo.manifests {
// Host target (no --target)
commands.push(format!(
"{} --manifest-path '{}'",
base,
manifest.display()
));
// Cross-compilation targets
for target in &cfg.targets {
commands.push(format!(
"{} --target '{}' --manifest-path '{}'",
base,
target,
manifest.display()
));
}
}
}
// ── 5. cargo install ─────────────────────────────────────
if !cargo.packages.is_empty() {
let mut cmd = String::from("cargo install");
if cargo.locked {
cmd.push_str(" --locked");
}
if let Some(ref registry) = cargo.registry {
cmd.push_str(&format!(" --registry '{}'", registry));
}
for a in &cargo.args {
cmd.push(' ');
cmd.push_str(a);
}
for pkg in &cargo.packages {
cmd.push(' ');
cmd.push_str(pkg);
}
commands.push(cmd);
}
// ── Execute ──────────────────────────────────────────────
for cmd in &commands {
log::info!("Rust UPM executing: {}", cmd);
let result = engine
.execute_command_with_delta(
cmd,
ctx,
event_tx,
&DeltaExecutionContext::default(),
)
.await;
if let Err(e) = result {
log::error!("Rust command failed: {}", e);
return Err(DependencyError::CustomFailed(e));
}
}
if commands.is_empty() {
log::warn!("Rust UPM configured with nothing to do");
}
Ok(())
}
}
// ── Tests ──────────────────────────────────────────────────────────
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_name() {
assert_eq!(Rust.name(), "rust");
}
#[test]
fn test_system_dependency() {
let deps = Rust.system_dependency(&serde_yaml::Value::Null);
assert_eq!(deps.packages, vec!["rustup"]);
assert!(!deps.mapped);
}
#[test]
fn test_parse_empty() {
let yaml = "{}";
let cfg: RustConfig = serde_yaml::from_str(yaml).unwrap();
assert!(cfg.toolchain.is_none());
assert!(cfg.components.is_empty());
assert!(cfg.targets.is_empty());
assert!(!cfg.cargo.fetch);
}
#[test]
fn test_parse_rustup_only() {
let yaml = r#"
toolchain: "nightly-2024-01-01"
components: ["clippy", "rustfmt"]
targets: ["wasm32-unknown-unknown", "x86_64-unknown-linux-musl"]
"#;
let cfg: RustConfig = serde_yaml::from_str(yaml).unwrap();
assert_eq!(cfg.toolchain.as_deref(), Some("nightly-2024-01-01"));
assert_eq!(cfg.components, vec!["clippy", "rustfmt"]);
assert_eq!(
cfg.targets,
vec!["wasm32-unknown-unknown", "x86_64-unknown-linux-musl"]
);
assert!(!cfg.cargo.fetch);
}
#[test]
fn test_parse_cargo_subconfig() {
let yaml = r#"
components: ["clippy"]
cargo:
fetch: true
locked: true
manifests:
- "/workspace/foo/Cargo.toml"
- "/workspace/bar/Cargo.toml"
packages: ["cargo-audit"]
registry: "https://mirrors.tuna.tsinghua.edu.cn/crates.io-index"
"#;
let cfg: RustConfig = serde_yaml::from_str(yaml).unwrap();
assert_eq!(cfg.components, vec!["clippy"]);
let c = &cfg.cargo;
assert!(c.fetch);
assert!(c.locked);
assert_eq!(
c.manifests,
vec![
PathBuf::from("/workspace/foo/Cargo.toml"),
PathBuf::from("/workspace/bar/Cargo.toml"),
]
);
assert_eq!(c.packages, vec!["cargo-audit"]);
assert_eq!(
c.registry.as_deref(),
Some("https://mirrors.tuna.tsinghua.edu.cn/crates.io-index")
);
}
#[test]
fn test_cargo_defaults() {
let yaml = r#"
components: ["rustfmt"]
cargo: {}
"#;
let cfg: RustConfig = serde_yaml::from_str(yaml).unwrap();
assert_eq!(cfg.components, vec!["rustfmt"]);
assert!(!cfg.cargo.fetch);
assert!(!cfg.cargo.locked);
assert_eq!(
cfg.cargo.manifests,
vec![PathBuf::from("/workspace/Cargo.toml")]
);
}
#[test]
fn test_default_manifests() {
assert_eq!(
default_manifests(),
vec![PathBuf::from("/workspace/Cargo.toml")]
);
}
}