From b0530fb858ac8159e77dfad9bbad87e6a28bbecd Mon Sep 17 00:00:00 2001 From: Catty Steve <4795515+Catty2014@user.noreply.gitee.com> Date: Tue, 28 Apr 2026 16:56:53 +0800 Subject: [PATCH] feat(upm): add Python and Go package manager support --- tests/upm/test_python.py | 163 ++++++++++++++ workshop-engine/src/upm.rs | 4 +- workshop-engine/src/upm/go.rs | 176 +++++++++++++++ workshop-engine/src/upm/python.rs | 356 ++++++++++++++++++++++++++++++ workshop-engine/src/upm/rust.rs | 25 ++- 5 files changed, 716 insertions(+), 8 deletions(-) create mode 100644 tests/upm/test_python.py create mode 100644 workshop-engine/src/upm/go.rs create mode 100644 workshop-engine/src/upm/python.rs diff --git a/tests/upm/test_python.py b/tests/upm/test_python.py new file mode 100644 index 0000000..7ed4a80 --- /dev/null +++ b/tests/upm/test_python.py @@ -0,0 +1,163 @@ +"""Python UPM system tests — pip, uv, poetry, pipx via bare prebake.""" + +import os +import pytest +from probes import binary_exists + +RUST_LOG = os.environ.get("RUST_LOG", "info") + + +def _yml(actions_yaml): + return """\ +version: "1.0" +environment: + builder: "baremetal" +bootstrap: + user: "vulcan" +security: + drop_after: never +dependencies: + config: + repology_endpoint: none + system: + pacman: + packages: + user: + python: +""" + _indent(actions_yaml, 6) + + +def _indent(text, spaces): + pad = " " * spaces + return "\n".join(pad + line if line.strip() else line + for line in text.splitlines()) + + +def _query(cmd, timeout=60): + import subprocess + try: + r = subprocess.run(cmd, capture_output=True, text=True, + timeout=timeout, env={**os.environ, "HOME": "/root"}) + return {"success": r.returncode == 0, "stdout": r.stdout, "stderr": r.stderr} + except Exception as e: + return {"success": False, "stdout": "", "stderr": str(e)} + + +def _run_prebake(run_baker, config_path, timeout, operation, expected): + env_vars = {"RUST_LOG": RUST_LOG} + r = run_baker("prebake", config_path, timeout=timeout, env=env_vars) + full_log = "[stdout]\n{}\n[stderr]\n{}".format(r["stdout"], r["stderr"]) + return r, full_log + + +def _check(r, full_log, operation, expected): + if r["success"]: + return + diagnosis = "" + try: + from llm import judge_log + j = judge_log(operation, expected, full_log) + diagnosis = "\n LLM diagnosis: [{}] {}".format(j.verdict, j.reason) + except Exception: + pass + pytest.fail("{} failed ({} chars):\n{}\n{}".format( + operation, len(full_log), full_log, diagnosis)) + + +# ── pip ───────────────────────────────────────────────────────────── + +def test_pip_install_package(run_baker, work_dir): + """Install a package via pip.""" + cfg = os.path.join(work_dir, "python.yml") + with open(cfg, "w") as f: + f.write(_yml("""\ +- type: pip + packages: [requests] + index: "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple" + args: ["--break-system-packages"] +""")) + + r, log = _run_prebake(run_baker, cfg, timeout=120, + operation="pip install requests", expected="requests module importable") + _check(r, log, "pip install", "pip installs requests successfully") + + r2 = _query(["python3", "-c", "import requests"]) + assert r2["success"], f"import requests failed: {r2['stderr']}" + + +# ── pip + venv ────────────────────────────────────────────────────── + +def test_pip_venv_install(run_baker, work_dir): + """Create venv and install inside it.""" + cfg = os.path.join(work_dir, "python.yml") + with open(cfg, "w") as f: + f.write(_yml("""\ +- type: pip + directory: "/tmp/test-venv" + index: "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple" + packages: [requests] +""")) + + r, log = _run_prebake(run_baker, cfg, timeout=120, + operation="venv + pip install", + expected="requests installed inside venv") + _check(r, log, "pip venv install", "venv created, requests installed") + + r2 = _query(["/tmp/test-venv/bin/python3", "-c", "import requests"]) + assert r2["success"], f"import from venv failed: {r2['stderr']}" + + +# ── pip + requirements ────────────────────────────────────────────── + +def test_pip_requirements_file(run_baker, work_dir): + """Install from a requirements.txt file.""" + req_path = os.path.join(work_dir, "requirements.txt") + with open(req_path, "w") as f: + f.write("requests\n") + + cfg = os.path.join(work_dir, "python.yml") + with open(cfg, "w") as f: + f.write(_yml("""\ +- type: pip + directory: "/tmp/test-venv2" + index: "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple" + manifest: "{}" +""".format(req_path))) + + r, log = _run_prebake(run_baker, cfg, timeout=120, + operation="pip install -r requirements.txt", + expected="requests installed from requirements file") + _check(r, log, "pip -r requirements.txt", "pip installs from requirements file") + + r2 = _query(["/tmp/test-venv2/bin/python3", "-c", "import requests"]) + assert r2["success"], f"import from venv failed: {r2['stderr']}" + + +# ── multiple actions ──────────────────────────────────────────────── + +def test_multiple_actions(run_baker, work_dir): + """Two pip actions in one config.""" + cfg = os.path.join(work_dir, "python.yml") + with open(cfg, "w") as f: + f.write(_yml("""\ +- type: pip + directory: "/tmp/test-venv-a" + index: "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple" + packages: [requests] + args: ["--break-system-packages"] +- type: pip + directory: "/tmp/test-venv-b" + index: "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple" + packages: [pytest] +""")) + + r, log = _run_prebake(run_baker, cfg, timeout=180, + operation="two pip actions", + expected="both venvs have their packages") + _check(r, log, "multiple pip actions", "both requests and pytest installed") + + r2 = _query(["/tmp/test-venv-a/bin/python3", "-c", "import requests"]) + assert r2["success"], f"venv-a missing requests: {r2['stderr']}" + + r3 = _query(["/tmp/test-venv-b/bin/python3", "-c", "import pytest"]) + assert r3["success"], f"venv-b missing pytest: {r3['stderr']}" diff --git a/workshop-engine/src/upm.rs b/workshop-engine/src/upm.rs index 54c3d21..49d0d18 100644 --- a/workshop-engine/src/upm.rs +++ b/workshop-engine/src/upm.rs @@ -2,6 +2,8 @@ use crate::types::EventSender; use async_trait::async_trait; use serde_yaml::Value; mod custom; +mod go; +mod python; mod rust; use crate::error::DependencyError; @@ -45,7 +47,7 @@ pub trait UserPackageManager { /// Returns a list of all available user package managers. pub fn all_managers() -> Vec> { - vec![Box::new(rust::Rust), Box::new(custom::Custom)] + vec![Box::new(rust::Rust), Box::new(python::Python), Box::new(go::Go), Box::new(custom::Custom)] } /// Selects a user package manager by name. diff --git a/workshop-engine/src/upm/go.rs b/workshop-engine/src/upm/go.rs new file mode 100644 index 0000000..d92717d --- /dev/null +++ b/workshop-engine/src/upm/go.rs @@ -0,0 +1,176 @@ +use async_trait::async_trait; +use serde::Deserialize; +use std::collections::HashMap; +use std::path::PathBuf; + +use crate::error::DependencyError; +use crate::types::{DeltaExecutionContext, Engine, EventSender, ExecutionContext}; +use crate::upm::UPMSysDeps; + +use super::UserPackageManager; + +#[derive(Clone)] +pub struct Go; + +fn default_go_type() -> String { + "go".to_string() +} + +// ── GoConfig ──────────────────────────────────────────────────────── + +#[derive(Debug, Clone, Deserialize)] +#[allow(dead_code)] +struct GoConfig { + /// UPM backend type (currently only `"go"`). + #[serde(default = "default_go_type", rename = "type")] + type_: String, + + /// Go module proxy URL (sets `GOPROXY`). + #[serde(default)] + proxy: Option, + + /// Packages to install via `go install`. + /// Use `@version` suffix, e.g. `golang.org/x/tools/cmd/goimports@latest`. + #[serde(default)] + packages: Vec, + + /// `go.mod` paths for `go mod download`. + /// Each path's directory is `cd`'d into before downloading. + #[serde(default)] + modules: Vec, + + /// Environment variables injected into child processes. + #[serde(default, rename = "env")] + env_: HashMap>, +} + +// ── UserPackageManager impl ───────────────────────────────────────── + +#[async_trait] +impl UserPackageManager for Go { + fn name(&self) -> &'static str { + "go" + } + + fn system_dependency(&self, _config: &serde_yaml::Value) -> UPMSysDeps { + UPMSysDeps::new(vec!["go".to_string()], false) + } + + async fn main( + &self, + config: &serde_yaml::Value, + ctx: &ExecutionContext, + event_tx: &EventSender, + ) -> Result<(), DependencyError> { + let cfg: GoConfig = serde_yaml::from_value(config.clone()).map_err(|e| { + log::error!("Failed to parse user.go config: {}", e); + DependencyError::ParseError() + })?; + + let engine = Engine::new(); + let mut delta_env = cfg.env_.clone(); + + // Inject GOPROXY if configured + if let Some(ref proxy) = cfg.proxy { + delta_env.insert("GOPROXY".into(), Some(proxy.clone())); + } + + let delta = DeltaExecutionContext { + env_vars: delta_env, + ..Default::default() + }; + + // ── go mod download ────────────────────────────────────── + for mod_path in &cfg.modules { + let dir = mod_path.parent().unwrap_or(mod_path); + let cmd = "go mod download".to_string(); + log::info!( + "Go UPM executing: cd {} && {}", + dir.display(), + cmd + ); + + let cd_delta = DeltaExecutionContext { + working_dir: Some(dir.to_path_buf()), + env_vars: delta.env_vars.clone(), + ..Default::default() + }; + + let result = engine + .execute_command_with_delta(&cmd, ctx, event_tx, &cd_delta) + .await; + if let Err(e) = result { + log::error!("go mod download failed in {}: {}", dir.display(), e); + return Err(DependencyError::CustomFailed(e)); + } + } + + // ── go install ────────────────────────────────────────── + if !cfg.packages.is_empty() { + let cmd = format!("go install {}", cfg.packages.join(" ")); + log::info!("Go UPM executing: {}", cmd); + let result = engine + .execute_command_with_delta(&cmd, ctx, event_tx, &delta) + .await; + if let Err(e) = result { + log::error!("Go install failed: {}", e); + return Err(DependencyError::CustomFailed(e)); + } + } + + if cfg.modules.is_empty() && cfg.packages.is_empty() { + log::warn!("Go UPM configured with nothing to do"); + } + + Ok(()) + } +} + +// ── Tests ────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_name() { + assert_eq!(Go.name(), "go"); + } + + #[test] + fn test_system_dependency() { + let deps = Go.system_dependency(&serde_yaml::Value::Null); + assert_eq!(deps.packages, vec!["go"]); + assert!(!deps.mapped); + } + + #[test] + fn test_parse_minimal() { + let yaml = r#" +packages: ["golang.org/x/tools/cmd/goimports@latest"] +"#; + let cfg: GoConfig = serde_yaml::from_str(yaml).unwrap(); + assert!(cfg.proxy.is_none()); + assert_eq!(cfg.packages.len(), 1); + } + + #[test] + fn test_parse_full() { + let yaml = r#" +proxy: "https://goproxy.cn,direct" +packages: ["honnef.co/go/tools/cmd/staticcheck@latest"] +modules: + - "/workspace/server/go.mod" +env: + GOMODCACHE: "/tmp/gocache" +"#; + let cfg: GoConfig = serde_yaml::from_str(yaml).unwrap(); + assert_eq!(cfg.proxy.as_deref(), Some("https://goproxy.cn,direct")); + assert_eq!(cfg.packages, vec!["honnef.co/go/tools/cmd/staticcheck@latest"]); + assert_eq!(cfg.modules, vec![PathBuf::from("/workspace/server/go.mod")]); + assert_eq!( + cfg.env_.get("GOMODCACHE").and_then(|v| v.as_deref()), + Some("/tmp/gocache"), + ); + } +} diff --git a/workshop-engine/src/upm/python.rs b/workshop-engine/src/upm/python.rs new file mode 100644 index 0000000..6a8617c --- /dev/null +++ b/workshop-engine/src/upm/python.rs @@ -0,0 +1,356 @@ +use async_trait::async_trait; +use serde::Deserialize; +use std::collections::HashMap; +use std::path::PathBuf; + +use crate::error::DependencyError; +use crate::types::{DeltaExecutionContext, Engine, EventSender, ExecutionContext}; +use crate::upm::UPMSysDeps; + +use super::UserPackageManager; + +#[derive(Clone)] +pub struct Python; + +fn default_type() -> String { + "pip".to_string() +} + +// ── PythonAction (one entry in the list) ──────────────────────────── + +/// A single Python package-management operation. +/// +/// `type` selects the backend: ``pip``, ``uv``, ``poetry``, or ``pipx``. +/// Everything else applies to that backend's invocation. +#[derive(Debug, Clone, Deserialize)] +struct PythonAction { + /// Backend tool name. + #[serde(default = "default_type", rename = "type")] + type_: String, + + /// Python interpreter to use (default: system ``python3``). + #[serde(default)] + python: Option, + + /// Virtualenv directory — created if set. + #[serde(default)] + directory: Option, + + /// PyPI index URL (``-i`` / ``--index-url``). + #[serde(default)] + index: Option, + + /// Package names. + #[serde(default)] + packages: Vec, + + /// Requirements file or pyproject.toml. + #[serde(default)] + manifest: Option, + + /// Extra CLI arguments forwarded to the backend. + #[serde(default)] + args: Vec, + + /// Environment variables injected into the child process. + #[serde(default, rename = "env")] + env_: HashMap>, +} + +// ── PythonConfig (top-level, transparent vec) ─────────────────────── + +#[derive(Debug, Clone, Deserialize)] +#[serde(transparent)] +struct PythonConfig(Vec); + +// ── helpers ───────────────────────────────────────────────────────── + +fn python_bin(action: &PythonAction) -> &str { + action.python.as_deref().unwrap_or("python3") +} + +fn index_flag(action: &PythonAction) -> String { + match &action.index { + Some(idx) => format!(" -i '{}'", idx), + None => String::new(), + } +} + +fn env_delta(action: &PythonAction) -> DeltaExecutionContext { + DeltaExecutionContext { + env_vars: action.env_.clone(), + ..Default::default() + } +} + +fn venv_pip(action: &PythonAction) -> String { + match &action.directory { + Some(dir) => format!("{}/bin/pip", dir.display()), + None => format!("{} -m pip", python_bin(action)), + } +} + +// ── UserPackageManager impl ───────────────────────────────────────── + +#[async_trait] +impl UserPackageManager for Python { + fn name(&self) -> &'static str { + "python" + } + + fn system_dependency(&self, config: &serde_yaml::Value) -> UPMSysDeps { + let mut pkgs = vec!["python3".to_string()]; + + if let Ok(cfg) = serde_yaml::from_value::(config.clone()) { + for a in &cfg.0 { + match a.type_.as_str() { + "uv" => pkgs.push("uv".into()), + "poetry" => pkgs.push("poetry".into()), + "pipx" => pkgs.push("pipx".into()), + _ => {} + } + } + pkgs.sort(); + pkgs.dedup(); + } + + UPMSysDeps::new(pkgs, false) + } + + async fn main( + &self, + config: &serde_yaml::Value, + ctx: &ExecutionContext, + event_tx: &EventSender, + ) -> Result<(), DependencyError> { + let cfg: PythonConfig = + serde_yaml::from_value(config.clone()).map_err(|e| { + log::error!("Failed to parse user.python config: {}", e); + DependencyError::ParseError() + })?; + + let engine = Engine::new(); + + for action in &cfg.0 { + self.run_action(&engine, action, ctx, event_tx).await?; + } + + Ok(()) + } +} + +impl Python { + async fn run_action( + &self, + engine: &Engine, + action: &PythonAction, + ctx: &ExecutionContext, + event_tx: &EventSender, + ) -> Result<(), DependencyError> { + let delta = env_delta(action); + let idx = index_flag(action); + + // ── 1. venv ────────────────────────────────────────────── + if let Some(ref dir) = action.directory { + let venv_cmd = + format!("{} -m venv '{}'", python_bin(action), dir.display()); + log::info!("Python UPM creating venv: {}", venv_cmd); + let r = engine + .execute_command_with_delta(&venv_cmd, ctx, event_tx, &delta) + .await; + if let Err(e) = r { + log::error!("Failed to create venv: {}", e); + return Err(DependencyError::CustomFailed(e)); + } + } + + // ── 2. backend command ────────────────────────────────── + match action.type_.as_str() { + "pip" => { + if !action.packages.is_empty() { + let cmd = format!( + "{} install{}{} {}", + venv_pip(action), + idx, + action.args.iter().fold(String::new(), |a, b| a + " " + b), + action.packages.join(" "), + ); + log::info!("Python UPM executing: {}", cmd); + engine + .execute_command_with_delta(&cmd, ctx, event_tx, &delta) + .await + .map_err(DependencyError::CustomFailed)?; + } + if let Some(ref manifest) = action.manifest { + let cmd = format!( + "{} install{} -r '{}'{}", + venv_pip(action), + idx, + manifest.display(), + action.args.iter().fold(String::new(), |a, b| a + " " + b), + ); + log::info!("Python UPM executing: {}", cmd); + engine + .execute_command_with_delta(&cmd, ctx, event_tx, &delta) + .await + .map_err(DependencyError::CustomFailed)?; + } + } + + "uv" => { + // uv pip install / uv pip sync + if !action.packages.is_empty() { + let cmd = format!( + "uv pip install{}{} {}", + idx, + action.args.iter().fold(String::new(), |a, b| a + " " + b), + action.packages.join(" "), + ); + log::info!("Python UPM executing: {}", cmd); + engine + .execute_command_with_delta(&cmd, ctx, event_tx, &delta) + .await + .map_err(DependencyError::CustomFailed)?; + } + if let Some(ref manifest) = action.manifest { + let cmd = format!( + "uv pip sync '{}'{}", + manifest.display(), + action.args.iter().fold(String::new(), |a, b| a + " " + b), + ); + log::info!("Python UPM executing: {}", cmd); + engine + .execute_command_with_delta(&cmd, ctx, event_tx, &delta) + .await + .map_err(DependencyError::CustomFailed)?; + } + } + + "poetry" => { + let cmd = format!( + "poetry install{}", + action.args.iter().fold(String::new(), |a, b| a + " " + b), + ); + log::info!("Python UPM executing: {}", cmd); + engine + .execute_command_with_delta(&cmd, ctx, event_tx, &delta) + .await + .map_err(DependencyError::CustomFailed)?; + } + + "pipx" => { + if !action.packages.is_empty() { + let cmd = format!( + "pipx install{}{} {}", + idx, + action.args.iter().fold(String::new(), |a, b| a + " " + b), + action.packages.join(" "), + ); + log::info!("Python UPM executing: {}", cmd); + engine + .execute_command_with_delta(&cmd, ctx, event_tx, &delta) + .await + .map_err(DependencyError::CustomFailed)?; + } + } + + other => { + log::error!("Unknown Python backend type: {}", other); + return Err(DependencyError::ParseError()); + } + } + + Ok(()) + } +} + +// ── Tests ────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_name() { + assert_eq!(Python.name(), "python"); + } + + #[test] + fn test_system_dependency_basic() { + let yaml = "- pip: {}"; + let v: serde_yaml::Value = serde_yaml::from_str(yaml).unwrap(); + let deps = Python.system_dependency(&v); + assert_eq!(deps.packages, vec!["python3"]); + } + + #[test] + fn test_system_dependency_uv() { + let yaml = r#" +- type: uv + packages: [pytest] +"#; + let v: serde_yaml::Value = serde_yaml::from_str(yaml).unwrap(); + let deps = Python.system_dependency(&v); + assert!(deps.packages.contains(&"python3".to_string())); + assert!(deps.packages.contains(&"uv".to_string())); + } + + #[test] + fn test_parse_pip_action() { + let yaml = r#" +- type: pip + packages: [pytest, requests] +"#; + let cfg: PythonConfig = serde_yaml::from_str(yaml).unwrap(); + assert_eq!(cfg.0.len(), 1); + let a = &cfg.0[0]; + assert_eq!(a.type_, "pip"); + assert_eq!(a.packages, vec!["pytest", "requests"]); + } + + #[test] + fn test_parse_full_action() { + let yaml = r#" +- type: uv + python: "python3.14" + directory: ".venv" + index: "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple" + packages: [pytest, requests] + manifest: "requirements.txt" + args: ["--upgrade"] + env: + UV_PYTHON: "python3.14" +"#; + let cfg: PythonConfig = serde_yaml::from_str(yaml).unwrap(); + let a = &cfg.0[0]; + assert_eq!(a.type_, "uv"); + assert_eq!(a.python.as_deref(), Some("python3.14")); + assert_eq!(a.directory, Some(PathBuf::from(".venv"))); + assert_eq!( + a.index.as_deref(), + Some("https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"), + ); + assert_eq!(a.packages, vec!["pytest", "requests"]); + assert_eq!(a.manifest, Some(PathBuf::from("requirements.txt"))); + assert_eq!(a.args, vec!["--upgrade"]); + assert_eq!( + a.env_.get("UV_PYTHON").and_then(|v| v.as_deref()), + Some("python3.14"), + ); + } + + #[test] + fn test_parse_multiple_actions() { + let yaml = r#" +- type: pip + packages: [pytest] +- type: uv + directory: ".venv" + manifest: "requirements.txt" +"#; + let cfg: PythonConfig = serde_yaml::from_str(yaml).unwrap(); + assert_eq!(cfg.0.len(), 2); + assert_eq!(cfg.0[0].type_, "pip"); + assert_eq!(cfg.0[1].type_, "uv"); + } +} diff --git a/workshop-engine/src/upm/rust.rs b/workshop-engine/src/upm/rust.rs index 96b29b2..e180ff0 100644 --- a/workshop-engine/src/upm/rust.rs +++ b/workshop-engine/src/upm/rust.rs @@ -1,7 +1,7 @@ use async_trait::async_trait; use serde::Deserialize; +use std::collections::HashMap; use std::path::PathBuf; - use crate::error::DependencyError; use crate::types::{DeltaExecutionContext, Engine, EventSender, ExecutionContext}; use crate::upm::UPMSysDeps; @@ -37,6 +37,16 @@ pub struct Rust; /// ``` #[derive(Debug, Clone, Deserialize)] struct RustConfig { + #[allow(dead_code)] + /// UPM backend type (currently only `"rustup"`). + #[serde(default = "default_rust_type", rename = "type")] + type_: String, + + /// Environment variables injected into every rustup/cargo child process. + /// Each entry maps to a var name; `None` unsets the variable. + #[serde(default, rename = "env")] + env_: HashMap>, + /// Rust toolchain to activate via `rustup default`. /// If unset, the existing default toolchain is left alone. #[serde(default)] @@ -57,6 +67,8 @@ struct RustConfig { cargo: CargoSubConfig, } +fn default_rust_type() -> String { "rustup".to_string() } + // ── Cargo sub-config ─────────────────────────────────────────────── /// Nested cargo configuration inside `rust`. @@ -190,13 +202,12 @@ impl UserPackageManager for Rust { // ── Execute ────────────────────────────────────────────── for cmd in &commands { log::info!("Rust UPM executing: {}", cmd); + let delta = DeltaExecutionContext { + env_vars: cfg.env_.clone(), + ..Default::default() + }; let result = engine - .execute_command_with_delta( - cmd, - ctx, - event_tx, - &DeltaExecutionContext::default(), - ) + .execute_command_with_delta(cmd, ctx, event_tx, &delta) .await; if let Err(e) = result { log::error!("Rust command failed: {}", e);