feat(upm): add Python and Go package manager support

This commit is contained in:
Catty Steve
2026-04-28 16:56:53 +08:00
parent fd18b547e5
commit b0530fb858
5 changed files with 716 additions and 8 deletions
+163
View File
@@ -0,0 +1,163 @@
"""Python UPM system tests — pip, uv, poetry, pipx via bare prebake."""
import os
import pytest
from probes import binary_exists
RUST_LOG = os.environ.get("RUST_LOG", "info")
def _yml(actions_yaml):
return """\
version: "1.0"
environment:
builder: "baremetal"
bootstrap:
user: "vulcan"
security:
drop_after: never
dependencies:
config:
repology_endpoint: none
system:
pacman:
packages:
user:
python:
""" + _indent(actions_yaml, 6)
def _indent(text, spaces):
pad = " " * spaces
return "\n".join(pad + line if line.strip() else line
for line in text.splitlines())
def _query(cmd, timeout=60):
import subprocess
try:
r = subprocess.run(cmd, capture_output=True, text=True,
timeout=timeout, env={**os.environ, "HOME": "/root"})
return {"success": r.returncode == 0, "stdout": r.stdout, "stderr": r.stderr}
except Exception as e:
return {"success": False, "stdout": "", "stderr": str(e)}
def _run_prebake(run_baker, config_path, timeout, operation, expected):
env_vars = {"RUST_LOG": RUST_LOG}
r = run_baker("prebake", config_path, timeout=timeout, env=env_vars)
full_log = "[stdout]\n{}\n[stderr]\n{}".format(r["stdout"], r["stderr"])
return r, full_log
def _check(r, full_log, operation, expected):
if r["success"]:
return
diagnosis = ""
try:
from llm import judge_log
j = judge_log(operation, expected, full_log)
diagnosis = "\n LLM diagnosis: [{}] {}".format(j.verdict, j.reason)
except Exception:
pass
pytest.fail("{} failed ({} chars):\n{}\n{}".format(
operation, len(full_log), full_log, diagnosis))
# ── pip ─────────────────────────────────────────────────────────────
def test_pip_install_package(run_baker, work_dir):
"""Install a package via pip."""
cfg = os.path.join(work_dir, "python.yml")
with open(cfg, "w") as f:
f.write(_yml("""\
- type: pip
packages: [requests]
index: "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
args: ["--break-system-packages"]
"""))
r, log = _run_prebake(run_baker, cfg, timeout=120,
operation="pip install requests", expected="requests module importable")
_check(r, log, "pip install", "pip installs requests successfully")
r2 = _query(["python3", "-c", "import requests"])
assert r2["success"], f"import requests failed: {r2['stderr']}"
# ── pip + venv ──────────────────────────────────────────────────────
def test_pip_venv_install(run_baker, work_dir):
"""Create venv and install inside it."""
cfg = os.path.join(work_dir, "python.yml")
with open(cfg, "w") as f:
f.write(_yml("""\
- type: pip
directory: "/tmp/test-venv"
index: "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
packages: [requests]
"""))
r, log = _run_prebake(run_baker, cfg, timeout=120,
operation="venv + pip install",
expected="requests installed inside venv")
_check(r, log, "pip venv install", "venv created, requests installed")
r2 = _query(["/tmp/test-venv/bin/python3", "-c", "import requests"])
assert r2["success"], f"import from venv failed: {r2['stderr']}"
# ── pip + requirements ──────────────────────────────────────────────
def test_pip_requirements_file(run_baker, work_dir):
"""Install from a requirements.txt file."""
req_path = os.path.join(work_dir, "requirements.txt")
with open(req_path, "w") as f:
f.write("requests\n")
cfg = os.path.join(work_dir, "python.yml")
with open(cfg, "w") as f:
f.write(_yml("""\
- type: pip
directory: "/tmp/test-venv2"
index: "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
manifest: "{}"
""".format(req_path)))
r, log = _run_prebake(run_baker, cfg, timeout=120,
operation="pip install -r requirements.txt",
expected="requests installed from requirements file")
_check(r, log, "pip -r requirements.txt", "pip installs from requirements file")
r2 = _query(["/tmp/test-venv2/bin/python3", "-c", "import requests"])
assert r2["success"], f"import from venv failed: {r2['stderr']}"
# ── multiple actions ────────────────────────────────────────────────
def test_multiple_actions(run_baker, work_dir):
"""Two pip actions in one config."""
cfg = os.path.join(work_dir, "python.yml")
with open(cfg, "w") as f:
f.write(_yml("""\
- type: pip
directory: "/tmp/test-venv-a"
index: "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
packages: [requests]
args: ["--break-system-packages"]
- type: pip
directory: "/tmp/test-venv-b"
index: "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
packages: [pytest]
"""))
r, log = _run_prebake(run_baker, cfg, timeout=180,
operation="two pip actions",
expected="both venvs have their packages")
_check(r, log, "multiple pip actions", "both requests and pytest installed")
r2 = _query(["/tmp/test-venv-a/bin/python3", "-c", "import requests"])
assert r2["success"], f"venv-a missing requests: {r2['stderr']}"
r3 = _query(["/tmp/test-venv-b/bin/python3", "-c", "import pytest"])
assert r3["success"], f"venv-b missing pytest: {r3['stderr']}"
+3 -1
View File
@@ -2,6 +2,8 @@ use crate::types::EventSender;
use async_trait::async_trait;
use serde_yaml::Value;
mod custom;
mod go;
mod python;
mod rust;
use crate::error::DependencyError;
@@ -45,7 +47,7 @@ pub trait UserPackageManager {
/// Returns a list of all available user package managers.
pub fn all_managers() -> Vec<Box<dyn UserPackageManager>> {
vec![Box::new(rust::Rust), Box::new(custom::Custom)]
vec![Box::new(rust::Rust), Box::new(python::Python), Box::new(go::Go), Box::new(custom::Custom)]
}
/// Selects a user package manager by name.
+176
View File
@@ -0,0 +1,176 @@
use async_trait::async_trait;
use serde::Deserialize;
use std::collections::HashMap;
use std::path::PathBuf;
use crate::error::DependencyError;
use crate::types::{DeltaExecutionContext, Engine, EventSender, ExecutionContext};
use crate::upm::UPMSysDeps;
use super::UserPackageManager;
#[derive(Clone)]
pub struct Go;
fn default_go_type() -> String {
"go".to_string()
}
// ── GoConfig ────────────────────────────────────────────────────────
#[derive(Debug, Clone, Deserialize)]
#[allow(dead_code)]
struct GoConfig {
/// UPM backend type (currently only `"go"`).
#[serde(default = "default_go_type", rename = "type")]
type_: String,
/// Go module proxy URL (sets `GOPROXY`).
#[serde(default)]
proxy: Option<String>,
/// Packages to install via `go install`.
/// Use `@version` suffix, e.g. `golang.org/x/tools/cmd/goimports@latest`.
#[serde(default)]
packages: Vec<String>,
/// `go.mod` paths for `go mod download`.
/// Each path's directory is `cd`'d into before downloading.
#[serde(default)]
modules: Vec<PathBuf>,
/// Environment variables injected into child processes.
#[serde(default, rename = "env")]
env_: HashMap<String, Option<String>>,
}
// ── UserPackageManager impl ─────────────────────────────────────────
#[async_trait]
impl UserPackageManager for Go {
fn name(&self) -> &'static str {
"go"
}
fn system_dependency(&self, _config: &serde_yaml::Value) -> UPMSysDeps {
UPMSysDeps::new(vec!["go".to_string()], false)
}
async fn main(
&self,
config: &serde_yaml::Value,
ctx: &ExecutionContext,
event_tx: &EventSender,
) -> Result<(), DependencyError> {
let cfg: GoConfig = serde_yaml::from_value(config.clone()).map_err(|e| {
log::error!("Failed to parse user.go config: {}", e);
DependencyError::ParseError()
})?;
let engine = Engine::new();
let mut delta_env = cfg.env_.clone();
// Inject GOPROXY if configured
if let Some(ref proxy) = cfg.proxy {
delta_env.insert("GOPROXY".into(), Some(proxy.clone()));
}
let delta = DeltaExecutionContext {
env_vars: delta_env,
..Default::default()
};
// ── go mod download ──────────────────────────────────────
for mod_path in &cfg.modules {
let dir = mod_path.parent().unwrap_or(mod_path);
let cmd = "go mod download".to_string();
log::info!(
"Go UPM executing: cd {} && {}",
dir.display(),
cmd
);
let cd_delta = DeltaExecutionContext {
working_dir: Some(dir.to_path_buf()),
env_vars: delta.env_vars.clone(),
..Default::default()
};
let result = engine
.execute_command_with_delta(&cmd, ctx, event_tx, &cd_delta)
.await;
if let Err(e) = result {
log::error!("go mod download failed in {}: {}", dir.display(), e);
return Err(DependencyError::CustomFailed(e));
}
}
// ── go install ──────────────────────────────────────────
if !cfg.packages.is_empty() {
let cmd = format!("go install {}", cfg.packages.join(" "));
log::info!("Go UPM executing: {}", cmd);
let result = engine
.execute_command_with_delta(&cmd, ctx, event_tx, &delta)
.await;
if let Err(e) = result {
log::error!("Go install failed: {}", e);
return Err(DependencyError::CustomFailed(e));
}
}
if cfg.modules.is_empty() && cfg.packages.is_empty() {
log::warn!("Go UPM configured with nothing to do");
}
Ok(())
}
}
// ── Tests ──────────────────────────────────────────────────────────
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_name() {
assert_eq!(Go.name(), "go");
}
#[test]
fn test_system_dependency() {
let deps = Go.system_dependency(&serde_yaml::Value::Null);
assert_eq!(deps.packages, vec!["go"]);
assert!(!deps.mapped);
}
#[test]
fn test_parse_minimal() {
let yaml = r#"
packages: ["golang.org/x/tools/cmd/goimports@latest"]
"#;
let cfg: GoConfig = serde_yaml::from_str(yaml).unwrap();
assert!(cfg.proxy.is_none());
assert_eq!(cfg.packages.len(), 1);
}
#[test]
fn test_parse_full() {
let yaml = r#"
proxy: "https://goproxy.cn,direct"
packages: ["honnef.co/go/tools/cmd/staticcheck@latest"]
modules:
- "/workspace/server/go.mod"
env:
GOMODCACHE: "/tmp/gocache"
"#;
let cfg: GoConfig = serde_yaml::from_str(yaml).unwrap();
assert_eq!(cfg.proxy.as_deref(), Some("https://goproxy.cn,direct"));
assert_eq!(cfg.packages, vec!["honnef.co/go/tools/cmd/staticcheck@latest"]);
assert_eq!(cfg.modules, vec![PathBuf::from("/workspace/server/go.mod")]);
assert_eq!(
cfg.env_.get("GOMODCACHE").and_then(|v| v.as_deref()),
Some("/tmp/gocache"),
);
}
}
+356
View File
@@ -0,0 +1,356 @@
use async_trait::async_trait;
use serde::Deserialize;
use std::collections::HashMap;
use std::path::PathBuf;
use crate::error::DependencyError;
use crate::types::{DeltaExecutionContext, Engine, EventSender, ExecutionContext};
use crate::upm::UPMSysDeps;
use super::UserPackageManager;
#[derive(Clone)]
pub struct Python;
fn default_type() -> String {
"pip".to_string()
}
// ── PythonAction (one entry in the list) ────────────────────────────
/// A single Python package-management operation.
///
/// `type` selects the backend: ``pip``, ``uv``, ``poetry``, or ``pipx``.
/// Everything else applies to that backend's invocation.
#[derive(Debug, Clone, Deserialize)]
struct PythonAction {
/// Backend tool name.
#[serde(default = "default_type", rename = "type")]
type_: String,
/// Python interpreter to use (default: system ``python3``).
#[serde(default)]
python: Option<String>,
/// Virtualenv directory — created if set.
#[serde(default)]
directory: Option<PathBuf>,
/// PyPI index URL (``-i`` / ``--index-url``).
#[serde(default)]
index: Option<String>,
/// Package names.
#[serde(default)]
packages: Vec<String>,
/// Requirements file or pyproject.toml.
#[serde(default)]
manifest: Option<PathBuf>,
/// Extra CLI arguments forwarded to the backend.
#[serde(default)]
args: Vec<String>,
/// Environment variables injected into the child process.
#[serde(default, rename = "env")]
env_: HashMap<String, Option<String>>,
}
// ── PythonConfig (top-level, transparent vec) ───────────────────────
#[derive(Debug, Clone, Deserialize)]
#[serde(transparent)]
struct PythonConfig(Vec<PythonAction>);
// ── helpers ─────────────────────────────────────────────────────────
fn python_bin(action: &PythonAction) -> &str {
action.python.as_deref().unwrap_or("python3")
}
fn index_flag(action: &PythonAction) -> String {
match &action.index {
Some(idx) => format!(" -i '{}'", idx),
None => String::new(),
}
}
fn env_delta(action: &PythonAction) -> DeltaExecutionContext {
DeltaExecutionContext {
env_vars: action.env_.clone(),
..Default::default()
}
}
fn venv_pip(action: &PythonAction) -> String {
match &action.directory {
Some(dir) => format!("{}/bin/pip", dir.display()),
None => format!("{} -m pip", python_bin(action)),
}
}
// ── UserPackageManager impl ─────────────────────────────────────────
#[async_trait]
impl UserPackageManager for Python {
fn name(&self) -> &'static str {
"python"
}
fn system_dependency(&self, config: &serde_yaml::Value) -> UPMSysDeps {
let mut pkgs = vec!["python3".to_string()];
if let Ok(cfg) = serde_yaml::from_value::<PythonConfig>(config.clone()) {
for a in &cfg.0 {
match a.type_.as_str() {
"uv" => pkgs.push("uv".into()),
"poetry" => pkgs.push("poetry".into()),
"pipx" => pkgs.push("pipx".into()),
_ => {}
}
}
pkgs.sort();
pkgs.dedup();
}
UPMSysDeps::new(pkgs, false)
}
async fn main(
&self,
config: &serde_yaml::Value,
ctx: &ExecutionContext,
event_tx: &EventSender,
) -> Result<(), DependencyError> {
let cfg: PythonConfig =
serde_yaml::from_value(config.clone()).map_err(|e| {
log::error!("Failed to parse user.python config: {}", e);
DependencyError::ParseError()
})?;
let engine = Engine::new();
for action in &cfg.0 {
self.run_action(&engine, action, ctx, event_tx).await?;
}
Ok(())
}
}
impl Python {
async fn run_action(
&self,
engine: &Engine,
action: &PythonAction,
ctx: &ExecutionContext,
event_tx: &EventSender,
) -> Result<(), DependencyError> {
let delta = env_delta(action);
let idx = index_flag(action);
// ── 1. venv ──────────────────────────────────────────────
if let Some(ref dir) = action.directory {
let venv_cmd =
format!("{} -m venv '{}'", python_bin(action), dir.display());
log::info!("Python UPM creating venv: {}", venv_cmd);
let r = engine
.execute_command_with_delta(&venv_cmd, ctx, event_tx, &delta)
.await;
if let Err(e) = r {
log::error!("Failed to create venv: {}", e);
return Err(DependencyError::CustomFailed(e));
}
}
// ── 2. backend command ──────────────────────────────────
match action.type_.as_str() {
"pip" => {
if !action.packages.is_empty() {
let cmd = format!(
"{} install{}{} {}",
venv_pip(action),
idx,
action.args.iter().fold(String::new(), |a, b| a + " " + b),
action.packages.join(" "),
);
log::info!("Python UPM executing: {}", cmd);
engine
.execute_command_with_delta(&cmd, ctx, event_tx, &delta)
.await
.map_err(DependencyError::CustomFailed)?;
}
if let Some(ref manifest) = action.manifest {
let cmd = format!(
"{} install{} -r '{}'{}",
venv_pip(action),
idx,
manifest.display(),
action.args.iter().fold(String::new(), |a, b| a + " " + b),
);
log::info!("Python UPM executing: {}", cmd);
engine
.execute_command_with_delta(&cmd, ctx, event_tx, &delta)
.await
.map_err(DependencyError::CustomFailed)?;
}
}
"uv" => {
// uv pip install <packages> / uv pip sync <manifest>
if !action.packages.is_empty() {
let cmd = format!(
"uv pip install{}{} {}",
idx,
action.args.iter().fold(String::new(), |a, b| a + " " + b),
action.packages.join(" "),
);
log::info!("Python UPM executing: {}", cmd);
engine
.execute_command_with_delta(&cmd, ctx, event_tx, &delta)
.await
.map_err(DependencyError::CustomFailed)?;
}
if let Some(ref manifest) = action.manifest {
let cmd = format!(
"uv pip sync '{}'{}",
manifest.display(),
action.args.iter().fold(String::new(), |a, b| a + " " + b),
);
log::info!("Python UPM executing: {}", cmd);
engine
.execute_command_with_delta(&cmd, ctx, event_tx, &delta)
.await
.map_err(DependencyError::CustomFailed)?;
}
}
"poetry" => {
let cmd = format!(
"poetry install{}",
action.args.iter().fold(String::new(), |a, b| a + " " + b),
);
log::info!("Python UPM executing: {}", cmd);
engine
.execute_command_with_delta(&cmd, ctx, event_tx, &delta)
.await
.map_err(DependencyError::CustomFailed)?;
}
"pipx" => {
if !action.packages.is_empty() {
let cmd = format!(
"pipx install{}{} {}",
idx,
action.args.iter().fold(String::new(), |a, b| a + " " + b),
action.packages.join(" "),
);
log::info!("Python UPM executing: {}", cmd);
engine
.execute_command_with_delta(&cmd, ctx, event_tx, &delta)
.await
.map_err(DependencyError::CustomFailed)?;
}
}
other => {
log::error!("Unknown Python backend type: {}", other);
return Err(DependencyError::ParseError());
}
}
Ok(())
}
}
// ── Tests ──────────────────────────────────────────────────────────
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_name() {
assert_eq!(Python.name(), "python");
}
#[test]
fn test_system_dependency_basic() {
let yaml = "- pip: {}";
let v: serde_yaml::Value = serde_yaml::from_str(yaml).unwrap();
let deps = Python.system_dependency(&v);
assert_eq!(deps.packages, vec!["python3"]);
}
#[test]
fn test_system_dependency_uv() {
let yaml = r#"
- type: uv
packages: [pytest]
"#;
let v: serde_yaml::Value = serde_yaml::from_str(yaml).unwrap();
let deps = Python.system_dependency(&v);
assert!(deps.packages.contains(&"python3".to_string()));
assert!(deps.packages.contains(&"uv".to_string()));
}
#[test]
fn test_parse_pip_action() {
let yaml = r#"
- type: pip
packages: [pytest, requests]
"#;
let cfg: PythonConfig = serde_yaml::from_str(yaml).unwrap();
assert_eq!(cfg.0.len(), 1);
let a = &cfg.0[0];
assert_eq!(a.type_, "pip");
assert_eq!(a.packages, vec!["pytest", "requests"]);
}
#[test]
fn test_parse_full_action() {
let yaml = r#"
- type: uv
python: "python3.14"
directory: ".venv"
index: "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
packages: [pytest, requests]
manifest: "requirements.txt"
args: ["--upgrade"]
env:
UV_PYTHON: "python3.14"
"#;
let cfg: PythonConfig = serde_yaml::from_str(yaml).unwrap();
let a = &cfg.0[0];
assert_eq!(a.type_, "uv");
assert_eq!(a.python.as_deref(), Some("python3.14"));
assert_eq!(a.directory, Some(PathBuf::from(".venv")));
assert_eq!(
a.index.as_deref(),
Some("https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"),
);
assert_eq!(a.packages, vec!["pytest", "requests"]);
assert_eq!(a.manifest, Some(PathBuf::from("requirements.txt")));
assert_eq!(a.args, vec!["--upgrade"]);
assert_eq!(
a.env_.get("UV_PYTHON").and_then(|v| v.as_deref()),
Some("python3.14"),
);
}
#[test]
fn test_parse_multiple_actions() {
let yaml = r#"
- type: pip
packages: [pytest]
- type: uv
directory: ".venv"
manifest: "requirements.txt"
"#;
let cfg: PythonConfig = serde_yaml::from_str(yaml).unwrap();
assert_eq!(cfg.0.len(), 2);
assert_eq!(cfg.0[0].type_, "pip");
assert_eq!(cfg.0[1].type_, "uv");
}
}
+18 -7
View File
@@ -1,7 +1,7 @@
use async_trait::async_trait;
use serde::Deserialize;
use std::collections::HashMap;
use std::path::PathBuf;
use crate::error::DependencyError;
use crate::types::{DeltaExecutionContext, Engine, EventSender, ExecutionContext};
use crate::upm::UPMSysDeps;
@@ -37,6 +37,16 @@ pub struct Rust;
/// ```
#[derive(Debug, Clone, Deserialize)]
struct RustConfig {
#[allow(dead_code)]
/// UPM backend type (currently only `"rustup"`).
#[serde(default = "default_rust_type", rename = "type")]
type_: String,
/// Environment variables injected into every rustup/cargo child process.
/// Each entry maps to a var name; `None` unsets the variable.
#[serde(default, rename = "env")]
env_: HashMap<String, Option<String>>,
/// Rust toolchain to activate via `rustup default`.
/// If unset, the existing default toolchain is left alone.
#[serde(default)]
@@ -57,6 +67,8 @@ struct RustConfig {
cargo: CargoSubConfig,
}
fn default_rust_type() -> String { "rustup".to_string() }
// ── Cargo sub-config ───────────────────────────────────────────────
/// Nested cargo configuration inside `rust`.
@@ -190,13 +202,12 @@ impl UserPackageManager for Rust {
// ── Execute ──────────────────────────────────────────────
for cmd in &commands {
log::info!("Rust UPM executing: {}", cmd);
let delta = DeltaExecutionContext {
env_vars: cfg.env_.clone(),
..Default::default()
};
let result = engine
.execute_command_with_delta(
cmd,
ctx,
event_tx,
&DeltaExecutionContext::default(),
)
.execute_command_with_delta(cmd, ctx, event_tx, &delta)
.await;
if let Err(e) = result {
log::error!("Rust command failed: {}", e);