feat: rename to workshop-baker and implement stage-based prebake
workflow - Rename crate/library/binary from workshop-executor to workshop-baker - Remove unused modules (decorator, parser, schedule, variable) from lib - Implement stage-based privilege dropping with prebake_drop_privilege() - Add dry-run support in executor and CLI - Add bootstrap.sh template script for user/workspace setup - Add PrebakeStage::from_str() and Default impl - Add get_drop_after() function for security config - Fix version matching to handle "1.0" format variants - Fix UPM all_managers() to return custom::Custom - Add Python integration tests with Docker - Simplify examples/prebake.yml to match template
This commit is contained in:
Catty Steve
+22
-21
@@ -88,7 +88,7 @@ dependencies:
|
||||
name: "arch4edu"
|
||||
keypackage: "arch4edu-keyring"
|
||||
mirror: "https://mirrors.tuna.tsinghua.edu.cn" # 指定软件包镜像源，可选
|
||||
dnf:
|
||||
#dnf:
|
||||
# 略
|
||||
|
||||
# 用户依赖
|
||||
@@ -96,27 +96,27 @@ dependencies:
|
||||
# 例外：语言依赖会在system中产生隐式依赖
|
||||
user:
|
||||
# 以下字段按serde Value原样传递给插件，config.rs不做解析
|
||||
rust:
|
||||
type: "rustup" # 此时会向系统引入隐式的rustup依赖
|
||||
toolchain: "nightly-x86_64-unknown-linux-gnu"
|
||||
manifest: "Cargo.toml"
|
||||
lock: "Cargo.lock"
|
||||
python:
|
||||
type: "uv"
|
||||
python: "python3.14"
|
||||
index: "http://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
|
||||
directory: ".venv"
|
||||
env: # 优先，高级选项
|
||||
UV_PYTHON: "python3.14"
|
||||
UV_INDEX: "http://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
|
||||
UV_DIRECTORY: ".venv"
|
||||
manifest: "requirements.txt" # 或pyproject.toml
|
||||
nix: {}
|
||||
# TBD: 没用过lol
|
||||
# PR welcome
|
||||
# rust:
|
||||
# type: "rustup" # 此时会向系统引入隐式的rustup依赖
|
||||
# toolchain: "nightly-x86_64-unknown-linux-gnu"
|
||||
# manifest: "Cargo.toml"
|
||||
# lock: "Cargo.lock"
|
||||
# python:
|
||||
# type: "uv"
|
||||
# python: "python3.14"
|
||||
# index: "http://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
|
||||
# directory: ".venv"
|
||||
# env: # 优先，高级选项
|
||||
# UV_PYTHON: "python3.14"
|
||||
# UV_INDEX: "http://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
|
||||
# UV_DIRECTORY: ".venv"
|
||||
# manifest: "requirements.txt" # 或pyproject.toml
|
||||
# nix: {}
|
||||
# # TBD: 没用过lol
|
||||
# # PR welcome
|
||||
custom: # 绝对的自定义，与上述全部配置互斥（custom独占）
|
||||
- command: "cargo fetch"
|
||||
working_dir: "/app"
|
||||
working_dir: "/tmp"
|
||||
|
||||
# 环境变量配置
|
||||
envvars:
|
||||
@@ -180,11 +180,12 @@ hooks:
|
||||
# 依赖安装前
|
||||
early:
|
||||
- command: "echo 'Starting dependency installation'"
|
||||
name: ""
|
||||
|
||||
# 依赖安装后
|
||||
late:
|
||||
- command: "cargo fetch"
|
||||
working_dir: "/app"
|
||||
working_dir: "/tmp"
|
||||
|
||||
# 元数据
|
||||
metadata:
|
||||
|
||||
Generated
+1
-1
@@ -3382,7 +3382,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59"
|
||||
|
||||
[[package]]
|
||||
name = "workshop-executor"
|
||||
name = "workshop-baker"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
[package]
|
||||
name = "workshop-executor"
|
||||
name = "workshop-baker"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
|
||||
[lib]
|
||||
name = "workshop_executor"
|
||||
name = "workshop_baker"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "workshop-executor"
|
||||
name = "workshop-baker"
|
||||
path = "src/main.rs"
|
||||
|
||||
[features]
|
||||
|
||||
+156
-129
@@ -1,52 +1,81 @@
|
||||
# prebake.yml - 蜜饼工坊 CI/CD 示例配置
|
||||
# 核心特性：使用 Docker pull 形态 image，适配 Rust + CUDA 机器学习场景
|
||||
version: "1.0.0"
|
||||
# prebake.yaml.tmpl
|
||||
#
|
||||
# 术语说明：
|
||||
# 可选：该字段可以不给出
|
||||
# 默认为...：隐含“可选”
|
||||
# 留空：尚未定义，留作后续
|
||||
|
||||
# 构建环境定义（核心：使用 Docker pull 镜像，不指定 Dockerfile）
|
||||
# 版本号，用于格式兼容性检查
|
||||
version: "1.0"
|
||||
|
||||
# 构建环境定义
|
||||
environment:
|
||||
# 构建机类型：docker（使用容器构建）
|
||||
# 构建机类型：docker | firecracker | custom | baremetal
|
||||
builder: "docker"
|
||||
|
||||
# Docker 构建机配置（仅指定 image，不指定 dockerfile）
|
||||
config:
|
||||
# docker:
|
||||
# 核心：Docker pull 形态的镜像（NVIDIA CUDA + Rust 基础镜像）
|
||||
image: "docker.1ms.run/library/ubuntu:latest"
|
||||
# 注意：dockerfile 字段未指定（与 image 二选一），build_args 也无需配置
|
||||
# 构建机配置，与builder一致的被启用。
|
||||
baremetal: {}
|
||||
|
||||
# 硬件资源要求（适配 CUDA 机器学习场景）
|
||||
docker:
|
||||
# 镜像名，与dockerfile互斥
|
||||
image: "rust:1.70-slim"
|
||||
|
||||
# Dockerfile路径，相对于项目根目录，与image互斥
|
||||
dockerfile: "/Dockerfile"
|
||||
# Docker构建参数，可选
|
||||
build_args:
|
||||
RUST_VERSION: "1.70"
|
||||
|
||||
firecracker: {}
|
||||
# TODO: 定义firecracker结构
|
||||
|
||||
custom:
|
||||
# 自定义构建机名称
|
||||
name: "my-custom-builder"
|
||||
# 设置脚本，相对于项目根目录
|
||||
setup_script: "setup-builder.sh"
|
||||
# 清洁脚本，相对于项目根目录
|
||||
cleanup_script: "cleanup-builder.sh"
|
||||
|
||||
# 硬件资源要求，满足最小要求的构建机可以被选中
|
||||
resources:
|
||||
cpu: 4 # 机器学习编译需要更多CPU核心
|
||||
memory: "16G" # 大内存支持CUDA编译
|
||||
disk: "50G" # 足够的磁盘空间存放CUDA依赖
|
||||
architecture:
|
||||
- "x86_64" # 优先x86_64（CUDA主力架构）
|
||||
- "aarch64" # 兼容ARM64（Jetson设备）
|
||||
gpu: true # 启用GPU（机器学习场景必需）
|
||||
gpu_type:
|
||||
- "cuda" # 支持CUDA
|
||||
- "rocm" # 兼容ROCm
|
||||
tags: # 仅选择带GPU标签的构建机
|
||||
- "gpu-enabled"
|
||||
- "cuda-12.2"
|
||||
cpu: 2 # CPU核心数，默认为1
|
||||
memory: "2G" # 内存大小，也可以写作2048MB,默认为"1G"
|
||||
disk: "10G" # 磁盘空间，也可以写作10240MB,默认为"1G"
|
||||
architecture: "x86_64" # 架构要求，只能是"noarch"或{"x86_64"/"amd64", "arm64"/"aarch64", "riscv64", "loongarch64"/"loong64"}中的一个或几个
|
||||
# TODO: 重新设计GPU字段
|
||||
# gpu: false # 是否需要GPU，默认为false
|
||||
# gpu_type: "cuda" # GPU类型（如果需要），可以是"cuda", "rocm", "vulkan", "oneapi"中的一个或几个
|
||||
tags: # 自定义标签，只有满足标签的构建机可以被选中
|
||||
- "custom_tag"
|
||||
|
||||
# 依赖定义（适配 Rust + CUDA 开发环境）
|
||||
# 网络配置
|
||||
network:
|
||||
enabled: true # 启用网络，对baremetal不适用，默认为true
|
||||
outbound: true # 允许出站连接，对baremetal不适用，默认为true
|
||||
dns_servers: # 指定构建机的DNS服务器，对baremetal不适用，custom应该自行处理DNS问题，可选
|
||||
- "8.8.8.8"
|
||||
- "1.1.1.1"
|
||||
proxies: # 代理配置，暂时留空，可选
|
||||
|
||||
# 依赖定义，安装依赖时按system-user顺序进行
|
||||
dependencies:
|
||||
# 系统包依赖（按发行版分类）
|
||||
# 系统包依赖，按包管理器枚举给出，只应该给出与环境契合的字段。
|
||||
# 根据security.drop-after的配置，在此阶段一般具有root或免密root权限
|
||||
# 不适用于裸机
|
||||
system:
|
||||
deb:
|
||||
packages:
|
||||
apt:
|
||||
packages: # 安装的软件包
|
||||
- "git"
|
||||
- "curl"
|
||||
- "build-essential"
|
||||
- "pkg-config"
|
||||
- "libssl-dev"
|
||||
- "nvidia-cuda-toolkit" # CUDA工具链
|
||||
- "libcudnn8" # cuDNN库（深度学习必需）
|
||||
- "python3-pip" # Python依赖（ML场景）
|
||||
repositories:
|
||||
- "ppa:deadsnakes/ppa" # Python版本仓库
|
||||
mirror: "https://mirrors.aliyun.com" # 阿里云镜像（国内加速）
|
||||
repositories: # 该字段按serde Value原样传递给插件，config.rs不做解析
|
||||
- url: "ppa:custom/ppa" # 自定义PPA（Ubuntu）
|
||||
- url: "https://download.docker.com/linux/ubuntu" # DEB822
|
||||
key: "https://download.docker.com/linux/ubuntu/gpg"
|
||||
mirror: "https://mirrors.tuna.tsinghua.edu.cn" # 指定软件包镜像源，可选
|
||||
pacman:
|
||||
packages:
|
||||
- "git"
|
||||
@@ -54,113 +83,111 @@ dependencies:
|
||||
- "base-devel"
|
||||
- "pkg-config"
|
||||
- "openssl"
|
||||
- "cuda" # Arch Linux CUDA包
|
||||
- "cudnn"
|
||||
- "python-pip"
|
||||
repositories:
|
||||
- "https://mirrors.tuna.tsinghua.edu.cn/arch4edu/$arch"
|
||||
mirror: "https://mirrors.tuna.tsinghua.edu.cn"
|
||||
- server: "https://mirrors.tuna.tsinghua.edu.cn/arch4edu/$arch" # 自定义仓库（Arch）
|
||||
name: "arch4edu"
|
||||
keypackage: "arch4edu-keyring"
|
||||
mirror: "https://mirrors.tuna.tsinghua.edu.cn" # 指定软件包镜像源，可选
|
||||
#dnf:
|
||||
# 略
|
||||
|
||||
# 语言特定依赖（Rust + Python）
|
||||
languages:
|
||||
rust:
|
||||
toolchain: "stable-x86_64-unknown-linux-gnu"
|
||||
components: # Rust 额外组件
|
||||
- "clippy"
|
||||
- "rustfmt"
|
||||
targets: # 交叉编译目标
|
||||
- "aarch64-unknown-linux-gnu"
|
||||
# 用户依赖
|
||||
# 根据security.drop-after的配置，在此阶段一般不具有root权限
|
||||
# 例外：语言依赖会在system中产生隐式依赖
|
||||
user:
|
||||
# 以下字段按serde Value原样传递给插件，config.rs不做解析
|
||||
# rust:
|
||||
# type: "rustup" # 此时会向系统引入隐式的rustup依赖
|
||||
# toolchain: "nightly-x86_64-unknown-linux-gnu"
|
||||
# manifest: "Cargo.toml"
|
||||
# lock: "Cargo.lock"
|
||||
# python:
|
||||
# version: "3.10"
|
||||
# packages: # Python ML依赖
|
||||
# - "torch==2.2.0"
|
||||
# - "torchvision==0.17.0"
|
||||
# - "numpy==1.26.4"
|
||||
# type: "uv"
|
||||
# python: "python3.14"
|
||||
# index: "http://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
|
||||
# directory: ".venv"
|
||||
# env: # 优先，高级选项
|
||||
# UV_PYTHON: "python3.14"
|
||||
# UV_INDEX: "http://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
|
||||
# UV_DIRECTORY: ".venv"
|
||||
# manifest: "requirements.txt" # 或pyproject.toml
|
||||
# nix: {}
|
||||
# # TBD: 没用过lol
|
||||
# # PR welcome
|
||||
custom: # 绝对的自定义，与上述全部配置互斥（custom独占）
|
||||
- command: "cargo fetch"
|
||||
working_dir: "/tmp"
|
||||
|
||||
# 自定义依赖（优先执行）
|
||||
custom_pre:
|
||||
- name: "configure-cuda-env"
|
||||
command: "echo 'export PATH=/usr/local/cuda/bin:$PATH' >> /etc/profile && ldconfig"
|
||||
environment:
|
||||
CUDA_HOME: "/usr/local/cuda"
|
||||
|
||||
# 自定义依赖（常规执行）
|
||||
custom:
|
||||
- name: "install-rust-cuda-bindings"
|
||||
command: "cargo install cargo-cuda"
|
||||
working_dir: "/app"
|
||||
timeout: 600 # 延长超时（编译CUDA绑定耗时）
|
||||
- name: "setup-ml-deps"
|
||||
command: "pip3 install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple"
|
||||
working_dir: "/app/ml"
|
||||
environment:
|
||||
PIP_NO_CACHE_DIR: "1"
|
||||
|
||||
# 环境变量配置（分阶段）
|
||||
environment_variables:
|
||||
# prebake阶段专属环境变量
|
||||
# 环境变量配置
|
||||
envvars:
|
||||
# 构建环境变量，只在 prebake 阶段有效
|
||||
# 用于控制 prebake 阶段的行为（apt/cargo/rustc 等）
|
||||
prebake:
|
||||
RUST_BACKTRACE: "full"
|
||||
CARGO_NET_GIT_FETCH_WITH_CLI: "true"
|
||||
CUDA_VERBOSE: "1" # CUDA编译详细日志
|
||||
RUSTFLAGS: "-C target-cpu=native" # 优化CPU编译
|
||||
RUST_BACKTRACE: "full" # Rust 调试
|
||||
CARGO_NET_GIT_FETCH_WITH_CLI: "true" # Cargo 配置
|
||||
NODE_ENV: "production" # Node 环境
|
||||
|
||||
# bake阶段传递的环境变量（合并到.env）
|
||||
# 运行时环境变量（会传递给 bake 阶段）
|
||||
# 这些变量会与项目中的 .env 文件合并后，作为最终环境
|
||||
bake:
|
||||
DATABASE_URL: "sqlite:///app/data/honeybiscuit.db" # 优先SQLite（符合轻量化要求）
|
||||
LOG_LEVEL: "debug"
|
||||
CUDA_DEVICE_ORDER: "PCI_BUS_ID"
|
||||
CUDA_VISIBLE_DEVICES: "0" # 指定使用第0块GPU
|
||||
__ENV_FILE: ".env.prod" # 合并项目中的.env.prod文件
|
||||
# 支持 secret 引用，格式: {{ secret.<key> }}
|
||||
DATABASE_URL: "postgresql://user:{{ secret.dbpassword }}@localhost/db"
|
||||
LOG_LEVEL: "info"
|
||||
|
||||
# 缓存配置（优化构建速度）
|
||||
# 特殊变量：指定项目中 .env 文件的位置
|
||||
# 系统会读取该文件，并将其中的变量与上面定义的变量合并
|
||||
# 合并规则：prebake.yml 中定义的变量优先级更高，会覆盖 .env 中的同名变量
|
||||
# 此变量本身不会传递给 bake 阶段
|
||||
# .env不应该包含任何 secret ！
|
||||
__ENV_FILE: ".env"
|
||||
|
||||
# 注意：
|
||||
# - secret 只存在于内存中，不会写入磁盘
|
||||
# - 日志中会自动遮蔽 secret 的值
|
||||
# - 如需临时查看 secret，可使用 PIPELINE_UNCOVER_SECRET 权限（需 2FA）
|
||||
|
||||
|
||||
# 缓存配置
|
||||
cache:
|
||||
directories:
|
||||
# Rust Cargo缓存（长期保留）
|
||||
- path: "/usr/local/cargo/registry"
|
||||
strategy: "always" # 始终缓存
|
||||
mode: "zstd" # zstd压缩（比gzip高效）
|
||||
# Rust 编译目标缓存
|
||||
- path: "/app/target"
|
||||
strategy: "always"
|
||||
mode: "dir" # 直接目录缓存（不压缩，加速访问）
|
||||
# Python pip缓存
|
||||
- path: "/root/.cache/pip"
|
||||
strategy: "clean" # 每次构建前清理
|
||||
mode: "gzip"
|
||||
# CUDA编译缓存（大文件，不压缩）
|
||||
- path: "/usr/local/cuda/bin/cache"
|
||||
strategy: "always"
|
||||
# 缓存目录
|
||||
directory:
|
||||
- path: "/usr/local/cargo"
|
||||
strategy: "always" # 在"always","readonly","clean"和"never"中选择一个，可以被流水线启动时配置覆盖，默认为"always"
|
||||
mode: "zstd" # 在"gzip","zstd","none"和"dir"中选择一个
|
||||
|
||||
- path: "/root/.npm"
|
||||
strategy: "clean"
|
||||
mode: "none"
|
||||
|
||||
# 缓存全局策略
|
||||
- path: "/app/target"
|
||||
strategy: "always"
|
||||
mode: "dir"
|
||||
|
||||
# 缓存策略
|
||||
strategy:
|
||||
ttl_days: 15 # 缓存保留15天（ML依赖更新快）
|
||||
max_size_gb: 100 # 更大缓存空间（CUDA依赖体积大）
|
||||
cleanup_policy: "size" # 按大小清理（优先删除大文件）
|
||||
ttl_days: 30 # 缓存生存时间
|
||||
max_size_gb: 20 # 最大缓存大小
|
||||
cleanup_policy: "lru" # 清理策略：lru, fifo, size
|
||||
|
||||
# 钩子脚本（生命周期管理）
|
||||
hooks:
|
||||
pre_dependencies:
|
||||
- command: "echo '=== 蜜饼工坊 - 开始安装依赖 ===' && apt update"
|
||||
post_dependencies:
|
||||
- command: "cargo cache clean --git --registry" # 清理无用Cargo缓存
|
||||
- command: "nvcc --version && rustc --version" # 验证环境
|
||||
ready:
|
||||
- command: "echo '=== 蜜饼工坊 - 环境准备完成，CUDA可用 ==='"
|
||||
- command: "python3 -c 'import torch; print(f\"CUDA可用: {torch.cuda.is_available()}\")'"
|
||||
|
||||
# 安全配置（基础）
|
||||
# 安全配置
|
||||
security:
|
||||
seccomp_profile: "default" # 默认seccomp配置
|
||||
capabilities: # 容器权限（GPU需要）
|
||||
- "CAP_SYS_ADMIN"
|
||||
- "CAP_NET_ADMIN"
|
||||
read_only_fs: false # 非只读文件系统（需要写入编译产物）
|
||||
#
|
||||
drop-after: ""
|
||||
# 留空
|
||||
|
||||
# 元数据（蜜饼工坊专属标识）
|
||||
# 钩子脚本
|
||||
hooks:
|
||||
# 依赖安装前
|
||||
early:
|
||||
- command: "echo 'Starting dependency installation'"
|
||||
name: ""
|
||||
|
||||
# 依赖安装后
|
||||
late:
|
||||
- command: "cargo fetch"
|
||||
working_dir: "/tmp"
|
||||
|
||||
# 元数据
|
||||
metadata:
|
||||
description: "蜜饼工坊 - Rust + CUDA 机器学习项目构建环境"
|
||||
maintainer: "honeybiscuit@arknights.com"
|
||||
tags: ["rust", "cuda", "machine-learning", "ml", "nvidia"]
|
||||
project_type: "rust-cuda-ml"
|
||||
description: "Rust project build environment"
|
||||
maintainer: "team@example.com"
|
||||
|
||||
@@ -1,166 +0,0 @@
|
||||
# prebake.yml - 蜜饼工坊 CI/CD 示例配置
|
||||
# 核心特性：使用 Docker pull 形态 image，适配 Rust + CUDA 机器学习场景
|
||||
version: "1.0.0"
|
||||
|
||||
# 构建环境定义（核心：使用 Docker pull 镜像，不指定 Dockerfile）
|
||||
environment:
|
||||
# 构建机类型：docker（使用容器构建）
|
||||
builder: "docker"
|
||||
|
||||
# Docker 构建机配置（仅指定 image，不指定 dockerfile）
|
||||
config:
|
||||
# docker:
|
||||
# 核心：Docker pull 形态的镜像（NVIDIA CUDA + Rust 基础镜像）
|
||||
dockerfile: "examples/Dockerfile_1"
|
||||
# 注意：dockerfile 字段未指定（与 image 二选一），build_args 也无需配置
|
||||
|
||||
# 硬件资源要求（适配 CUDA 机器学习场景）
|
||||
resources:
|
||||
cpu: 4 # 机器学习编译需要更多CPU核心
|
||||
memory: "16G" # 大内存支持CUDA编译
|
||||
disk: "50G" # 足够的磁盘空间存放CUDA依赖
|
||||
architecture:
|
||||
- "x86_64" # 优先x86_64（CUDA主力架构）
|
||||
- "aarch64" # 兼容ARM64（Jetson设备）
|
||||
gpu: true # 启用GPU（机器学习场景必需）
|
||||
gpu_type:
|
||||
- "cuda" # 支持CUDA
|
||||
- "rocm" # 兼容ROCm
|
||||
tags: # 仅选择带GPU标签的构建机
|
||||
- "gpu-enabled"
|
||||
- "cuda-12.2"
|
||||
|
||||
# 依赖定义（适配 Rust + CUDA 开发环境）
|
||||
dependencies:
|
||||
# 系统包依赖（按发行版分类）
|
||||
system:
|
||||
deb:
|
||||
packages:
|
||||
- "git"
|
||||
- "curl"
|
||||
- "build-essential"
|
||||
- "pkg-config"
|
||||
- "libssl-dev"
|
||||
- "nvidia-cuda-toolkit" # CUDA工具链
|
||||
- "libcudnn8" # cuDNN库（深度学习必需）
|
||||
- "python3-pip" # Python依赖（ML场景）
|
||||
repositories:
|
||||
- "ppa:deadsnakes/ppa" # Python版本仓库
|
||||
mirror: "https://mirrors.aliyun.com" # 阿里云镜像（国内加速）
|
||||
pacman:
|
||||
packages:
|
||||
- "git"
|
||||
- "curl"
|
||||
- "base-devel"
|
||||
- "pkg-config"
|
||||
- "openssl"
|
||||
- "cuda" # Arch Linux CUDA包
|
||||
- "cudnn"
|
||||
- "python-pip"
|
||||
repositories:
|
||||
- "https://mirrors.tuna.tsinghua.edu.cn/arch4edu/$arch"
|
||||
mirror: "https://mirrors.tuna.tsinghua.edu.cn"
|
||||
|
||||
# 语言特定依赖（Rust + Python）
|
||||
languages:
|
||||
rust:
|
||||
toolchain: "stable-x86_64-unknown-linux-gnu"
|
||||
components: # Rust 额外组件
|
||||
- "clippy"
|
||||
- "rustfmt"
|
||||
targets: # 交叉编译目标
|
||||
- "aarch64-unknown-linux-gnu"
|
||||
# python:
|
||||
# version: "3.10"
|
||||
# packages: # Python ML依赖
|
||||
# - "torch==2.2.0"
|
||||
# - "torchvision==0.17.0"
|
||||
# - "numpy==1.26.4"
|
||||
|
||||
# 自定义依赖（优先执行）
|
||||
custom_pre:
|
||||
- name: "configure-cuda-env"
|
||||
command: "echo 'export PATH=/usr/local/cuda/bin:$PATH' >> /etc/profile && ldconfig"
|
||||
environment:
|
||||
CUDA_HOME: "/usr/local/cuda"
|
||||
|
||||
# 自定义依赖（常规执行）
|
||||
custom:
|
||||
- name: "install-rust-cuda-bindings"
|
||||
command: "cargo install cargo-cuda"
|
||||
working_dir: "/app"
|
||||
timeout: 600 # 延长超时（编译CUDA绑定耗时）
|
||||
- name: "setup-ml-deps"
|
||||
command: "pip3 install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple"
|
||||
working_dir: "/app/ml"
|
||||
environment:
|
||||
PIP_NO_CACHE_DIR: "1"
|
||||
|
||||
# 环境变量配置（分阶段）
|
||||
environment_variables:
|
||||
# prebake阶段专属环境变量
|
||||
prebake:
|
||||
RUST_BACKTRACE: "full"
|
||||
CARGO_NET_GIT_FETCH_WITH_CLI: "true"
|
||||
CUDA_VERBOSE: "1" # CUDA编译详细日志
|
||||
RUSTFLAGS: "-C target-cpu=native" # 优化CPU编译
|
||||
|
||||
# bake阶段传递的环境变量（合并到.env）
|
||||
bake:
|
||||
DATABASE_URL: "sqlite:///app/data/honeybiscuit.db" # 优先SQLite（符合轻量化要求）
|
||||
LOG_LEVEL: "debug"
|
||||
CUDA_DEVICE_ORDER: "PCI_BUS_ID"
|
||||
CUDA_VISIBLE_DEVICES: "0" # 指定使用第0块GPU
|
||||
__ENV_FILE: ".env.prod" # 合并项目中的.env.prod文件
|
||||
|
||||
# 缓存配置（优化构建速度）
|
||||
cache:
|
||||
directories:
|
||||
# Rust Cargo缓存（长期保留）
|
||||
- path: "/usr/local/cargo/registry"
|
||||
strategy: "always" # 始终缓存
|
||||
mode: "zstd" # zstd压缩（比gzip高效）
|
||||
# Rust 编译目标缓存
|
||||
- path: "/app/target"
|
||||
strategy: "always"
|
||||
mode: "dir" # 直接目录缓存（不压缩，加速访问）
|
||||
# Python pip缓存
|
||||
- path: "/root/.cache/pip"
|
||||
strategy: "clean" # 每次构建前清理
|
||||
mode: "gzip"
|
||||
# CUDA编译缓存（大文件，不压缩）
|
||||
- path: "/usr/local/cuda/bin/cache"
|
||||
strategy: "always"
|
||||
mode: "none"
|
||||
|
||||
# 缓存全局策略
|
||||
strategy:
|
||||
ttl_days: 15 # 缓存保留15天（ML依赖更新快）
|
||||
max_size_gb: 100 # 更大缓存空间（CUDA依赖体积大）
|
||||
cleanup_policy: "size" # 按大小清理（优先删除大文件）
|
||||
|
||||
# 钩子脚本（生命周期管理）
|
||||
hooks:
|
||||
pre_dependencies:
|
||||
- command: "echo '=== 蜜饼工坊 - 开始安装依赖 ===' && apt update"
|
||||
post_dependencies:
|
||||
- command: "cargo cache clean --git --registry" # 清理无用Cargo缓存
|
||||
- command: "nvcc --version && rustc --version" # 验证环境
|
||||
ready:
|
||||
- command: "echo '=== 蜜饼工坊 - 环境准备完成，CUDA可用 ==='"
|
||||
- command: "python3 -c 'import torch; print(f\"CUDA可用: {torch.cuda.is_available()}\")'"
|
||||
|
||||
# 安全配置（基础）
|
||||
security:
|
||||
seccomp_profile: "default" # 默认seccomp配置
|
||||
capabilities: # 容器权限（GPU需要）
|
||||
- "CAP_SYS_ADMIN"
|
||||
- "CAP_NET_ADMIN"
|
||||
read_only_fs: false # 非只读文件系统（需要写入编译产物）
|
||||
|
||||
# 元数据（蜜饼工坊专属标识）
|
||||
metadata:
|
||||
description: "蜜饼工坊 - Rust + CUDA 机器学习项目构建环境"
|
||||
maintainer: "honeybiscuit@arknights.com"
|
||||
tags: ["rust", "cuda", "machine-learning", "ml", "nvidia"]
|
||||
project_type: "rust-cuda-ml"
|
||||
@@ -0,0 +1,11 @@
|
||||
[pytest]
|
||||
testpaths = tests/integration
|
||||
python_files = test_*.py
|
||||
python_classes = Test*
|
||||
python_functions = test_*
|
||||
addopts = -v --tb=short
|
||||
markers =
|
||||
integration: marks tests as integration tests (deselect with '-m "not integration"')
|
||||
slow: marks tests as slow running
|
||||
filterwarnings =
|
||||
ignore::DeprecationWarning
|
||||
@@ -142,6 +142,13 @@ impl super::types::Engine {
|
||||
|
||||
self.construct_command(&mut command, ctx);
|
||||
|
||||
if ctx.dry_run {
|
||||
log::info!("[DRY_RUN] {:?}", command);
|
||||
return Ok(ExecutionResult {
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
|
||||
let child = command.spawn().map_err(|e| {
|
||||
ExecutionError::InvalidCommand(format!("Failed to spawn command: {}", e))
|
||||
})?;
|
||||
@@ -345,6 +352,8 @@ mod tests {
|
||||
timeout: Some(Duration::from_secs(30)),
|
||||
cgroup_path: None,
|
||||
shell: "bash".to_string(),
|
||||
dry_run: false,
|
||||
privileged: false,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -18,7 +18,8 @@ impl PackageManager for Apt {
|
||||
}
|
||||
|
||||
fn adopt(&self, distro: &Info) -> bool {
|
||||
todo!();
|
||||
return false;
|
||||
// todo!();
|
||||
}
|
||||
|
||||
fn binary(&self) -> &'static str {
|
||||
|
||||
@@ -78,7 +78,7 @@ impl PackageManager for Pacman {
|
||||
let d = distro.os_type().to_string().to_lowercase();
|
||||
matches!(
|
||||
d.as_str(),
|
||||
"arch"
|
||||
"arch linux"
|
||||
| "manjaro"
|
||||
| "endeavouros"
|
||||
| "garuda"
|
||||
|
||||
@@ -25,6 +25,8 @@ pub struct ExecutionContext {
|
||||
pub timeout: Option<Duration>,
|
||||
pub cgroup_path: Option<PathBuf>,
|
||||
pub shell: String,
|
||||
pub dry_run: bool,
|
||||
pub privileged: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -45,6 +47,8 @@ impl Default for ExecutionContext {
|
||||
timeout: None,
|
||||
cgroup_path: None,
|
||||
shell: String::from("bash"),
|
||||
dry_run: false,
|
||||
privileged: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -68,6 +72,20 @@ pub struct ExecutionResult {
|
||||
pub resource_usage: Option<ResourceUsage>,
|
||||
}
|
||||
|
||||
impl Default for ExecutionResult {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
task_id: String::new(),
|
||||
exit_code: 0,
|
||||
success: true,
|
||||
duration: Duration::default(),
|
||||
stdout: String::new(),
|
||||
stderr: String::new(),
|
||||
resource_usage: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub enum StreamType {
|
||||
Stdout,
|
||||
|
||||
@@ -31,7 +31,9 @@ pub trait UserPackageManager {
|
||||
}
|
||||
|
||||
pub fn all_managers() -> Vec<Box<dyn UserPackageManager>> {
|
||||
vec![]
|
||||
vec![
|
||||
Box::new(custom::Custom),
|
||||
]
|
||||
}
|
||||
|
||||
pub fn select(manager: &str) -> Option<Box<dyn UserPackageManager>> {
|
||||
|
||||
@@ -57,6 +57,9 @@ pub enum PrebakeError {
|
||||
|
||||
#[error("YAML parse error: {0}")]
|
||||
YamlParseError(#[from] serde_yaml::Error),
|
||||
|
||||
#[error("Bad prebake.yml: {0}")]
|
||||
ConfigError(String),
|
||||
}
|
||||
|
||||
impl PrebakeError {
|
||||
@@ -69,6 +72,7 @@ impl PrebakeError {
|
||||
PrebakeError::InvalidStage { .. } => EXITCODE_PRIV_DROP_FAILED,
|
||||
PrebakeError::IoError(_) => EXITCODE_IO_ERROR,
|
||||
PrebakeError::YamlParseError(_) => EXITCODE_PARSE_ERROR,
|
||||
PrebakeError::ConfigError(_) => EXITCODE_PARSE_ERROR,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,15 +2,11 @@
|
||||
// Selective exports: only modules used by external callers or tests
|
||||
|
||||
pub mod daemon;
|
||||
pub mod decorator;
|
||||
pub mod engine;
|
||||
pub mod error;
|
||||
pub mod monitor;
|
||||
pub mod parser;
|
||||
pub mod prebake;
|
||||
pub mod schedule;
|
||||
pub mod socket;
|
||||
pub mod variable;
|
||||
pub mod types;
|
||||
|
||||
// External crates re-exports
|
||||
@@ -59,6 +55,8 @@ pub mod cli {
|
||||
Monitor { group: String },
|
||||
/// Prepare a task
|
||||
Prebake { config: String },
|
||||
/// Run in client mode
|
||||
Client { config: String },
|
||||
/// Run a task
|
||||
Bake {
|
||||
/// Path to the script to execute
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use env_logger;
|
||||
use workshop_executor::{
|
||||
use workshop_baker::{
|
||||
cli::{Cli, Commands, Parser},
|
||||
config, daemon,
|
||||
monitor::{self, monitor},
|
||||
@@ -10,25 +10,32 @@ use workshop_executor::{
|
||||
async fn main() {
|
||||
env_logger::init();
|
||||
let cli = Cli::parse();
|
||||
if cli.dry_run {
|
||||
log::warn!("Dry run enabled, no changes will be made.");
|
||||
}
|
||||
match &cli.command {
|
||||
Some(Commands::Monitor { group: _ }) => {
|
||||
unimplemented!("executor-monitor should be moved to daemon instead.")
|
||||
// log::info!("Starting monitor for group: {}", group);
|
||||
// let result = monitor(&group, Some(1000), settings).await;
|
||||
// result.unwrap();
|
||||
}
|
||||
Some(Commands::Client { config }) => {}
|
||||
Some(Commands::Prebake { config }) => {
|
||||
let result = prebake(config, &cli).await;
|
||||
result.unwrap();
|
||||
log::info!("Running in standalone mode.");
|
||||
let result = prebake(config, &cli, None).await;
|
||||
dbg!(&result);
|
||||
if result.is_err() {
|
||||
log::error!("Prebake stage failed! Exiting...");
|
||||
std::process::exit(1);
|
||||
}
|
||||
// result.unwrap();
|
||||
}
|
||||
Some(Commands::Bake {
|
||||
script: _,
|
||||
bake_base: _,
|
||||
}) => {
|
||||
// builder::bake(script, bake_base, username, pipeline, *dry_run);
|
||||
unimplemented!("bake not implemented.")
|
||||
}
|
||||
Some(Commands::Finalize { config: _ }) => {
|
||||
// builder::finalize(config);
|
||||
unimplemented!("finalize not implemented.")
|
||||
}
|
||||
None => {
|
||||
// Running in daemon mode
|
||||
|
||||
@@ -3,7 +3,11 @@ pub mod env;
|
||||
pub mod security;
|
||||
pub mod stage;
|
||||
|
||||
use crate::{cli::Cli, error::PrebakeError};
|
||||
use crate::{
|
||||
cli::Cli,
|
||||
error::PrebakeError,
|
||||
prebake::{security::get_drop_after, stage::PrebakeStage},
|
||||
};
|
||||
|
||||
pub use config::PrebakeConfig;
|
||||
pub use security::prebake_drop_privilege;
|
||||
@@ -16,9 +20,17 @@ pub fn serialize_prebake_config(config: &PrebakeConfig) -> Result<String, serde_
|
||||
serde_yaml::to_string(config)
|
||||
}
|
||||
|
||||
pub async fn prebake(config_path: &str, cli: &Cli) -> anyhow::Result<()> {
|
||||
pub async fn prebake(
|
||||
config_path: &str,
|
||||
cli: &Cli,
|
||||
stage: Option<PrebakeStage>,
|
||||
) -> anyhow::Result<()> {
|
||||
// Initialize
|
||||
let config_content = std::fs::read_to_string(config_path)?;
|
||||
let config = parse_prebake_config(config_content.as_str())?;
|
||||
let config = parse_prebake_config(config_content.as_str()).map_err(|e| {
|
||||
log::error!("Failed to parse prebake.yml: {}", e);
|
||||
PrebakeError::YamlParseError(e)
|
||||
})?;
|
||||
let result = config.validate();
|
||||
if let Err(errors) = result {
|
||||
for error in errors {
|
||||
@@ -26,55 +38,85 @@ pub async fn prebake(config_path: &str, cli: &Cli) -> anyhow::Result<()> {
|
||||
}
|
||||
anyhow::bail!(PrebakeError::ValidateError());
|
||||
}
|
||||
|
||||
let dropafter = get_drop_after(&config.security).map_err(|e| {
|
||||
log::error!("Failed to parse drop-after: {}", e);
|
||||
PrebakeError::ConfigError(e)
|
||||
})?;
|
||||
let target_user = "vulcan";
|
||||
let mut ctx = crate::ExecutionContext {
|
||||
task_id: format!("{}-{}-init", cli.pipeline, cli.build_id),
|
||||
pipeline_name: cli.pipeline.clone(),
|
||||
username: cli.username.clone(),
|
||||
dry_run: cli.dry_run,
|
||||
privileged: true,
|
||||
..Default::default()
|
||||
};
|
||||
// Wait for bakerd to finish Fetch-1
|
||||
let stage = stage.unwrap_or(PrebakeStage::Bootstrap);
|
||||
if let Some(env) = &config.envvars {
|
||||
if let Some(prebake) = &env.prebake {
|
||||
ctx.env_vars.extend(prebake.clone());
|
||||
}
|
||||
}
|
||||
|
||||
// Bootstrap stage
|
||||
ctx.task_id = format!("{}-{}-bootstrap", cli.pipeline, cli.build_id);
|
||||
// TODO: Pass real event_tx
|
||||
stage::bootstrap::bootstrap(&config, &ctx, &None).await?;
|
||||
// TODO: Early security implementation
|
||||
|
||||
if stage <= PrebakeStage::Bootstrap {
|
||||
log::info!("Running PBStage: Bootstrap");
|
||||
prebake_drop_privilege(PrebakeStage::Bootstrap, dropafter, target_user, &mut ctx)?;
|
||||
ctx.task_id = format!("{}-{}-bootstrap", cli.pipeline, cli.build_id);
|
||||
// TODO: Pass real event_tx
|
||||
stage::bootstrap::bootstrap(&config, &ctx, &None).await?;
|
||||
}
|
||||
|
||||
// EarlyHook stage
|
||||
ctx.task_id = format!("{}-{}-earlyhook", cli.pipeline, cli.build_id);
|
||||
let earlyhook = match &config.hooks {
|
||||
Some(hooks) => hooks.early.clone(),
|
||||
None => None,
|
||||
};
|
||||
stage::hook::hook(earlyhook, &ctx, None).await?;
|
||||
if stage <= PrebakeStage::EarlyHook {
|
||||
log::info!("Running PBStage: EarlyHook");
|
||||
prebake_drop_privilege(PrebakeStage::EarlyHook, dropafter, target_user, &mut ctx)?;
|
||||
ctx.task_id = format!("{}-{}-earlyhook", cli.pipeline, cli.build_id);
|
||||
let earlyhook = match &config.hooks {
|
||||
Some(hooks) => hooks.early.clone(),
|
||||
None => None,
|
||||
};
|
||||
stage::hook::hook(earlyhook, &ctx, None).await?;
|
||||
}
|
||||
|
||||
if let Some(dependencies) = config.dependencies {
|
||||
// DepsSystem stage
|
||||
ctx.task_id = format!("{}-{}-depssystem", cli.pipeline, cli.build_id);
|
||||
if let Some(system) = dependencies.system {
|
||||
stage::depssystem::depssystem(&system, &ctx, None).await?;
|
||||
if stage <= PrebakeStage::DepsSystem {
|
||||
log::info!("Running PBStage: DepsSystem");
|
||||
prebake_drop_privilege(PrebakeStage::DepsSystem, dropafter, target_user, &mut ctx)?;
|
||||
ctx.task_id = format!("{}-{}-depssystem", cli.pipeline, cli.build_id);
|
||||
if let Some(system) = dependencies.system {
|
||||
stage::depssystem::depssystem(&system, &ctx, None).await?;
|
||||
}
|
||||
}
|
||||
|
||||
// DepsUser stage
|
||||
ctx.task_id = format!("{}-{}-depsuser", cli.pipeline, cli.build_id);
|
||||
if let Some(user) = dependencies.user {
|
||||
stage::depsuser::depsuser(&user, &ctx, None).await?;
|
||||
if stage <= PrebakeStage::DepsUser {
|
||||
log::info!("Running PBStage: DepsUser");
|
||||
prebake_drop_privilege(PrebakeStage::DepsUser, dropafter, target_user, &mut ctx)?;
|
||||
ctx.task_id = format!("{}-{}-depsuser", cli.pipeline, cli.build_id);
|
||||
if let Some(user) = dependencies.user {
|
||||
stage::depsuser::depsuser(&user, &ctx, None).await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// LateHook stage
|
||||
ctx.task_id = format!("{}-{}-latehook", cli.pipeline, cli.build_id);
|
||||
let latehook = match &config.hooks {
|
||||
Some(hooks) => hooks.late.clone(),
|
||||
None => None,
|
||||
};
|
||||
stage::hook::hook(latehook, &ctx, None).await?;
|
||||
if stage <= PrebakeStage::LateHook {
|
||||
log::info!("Running PBStage: LateHook");
|
||||
prebake_drop_privilege(PrebakeStage::LateHook, dropafter, target_user, &mut ctx)?;
|
||||
ctx.task_id = format!("{}-{}-latehook", cli.pipeline, cli.build_id);
|
||||
let latehook = match &config.hooks {
|
||||
Some(hooks) => hooks.late.clone(),
|
||||
None => None,
|
||||
};
|
||||
stage::hook::hook(latehook, &ctx, None).await?;
|
||||
}
|
||||
|
||||
// Ready stage
|
||||
ctx.task_id = format!("{}-{}-ready", cli.pipeline, cli.build_id);
|
||||
// TODO: Security stage
|
||||
|
||||
log::info!("Prebake done!");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -195,7 +195,10 @@ pub struct Cache {
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct Security {} // TODO: Security module
|
||||
pub struct Security {
|
||||
#[serde(default)]
|
||||
pub drop_after: Option<String>,
|
||||
} // TODO: More security module
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct Hooks {
|
||||
@@ -219,7 +222,13 @@ impl PrebakeConfig {
|
||||
|
||||
let version_requirement =
|
||||
VersionReq::parse(VERSION_REQUIREMENT).expect("Invalid version requirement");
|
||||
match Version::parse(&self.version) {
|
||||
let version = &self.version;
|
||||
let version = match version.matches('.').count() {
|
||||
0 => format!("{}.0.0", version),
|
||||
1 => format!("{}.0", version),
|
||||
_ => version.clone(),
|
||||
};
|
||||
match Version::parse(&version) {
|
||||
Ok(version) if version_requirement.matches(&version) => {}
|
||||
Ok(version) => errors.push(format!(
|
||||
"Version {} does not satisfy requirement {}",
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
目前的prebake流程如下：
|
||||
PBStage-init:
|
||||
此时ws-agent尚在获取流水线数据，PBStage0仅仅是等待其获取流水线定义文件(prebake.yml, bake.sh, finalize.yml)并尝试优先获取重要文件（如Dockerfile）。
|
||||
ws-agent在获取完数据后还会继续获取与配置缓存。此时PBStage与ws-agent是异步的。
|
||||
ws-agent获取数据可能随时结束，也可能随时出错，此时若不可恢复，则流水线生命周期终止。
|
||||
PBStage-env:
|
||||
ws-executor于主机运行，并开始根据environment配置环境。此时可能会拉取镜像，下载img文件或完成自定义环境准备。
|
||||
特殊地，若environment=container且dockerfile被指定，容器便会开始构建，但只要其依赖流水线的其它文件（且尚未被获取到）则PBStage1暂停于此，等待ws-agent的“获取结束”信号。这样做可以保证其依赖获取(FROM)字段能被处理，节省时间。
|
||||
PBStage-bootstrap:
|
||||
ws-executor被复制到目标环境并以root运行。其会与主机ws-executor通信并获取固定的环境初始化程序（有点像cloud-init），设置一般用户Vulcan和必要的免密权限，根据环境不同可能会使用systemd-run0/sudo/直接使用root，用户也可以使用模板或手动编写的程序覆盖此行为。若条件不可满足（如尝试在busybox中使用普通用户，但没有包管理器也没有用户管理手段）则流水线终止于此。该阶段还可能设置其它必要的信息。prebake环境变量也于此时注入。
|
||||
PBStage-earlysecurity:
|
||||
ws-executor@host,ws-executor@env开始按照配置文件设置必要的早期安全环境，不过目前大概除了创建cgroups组外什么都不会发生，暂时可以忽略此步骤
|
||||
PBStage-prehook:
|
||||
ws-executor在目标环境于Vulcan（或用户指定的，或者root，下略）运行，开始预烘焙钩子。
|
||||
PBStage-snapshot:
|
||||
这是一个伪阶段，executor@host会按配置（在可能的环境）创建目标环境的快照。若不可实现则自动忽略，此时dependency阶段失败将按照配置从bootstrap或dependency自己重新开始。
|
||||
目前我们尚不清楚容器/虚拟机快照能做到哪种地步。
|
||||
我们尚未设计这个字段retry-policy，其应该在prebake.yml的dependency中。
|
||||
PBStage-dependency:
|
||||
理论上，若存在缓存，则ws-executor@host必须等待ws-agent结束，ws-agent发射结束信号时，其确保获取了文件，同步到共享分区并（按配置）检查过。
|
||||
实际上可能会让依赖自行认领cache，不被认领的cache认为是bake cache从而允许忽略
|
||||
ws-executor在目标环境于Vulcan，从custom_pre到system到language到custom完成依赖安装动作
|
||||
用户可以配置可选的custom_*的清除脚本，使得custom+custom_clear是幂等的
|
||||
（如果不配置且存在不幂等的风险，用户应当自行设置合理的retry-poicy）
|
||||
PBStage-posthook:
|
||||
开始后依赖钩子
|
||||
PBStage-security:
|
||||
ws-executor@host,ws-executor@env开始按照配置文件设置必要的安全环境，不过目前大概什么都不会发生，暂时可以忽略此步骤
|
||||
PBStage-readyhook:
|
||||
开始完成时钩子
|
||||
PBStage-ready:
|
||||
完成bake环境变量的注入，标识准备结束，ws-executor@env会告知ws-executor@host可以开始bake了。
|
||||
这样编排好吗？可能有什么问题或遗漏吗？
|
||||
@@ -1,4 +1,7 @@
|
||||
use std::str::FromStr;
|
||||
use crate::ExecutionContext;
|
||||
use crate::error::PrebakeError;
|
||||
use crate::prebake::config::Security;
|
||||
use crate::prebake::stage::PrebakeStage;
|
||||
use privdrop::PrivDrop;
|
||||
use std::ffi::CString;
|
||||
@@ -171,7 +174,11 @@ pub fn prebake_drop_privilege(
|
||||
stage_current: PrebakeStage,
|
||||
stage_expected: PrebakeStage,
|
||||
user: &str,
|
||||
ctx: &mut ExecutionContext,
|
||||
) -> Result<(), PrebakeError> {
|
||||
if ctx.dry_run {
|
||||
return Ok(())
|
||||
}
|
||||
if stage_current > stage_expected {
|
||||
let current_uid = unsafe { libc::geteuid() };
|
||||
|
||||
@@ -216,6 +223,14 @@ pub fn prebake_drop_privilege(
|
||||
new_uid,
|
||||
user
|
||||
);
|
||||
ctx.privileged = false;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn get_drop_after(security: &Option<Security>) -> Result<PrebakeStage, String> {
|
||||
security.clone()
|
||||
.and_then(|sec| sec.drop_after)
|
||||
.map(|stage_str| PrebakeStage::from_str(&stage_str))
|
||||
.unwrap_or(Ok(PrebakeStage::default()))
|
||||
}
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
use std::str::FromStr;
|
||||
|
||||
pub mod bootstrap;
|
||||
pub mod environment;
|
||||
pub mod hook;
|
||||
@@ -42,3 +44,26 @@ impl std::fmt::Display for PrebakeStage {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for PrebakeStage {
|
||||
type Err = String;
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
match s.to_lowercase().as_str() {
|
||||
"init" => Ok(PrebakeStage::Init),
|
||||
"bootstrap" => Ok(PrebakeStage::Bootstrap),
|
||||
"earlyhook" => Ok(PrebakeStage::EarlyHook),
|
||||
"depssystem" => Ok(PrebakeStage::DepsSystem),
|
||||
"depsuser" => Ok(PrebakeStage::DepsUser),
|
||||
"latehook" => Ok(PrebakeStage::LateHook),
|
||||
"ready" => Ok(PrebakeStage::Ready),
|
||||
_ => Err(format!("Unknown stage: {}", s)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for PrebakeStage {
|
||||
fn default() -> Self {
|
||||
PrebakeStage::DepsSystem
|
||||
}
|
||||
}
|
||||
|
||||
@@ -45,6 +45,7 @@ pub async fn bootstrap(
|
||||
|
||||
// Check if bootstrap script exists before execution
|
||||
if !bootstrap_path.exists() {
|
||||
log::error!("/bootstrap.sh does not exist");
|
||||
return Err(ExecutionError::ScriptNotFound(bootstrap_path));
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
#!/bin/sh
|
||||
# Template from HBW baker - Minimal bootstrap
|
||||
set -e
|
||||
|
||||
# Create user (if not exists)
|
||||
if ! id -u vulcan >/dev/null 2>&1; then
|
||||
if command -v useradd >/dev/null 2>&1; then
|
||||
useradd -m -s /bin/nologin vulcan
|
||||
elif command -v adduser >/dev/null 2>&1; then
|
||||
adduser --disabled-password --gecos "" --shell /bin/nologin vulcan
|
||||
fi
|
||||
fi
|
||||
|
||||
# Configure sudoers (allow package manager commands)
|
||||
mkdir -p /etc/sudoers.d/
|
||||
cat > /etc/sudoers.d/vulcan << 'EOF'
|
||||
vulcan ALL=(ALL) NOPASSWD: /usr/bin/pacman, /usr/bin/apt-get, /usr/bin/dnf, /usr/bin/yum, /usr/bin/apk
|
||||
EOF
|
||||
chmod 440 /etc/sudoers.d/vulcan
|
||||
|
||||
# Set up workspace directory
|
||||
mkdir -p /home/vulcan/workspace
|
||||
chown vulcan:vulcan /home/vulcan/workspace
|
||||
ln -sf /home/vulcan/workspace /workspace # as a fallback path
|
||||
|
||||
echo "Bootstrap completed"
|
||||
@@ -39,4 +39,4 @@ pub struct CustomConfig {
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct BaremetalConfig;
|
||||
pub struct BaremetalConfig {}
|
||||
|
||||
@@ -2,6 +2,7 @@ use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct CustomCommand {
|
||||
#[serde(default="default_name")]
|
||||
pub name: String,
|
||||
pub command: String,
|
||||
#[serde(default)]
|
||||
@@ -15,3 +16,7 @@ pub struct CustomCommand {
|
||||
fn default_timeout() -> u32 {
|
||||
300
|
||||
}
|
||||
|
||||
fn default_name() -> String {
|
||||
"(unnamed)".to_string()
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
use workshop_executor::engine::{Engine, ExecutionContext};
|
||||
use workshop_baker::engine::{Engine, ExecutionContext};
|
||||
|
||||
fn create_executor() -> Engine {
|
||||
Engine::new()
|
||||
@@ -25,6 +25,8 @@ fn create_context() -> ExecutionContext {
|
||||
timeout: Some(Duration::from_secs(30)),
|
||||
cgroup_path: None,
|
||||
shell: "bash".to_string(),
|
||||
dry_run: false,
|
||||
privileged: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -171,6 +173,7 @@ async fn test_execute_respects_working_directory() {
|
||||
timeout: Some(Duration::from_secs(30)),
|
||||
cgroup_path: None,
|
||||
shell: "bash".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
use tokio::process::Command;
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
__pycache__
|
||||
@@ -0,0 +1,61 @@
|
||||
# Integration Tests
|
||||
|
||||
This directory contains integration tests for the workshop-baker prebake workflow.
|
||||
|
||||
## Requirements
|
||||
|
||||
- Python 3.10+
|
||||
- Docker
|
||||
- pytest
|
||||
|
||||
## Setup
|
||||
|
||||
Install Python dependencies:
|
||||
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
## Running Tests
|
||||
|
||||
### Run all integration tests
|
||||
|
||||
```bash
|
||||
pytest tests/integration/test_prebake.py -v
|
||||
```
|
||||
|
||||
### Run specific test class
|
||||
|
||||
```bash
|
||||
# Run only Docker container tests
|
||||
pytest tests/integration/test_prebake.py::TestPrebakeIntegration -v
|
||||
|
||||
# Run only configuration validation tests
|
||||
pytest tests/integration/test_prebake.py::TestPrebakeConfigValidation -v
|
||||
```
|
||||
|
||||
### Run specific test
|
||||
|
||||
```bash
|
||||
pytest tests/integration/test_prebake.py::TestPrebakeIntegration::test_project_compiles -v
|
||||
```
|
||||
|
||||
### Skip Docker tests (for local development)
|
||||
|
||||
```bash
|
||||
pytest tests/integration/test_prebake.py::TestPrebakeConfigValidation -v
|
||||
pytest tests/integration/test_prebake.py::TestPrebakeExamples -v
|
||||
```
|
||||
|
||||
## Test Structure
|
||||
|
||||
- `test_prebake.py::TestPrebakeIntegration` - Docker-based integration tests
|
||||
- `test_prebake.py::TestPrebakeConfigValidation` - YAML configuration validation tests
|
||||
- `test_prebake.py::TestPrebakeExamples` - Example files validation tests
|
||||
|
||||
## Notes
|
||||
|
||||
- Integration tests require Docker to be running
|
||||
- Tests use `archlinux:latest` Docker image
|
||||
- Tests mount the project directory into the container
|
||||
- Some tests may take longer due to Rust compilation
|
||||
@@ -0,0 +1,4 @@
|
||||
# Python dependencies for integration tests
|
||||
docker>=7.0.0
|
||||
pytest>=8.0.0
|
||||
PyYAML>=6.0
|
||||
+87
@@ -0,0 +1,87 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Run integration tests for workshop-baker
|
||||
#
|
||||
# Usage:
|
||||
# ./run_tests.sh # Run all tests
|
||||
# ./run_tests.sh --skip-docker # Skip Docker-based tests
|
||||
# ./run_tests.sh --dry-run # Run dry-run prebake and save to dryrun.log
|
||||
#
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
|
||||
LOG_FILE="${PROJECT_ROOT}/dryrun.log"
|
||||
|
||||
# Parse arguments
|
||||
SKIP_DOCKER=false
|
||||
RUN_DRY_RUN=false
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--skip-docker)
|
||||
SKIP_DOCKER=true
|
||||
shift
|
||||
;;
|
||||
--dry-run)
|
||||
RUN_DRY_RUN=true
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option: $1"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
cd "$PROJECT_ROOT/workshop-baker"
|
||||
|
||||
# Function to run Python tests
|
||||
run_python_tests() {
|
||||
echo "=== Running Integration Tests ==="
|
||||
if [ "$SKIP_DOCKER" = true ]; then
|
||||
echo "Skipping Docker-based tests..."
|
||||
python3 -m pytest tests/integration/test_prebake.py::TestPrebakeConfigValidation -v
|
||||
python3 -m pytest tests/integration/test_prebake.py::TestPrebakeExamples -v
|
||||
else
|
||||
python3 -m pytest tests/integration/test_prebake.py -v
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to run Rust tests
|
||||
run_rust_tests() {
|
||||
echo ""
|
||||
echo "=== Running Rust Tests ==="
|
||||
cargo test --lib
|
||||
}
|
||||
|
||||
# Function to run prebake dry-run
|
||||
run_prebake_dry_run() {
|
||||
echo ""
|
||||
echo "=== Running Prebake Dry-Run ==="
|
||||
echo "Output will be saved to: $LOG_FILE"
|
||||
|
||||
cargo run -- \
|
||||
--username test-user \
|
||||
--pipeline test-pipeline \
|
||||
--build-id test-build-001 \
|
||||
--dry-run \
|
||||
prebake examples/prebake.yml > "$LOG_FILE" 2>&1
|
||||
|
||||
echo "Dry-run completed. Results saved to: $LOG_FILE"
|
||||
echo ""
|
||||
echo "=== Last 30 lines of dryrun.log ==="
|
||||
tail -30 "$LOG_FILE"
|
||||
}
|
||||
|
||||
# Main execution
|
||||
if [ "$RUN_DRY_RUN" = true ]; then
|
||||
run_prebake_dry_run
|
||||
else
|
||||
run_python_tests
|
||||
run_rust_tests
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== All tests completed successfully ==="
|
||||
@@ -0,0 +1,593 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Integration tests for workshop-baker prebake workflow.
|
||||
|
||||
These tests execute the complete prebake pipeline in a Docker container
|
||||
running archlinux, using pre-built binaries from the host system.
|
||||
|
||||
Usage:
|
||||
pytest tests/integration/test_prebake.py -v
|
||||
pytest tests/integration/test_prebake.py -v -k "test_bootstrap" # Run specific test
|
||||
"""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import docker
|
||||
import pytest
|
||||
|
||||
|
||||
class DockerContainer:
|
||||
"""Context manager for Docker container lifecycle."""
|
||||
|
||||
def __init__(self, image: str, name: Optional[str] = None, **kwargs):
|
||||
self.image = image
|
||||
self.name = name or f"workshop-baker-test-{int(time.time())}"
|
||||
self.container = None
|
||||
self.client = docker.from_env()
|
||||
self.host_dir = Path(tempfile.mkdtemp(prefix="workshop-baker-"))
|
||||
self.pull_output = None
|
||||
self.container_kwargs = kwargs
|
||||
|
||||
def __enter__(self):
|
||||
try:
|
||||
print(f"Checking for image {self.image}...")
|
||||
self.client.images.get(self.image)
|
||||
print(f"Image {self.image} found locally")
|
||||
except docker.errors.ImageNotFound:
|
||||
try:
|
||||
print(f"Pulling image {self.image}...")
|
||||
self.pull_output = self.client.images.pull(self.image)
|
||||
print(f"Successfully pulled {self.image}")
|
||||
except docker.errors.DockerException as e:
|
||||
print(f"Docker exception: {e}")
|
||||
raise
|
||||
|
||||
self.container = self.client.containers.run(
|
||||
self.image,
|
||||
detach=True,
|
||||
name=self.name,
|
||||
remove=True,
|
||||
command="sleep infinity",
|
||||
tty=True,
|
||||
stdin_open=True,
|
||||
security_opt=["seccomp=unconfined"],
|
||||
cap_add=["SYS_ADMIN"],
|
||||
**self.container_kwargs,
|
||||
)
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
if self.container:
|
||||
try:
|
||||
self.container.stop(timeout=10)
|
||||
except Exception as e:
|
||||
print(f"Warning: Failed to stop container: {e}")
|
||||
self.client.close()
|
||||
return False
|
||||
|
||||
def exec_run(self, cmd: str, workdir: Optional[str] = None) -> tuple[int, str, str]:
|
||||
"""Execute command in container, return (exit_code, stdout, stderr)."""
|
||||
if self.container is None:
|
||||
return -1, "", "Container not running"
|
||||
result = self.container.exec_run(
|
||||
cmd,
|
||||
workdir=workdir,
|
||||
demux=True,
|
||||
)
|
||||
exit_code = result.exit_code
|
||||
stdout, stderr = result.output
|
||||
return (
|
||||
exit_code,
|
||||
(stdout or b"").decode("utf-8", errors="replace"),
|
||||
(stderr or b"").decode("utf-8", errors="replace"),
|
||||
)
|
||||
|
||||
def put_archive(self, local_path: Path, container_path: str):
|
||||
"""Copy local directory to container."""
|
||||
if self.container is None:
|
||||
raise RuntimeError("Container not running")
|
||||
import tarfile
|
||||
import io
|
||||
|
||||
tar_stream = io.BytesIO()
|
||||
with tarfile.open(fileobj=tar_stream, mode="w") as tar:
|
||||
tar.add(local_path, arcname=local_path.name)
|
||||
tar_stream.seek(0)
|
||||
|
||||
self.container.put_archive(os.path.dirname(container_path), tar_stream)
|
||||
|
||||
def copy_file(self, host_path: Path, container_path: str):
|
||||
"""Copy a single file to the container."""
|
||||
if self.container is None:
|
||||
raise RuntimeError("Container not running")
|
||||
import tarfile
|
||||
import io
|
||||
|
||||
tar_stream = io.BytesIO()
|
||||
with tarfile.open(fileobj=tar_stream, mode="w") as tar:
|
||||
tar.add(host_path, arcname=os.path.basename(container_path))
|
||||
tar_stream.seek(0)
|
||||
|
||||
self.container.put_archive(os.path.dirname(container_path), tar_stream)
|
||||
|
||||
|
||||
class TestPrebakeIntegration:
|
||||
"""Integration tests for workshop-baker prebake workflow."""
|
||||
|
||||
IMAGE = "archlinux:latest"
|
||||
CONTAINER_NAME = "workshop-baker-integration-test"
|
||||
PROJECT_ROOT = Path(__file__).parent.parent.parent
|
||||
BINARY_NAME = "workshop-executor"
|
||||
EXAMPLES_DIR = PROJECT_ROOT / "examples"
|
||||
PREBAKE_YML = EXAMPLES_DIR / "prebake.yml"
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def container_with_binary(self):
|
||||
"""Provide a running archlinux container with pre-built binary copied."""
|
||||
# Determine binary path (prefer release)
|
||||
release_binary = self.PROJECT_ROOT / "target" / "release" / self.BINARY_NAME
|
||||
debug_binary = self.PROJECT_ROOT / "target" / "debug" / self.BINARY_NAME
|
||||
|
||||
if release_binary.exists():
|
||||
binary_path = release_binary
|
||||
elif debug_binary.exists():
|
||||
binary_path = debug_binary
|
||||
else:
|
||||
pytest.skip(
|
||||
f"Binary {self.BINARY_NAME} not found. Run 'cargo build --release' first."
|
||||
)
|
||||
|
||||
# Check if image exists locally
|
||||
client = docker.from_env()
|
||||
try:
|
||||
client.images.get(self.IMAGE)
|
||||
except docker.errors.ImageNotFound:
|
||||
try:
|
||||
print(f"Pulling image {self.IMAGE}...")
|
||||
client.images.pull(self.IMAGE)
|
||||
except docker.errors.DockerException as e:
|
||||
pytest.skip(f"Could not pull image {self.IMAGE}: {e}")
|
||||
except Exception as e:
|
||||
pytest.skip(f"Could not access Docker: {e}")
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
container_kwargs = {
|
||||
"volumes": {
|
||||
str(self.PROJECT_ROOT): {"bind": "/workshop-baker", "mode": "rw"},
|
||||
},
|
||||
"working_dir": "/workshop-baker",
|
||||
}
|
||||
|
||||
container = DockerContainer(
|
||||
self.IMAGE,
|
||||
self.CONTAINER_NAME,
|
||||
**container_kwargs,
|
||||
)
|
||||
|
||||
with container:
|
||||
# Copy pre-built binary to container
|
||||
container.copy_file(binary_path, f"/usr/local/bin/{self.BINARY_NAME}")
|
||||
|
||||
# Make binary executable
|
||||
exit_code, stdout, stderr = container.exec_run(
|
||||
f"chmod +x /usr/local/bin/{self.BINARY_NAME}"
|
||||
)
|
||||
if exit_code != 0:
|
||||
pytest.fail(f"Failed to make binary executable: {stderr}")
|
||||
|
||||
# Create a minimal bootstrap.sh (required by prebake.yml)
|
||||
bootstrap_sh = container.host_dir / "bootstrap.sh"
|
||||
bootstrap_sh.write_text("#!/bin/bash\nexit 0\n")
|
||||
container.copy_file(bootstrap_sh, "/bootstrap.sh")
|
||||
|
||||
exit_code, stdout, stderr = container.exec_run("chmod +x /bootstrap.sh")
|
||||
if exit_code != 0:
|
||||
pytest.fail(f"Failed to set bootstrap.sh executable: {stderr}")
|
||||
|
||||
yield container
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def container(self):
|
||||
"""Provide a running archlinux container (without binary copy)."""
|
||||
container_kwargs = {
|
||||
"volumes": {
|
||||
str(self.PROJECT_ROOT): {"bind": "/workshop-baker", "mode": "rw"},
|
||||
},
|
||||
"working_dir": "/workshop-baker",
|
||||
}
|
||||
|
||||
container = DockerContainer(
|
||||
self.IMAGE,
|
||||
None,
|
||||
**container_kwargs,
|
||||
)
|
||||
|
||||
with container:
|
||||
yield container
|
||||
|
||||
@staticmethod
|
||||
def _get_setup_script() -> str:
|
||||
"""Return shell script to setup container environment (for non-binary tests)."""
|
||||
return """
|
||||
set -e
|
||||
|
||||
# Update package database
|
||||
pacman -Sy --noconfirm
|
||||
|
||||
# Install base development tools
|
||||
pacman -S --noconfirm \
|
||||
base-devel \
|
||||
git \
|
||||
curl \
|
||||
sudo \
|
||||
python
|
||||
|
||||
echo "=== Container Setup Complete ==="
|
||||
"""
|
||||
|
||||
def test_container_connectivity(self, container: DockerContainer):
|
||||
"""Test that container can reach external networks."""
|
||||
exit_code, stdout, stderr = container.exec_run("ping -c 1 8.8.8.8")
|
||||
assert exit_code == 0, f"Container network connectivity failed: {stderr}"
|
||||
|
||||
def test_binary_exists(self):
|
||||
"""Verify pre-built binary exists."""
|
||||
release_binary = self.PROJECT_ROOT / "target" / "release" / self.BINARY_NAME
|
||||
debug_binary = self.PROJECT_ROOT / "target" / "debug" / self.BINARY_NAME
|
||||
|
||||
assert release_binary.exists() or debug_binary.exists(), (
|
||||
f"Binary {self.BINARY_NAME} not found. Run 'cargo build --release' first."
|
||||
)
|
||||
|
||||
def test_binary_runs_in_container(self, container_with_binary: DockerContainer):
|
||||
"""Test that the pre-built binary can execute in the container."""
|
||||
cmd = f"/usr/local/bin/{self.BINARY_NAME} --help"
|
||||
exit_code, stdout, stderr = container_with_binary.exec_run(cmd)
|
||||
|
||||
print(f"Binary help output:\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}")
|
||||
|
||||
# Binary should at least run and show help or version
|
||||
assert exit_code == 0, f"Binary failed to execute: {stderr}"
|
||||
assert (
|
||||
"usage" in stdout.lower()
|
||||
or "help" in stdout.lower()
|
||||
or "version" in stdout.lower()
|
||||
), f"Binary help output unexpected: {stdout}"
|
||||
|
||||
def test_prebake_help_command(self, container_with_binary: DockerContainer):
|
||||
"""Test that the CLI help works."""
|
||||
cmd = f"/usr/local/bin/{self.BINARY_NAME} --help"
|
||||
exit_code, stdout, stderr = container_with_binary.exec_run(cmd)
|
||||
|
||||
assert exit_code == 0, f"Help command failed: {stderr}"
|
||||
assert (
|
||||
"Honey Biscuit Workshop" in stdout
|
||||
or "Usage" in stdout
|
||||
or "help" in stdout.lower()
|
||||
)
|
||||
|
||||
def test_prebake_config_parsing(self, container_with_binary: DockerContainer):
|
||||
"""Test that prebake.yml can be parsed correctly."""
|
||||
cmd = (
|
||||
f"/usr/local/bin/{self.BINARY_NAME} "
|
||||
"--username test-user "
|
||||
"--pipeline test-pipeline "
|
||||
"--build-id test-build-001 "
|
||||
"--dry-run "
|
||||
"prebake examples/prebake.yml"
|
||||
)
|
||||
exit_code, stdout, stderr = container_with_binary.exec_run(
|
||||
cmd, workdir="/workshop-baker"
|
||||
)
|
||||
|
||||
print(f"Config parsing output:\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}")
|
||||
|
||||
# dry-run should succeed even if actual execution would fail
|
||||
assert exit_code == 0 or "ERROR" not in stderr.upper(), (
|
||||
f"Config parsing failed: {stderr}"
|
||||
)
|
||||
|
||||
def test_prebake_bootstrap_stage(self, container_with_binary: DockerContainer):
|
||||
"""Test bootstrap stage execution."""
|
||||
cmd = (
|
||||
f"/usr/local/bin/{self.BINARY_NAME} "
|
||||
"--username test-user "
|
||||
"--pipeline test-pipeline "
|
||||
"--build-id test-build-001 "
|
||||
"prebake examples/prebake.yml"
|
||||
)
|
||||
exit_code, stdout, stderr = container_with_binary.exec_run(
|
||||
cmd, workdir="/workshop-baker"
|
||||
)
|
||||
|
||||
print(f"Bootstrap test output:\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}")
|
||||
|
||||
combined_output = stdout + stderr
|
||||
assert exit_code == 0, f"Bootstrap stage failed: {stderr}"
|
||||
|
||||
def test_prebake_hooks_execution(self, container_with_binary: DockerContainer):
|
||||
"""Test that hook commands are properly executed."""
|
||||
cmd = (
|
||||
f"/usr/local/bin/{self.BINARY_NAME} "
|
||||
"--username test-user "
|
||||
"--pipeline test-pipeline "
|
||||
"--build-id test-build-001 "
|
||||
"prebake examples/prebake.yml"
|
||||
)
|
||||
exit_code, stdout, stderr = container_with_binary.exec_run(
|
||||
cmd, workdir="/workshop-baker"
|
||||
)
|
||||
|
||||
combined_output = stdout + stderr
|
||||
print(f"Hooks test output:\n{combined_output}")
|
||||
|
||||
assert exit_code == 0, f"Hooks execution failed: {stderr}"
|
||||
|
||||
def test_prebake_dependencies_config(self, container_with_binary: DockerContainer):
|
||||
"""Test that dependencies section is properly processed."""
|
||||
cmd = (
|
||||
f"/usr/local/bin/{self.BINARY_NAME} "
|
||||
"--username test-user "
|
||||
"--pipeline test-pipeline "
|
||||
"--build-id test-build-001 "
|
||||
"prebake examples/prebake.yml"
|
||||
)
|
||||
exit_code, stdout, stderr = container_with_binary.exec_run(
|
||||
cmd, workdir="/workshop-baker"
|
||||
)
|
||||
|
||||
print(f"Dependencies test output:\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}")
|
||||
|
||||
assert exit_code == 0, f"Dependencies processing failed: {stderr}"
|
||||
|
||||
def test_prebake_environment_config(self, container_with_binary: DockerContainer):
|
||||
"""Test that environment section is properly processed."""
|
||||
cmd = (
|
||||
f"/usr/local/bin/{self.BINARY_NAME} "
|
||||
"--username test-user "
|
||||
"--pipeline test-pipeline "
|
||||
"--build-id test-build-001 "
|
||||
"prebake examples/prebake.yml"
|
||||
)
|
||||
exit_code, stdout, stderr = container_with_binary.exec_run(
|
||||
cmd, workdir="/workshop-baker"
|
||||
)
|
||||
|
||||
print(f"Environment test output:\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}")
|
||||
|
||||
assert exit_code == 0, f"Environment processing failed: {stderr}"
|
||||
|
||||
def test_prebake_cache_config(self, container_with_binary: DockerContainer):
|
||||
"""Test that cache configuration is properly processed."""
|
||||
cmd = (
|
||||
f"/usr/local/bin/{self.BINARY_NAME} "
|
||||
"--username test-user "
|
||||
"--pipeline test-pipeline "
|
||||
"--build-id test-build-001 "
|
||||
"prebake examples/prebake.yml"
|
||||
)
|
||||
exit_code, stdout, stderr = container_with_binary.exec_run(
|
||||
cmd, workdir="/workshop-baker"
|
||||
)
|
||||
|
||||
print(f"Cache config test output:\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}")
|
||||
|
||||
assert exit_code == 0, f"Cache configuration failed: {stderr}"
|
||||
|
||||
def test_prebake_security_config(self, container_with_binary: DockerContainer):
|
||||
"""Test that security configuration is properly processed."""
|
||||
cmd = (
|
||||
f"/usr/local/bin/{self.BINARY_NAME} "
|
||||
"--username test-user "
|
||||
"--pipeline test-pipeline "
|
||||
"--build-id test-build-001 "
|
||||
"prebake examples/prebake.yml"
|
||||
)
|
||||
exit_code, stdout, stderr = container_with_binary.exec_run(
|
||||
cmd, workdir="/workshop-baker"
|
||||
)
|
||||
|
||||
print(f"Security config test output:\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}")
|
||||
|
||||
assert exit_code == 0, f"Security configuration failed: {stderr}"
|
||||
|
||||
def test_prebake_full_run(self, container_with_binary: DockerContainer):
|
||||
"""Test complete prebake workflow in actual run mode."""
|
||||
cmd = (
|
||||
f"/usr/local/bin/{self.BINARY_NAME} "
|
||||
"--username test-user "
|
||||
"--pipeline test-pipeline "
|
||||
"--build-id test-build-001 "
|
||||
"prebake examples/prebake.yml"
|
||||
)
|
||||
exit_code, stdout, stderr = container_with_binary.exec_run(
|
||||
cmd, workdir="/workshop-baker"
|
||||
)
|
||||
|
||||
print(f"Full run output:\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}")
|
||||
|
||||
combined_output = stdout + stderr
|
||||
|
||||
assert "prebake" in combined_output.lower() or exit_code == 0, (
|
||||
f"Prebake command failed to execute properly. Exit code: {exit_code}"
|
||||
)
|
||||
|
||||
def test_prebake_metadata(self, container_with_binary: DockerContainer):
|
||||
"""Test that metadata is properly processed."""
|
||||
cmd = (
|
||||
f"/usr/local/bin/{self.BINARY_NAME} "
|
||||
"--username test-user "
|
||||
"--pipeline test-pipeline "
|
||||
"--build-id test-build-001 "
|
||||
"prebake examples/prebake.yml"
|
||||
)
|
||||
exit_code, stdout, stderr = container_with_binary.exec_run(
|
||||
cmd, workdir="/workshop-baker"
|
||||
)
|
||||
|
||||
print(f"Metadata test output:\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}")
|
||||
|
||||
assert exit_code == 0, f"Metadata processing failed: {stderr}"
|
||||
|
||||
def test_prebake_late_hooks(self, container_with_binary: DockerContainer):
|
||||
"""Test that late hooks are properly executed."""
|
||||
cmd = (
|
||||
f"/usr/local/bin/{self.BINARY_NAME} "
|
||||
"--username test-user "
|
||||
"--pipeline test-pipeline "
|
||||
"--build-id test-build-001 "
|
||||
"prebake examples/prebake.yml"
|
||||
)
|
||||
exit_code, stdout, stderr = container_with_binary.exec_run(
|
||||
cmd, workdir="/workshop-baker"
|
||||
)
|
||||
|
||||
print(f"Late hooks test output:\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}")
|
||||
|
||||
assert exit_code == 0, f"Late hooks execution failed: {stderr}"
|
||||
|
||||
|
||||
class TestPrebakeConfigValidation:
|
||||
"""Tests for prebake.yml configuration validation."""
|
||||
|
||||
PROJECT_ROOT = Path(__file__).parent.parent.parent
|
||||
EXAMPLES_DIR = PROJECT_ROOT / "examples"
|
||||
PREBAKE_YML = EXAMPLES_DIR / "prebake.yml"
|
||||
|
||||
def test_prebake_yml_exists(self):
|
||||
"""Verify prebake.yml exists and is readable."""
|
||||
assert self.PREBAKE_YML.exists(), f"prebake.yml not found at {self.PREBAKE_YML}"
|
||||
assert self.PREBAKE_YML.is_file(), f"prebake.yml is not a file"
|
||||
|
||||
def test_prebake_yml_valid_yaml(self):
|
||||
"""Verify prebake.yml is valid YAML."""
|
||||
import yaml
|
||||
|
||||
content = self.PREBAKE_YML.read_text()
|
||||
try:
|
||||
data = yaml.safe_load(content)
|
||||
assert data is not None, "prebake.yml is empty"
|
||||
assert isinstance(data, dict), "prebake.yml root should be a dictionary"
|
||||
except yaml.YAMLError as e:
|
||||
pytest.fail(f"Invalid YAML in prebake.yml: {e}")
|
||||
|
||||
def test_prebake_yml_required_fields(self):
|
||||
"""Verify prebake.yml has all required top-level fields."""
|
||||
import yaml
|
||||
|
||||
content = self.PREBAKE_YML.read_text()
|
||||
data = yaml.safe_load(content)
|
||||
|
||||
required_fields = ["version", "environment", "dependencies"]
|
||||
for field in required_fields:
|
||||
assert field in data, f"Missing required field: {field}"
|
||||
|
||||
def test_prebake_yml_version_format(self):
|
||||
"""Verify version field is properly formatted."""
|
||||
import yaml
|
||||
|
||||
content = self.PREBAKE_YML.read_text()
|
||||
data = yaml.safe_load(content)
|
||||
|
||||
version = data.get("version", "")
|
||||
assert re.match(r"^\d+\.\d+(\.\d+)?$", str(version)), (
|
||||
f"Invalid version format: {version}"
|
||||
)
|
||||
|
||||
def test_prebake_yml_environment_builder(self):
|
||||
"""Verify environment.builder is set."""
|
||||
import yaml
|
||||
|
||||
content = self.PREBAKE_YML.read_text()
|
||||
data = yaml.safe_load(content)
|
||||
|
||||
builder = data.get("environment", {}).get("builder", "")
|
||||
assert builder in ["docker", "firecracker", "custom", "baremetal"], (
|
||||
f"Invalid builder type: {builder}"
|
||||
)
|
||||
|
||||
def test_prebake_yml_dependencies_structure(self):
|
||||
"""Verify dependencies section has expected structure."""
|
||||
import yaml
|
||||
|
||||
content = self.PREBAKE_YML.read_text()
|
||||
data = yaml.safe_load(content)
|
||||
|
||||
deps = data.get("dependencies", {})
|
||||
if deps:
|
||||
has_system = "system" in deps
|
||||
has_user = "user" in deps
|
||||
assert has_system or has_user, (
|
||||
"Dependencies must have at least one of: system, user"
|
||||
)
|
||||
|
||||
def test_prebake_yml_hooks_structure(self):
|
||||
"""Verify hooks section has valid structure if present."""
|
||||
import yaml
|
||||
|
||||
content = self.PREBAKE_YML.read_text()
|
||||
data = yaml.safe_load(content)
|
||||
|
||||
hooks = data.get("hooks", {})
|
||||
if hooks:
|
||||
valid_hook_types = ["early", "late"]
|
||||
for hook_type in hooks.keys():
|
||||
assert hook_type in valid_hook_types, f"Invalid hook type: {hook_type}"
|
||||
|
||||
for hook_type, hook_list in hooks.items():
|
||||
assert isinstance(hook_list, list), f"Hook {hook_type} should be a list"
|
||||
|
||||
|
||||
class TestPrebakeExamples:
|
||||
"""Test suite for example files in the project."""
|
||||
|
||||
PROJECT_ROOT = Path(__file__).parent.parent.parent
|
||||
EXAMPLES_DIR = PROJECT_ROOT / "examples"
|
||||
|
||||
def test_examples_directory_exists(self):
|
||||
"""Verify examples directory exists."""
|
||||
assert self.EXAMPLES_DIR.exists(), f"Examples directory not found"
|
||||
assert self.EXAMPLES_DIR.is_dir(), f"Examples path is not a directory"
|
||||
|
||||
def test_dockerfile_example_exists(self):
|
||||
"""Verify Dockerfile example exists."""
|
||||
dockerfile = self.EXAMPLES_DIR / "Dockerfile_1"
|
||||
assert dockerfile.exists(), f"Dockerfile example not found at {dockerfile}"
|
||||
|
||||
def test_prebake_build_yml_exists(self):
|
||||
"""Verify prebake_build.yml example exists."""
|
||||
prebake_build = self.EXAMPLES_DIR / "prebake_build.yml"
|
||||
if prebake_build.exists():
|
||||
import yaml
|
||||
|
||||
content = prebake_build.read_text()
|
||||
try:
|
||||
yaml.safe_load(content)
|
||||
except yaml.YAMLError as e:
|
||||
pytest.fail(f"Invalid YAML in prebake_build.yml: {e}")
|
||||
|
||||
|
||||
def run_integration_tests():
|
||||
"""Run integration tests with Docker."""
|
||||
import sys
|
||||
|
||||
pytest.main(
|
||||
[
|
||||
__file__,
|
||||
"-v",
|
||||
"--tb=short",
|
||||
"-p",
|
||||
"no:warnings",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_integration_tests()
|
||||
Reference in New Issue
Block a user