feat: rename workshop-executor to workshop-baker and add comprehensive
documentation - Rename crate from workshop-executor to workshop-baker - Add MDBook documentation infrastructure with Chinese docs (Vol1/2/3) - Implement engine module with cgroups resource management - Add package manager support (apt, pacman) - Add daemon module with Unix socket IPC and heartbeat - Add privdrop dependency for privilege dropping - Implement prebake stages (bootstrap, depssystem, depsuser, environment, hooks) - Add new bake.sh.tmpl pipeline template for osu! project - Add pipeline implicit parameter documentation (PIPELINE_BAREMETAL_*, PIPELINE_CONTAINER_*, PIPELINE_UNCOVER_SECRET)
This commit is contained in:
Catty Steve
@@ -0,0 +1,85 @@
|
||||
#!/bin/bash
|
||||
|
||||
#Implicit set -euo pipefail
|
||||
|
||||
# function to be used by any pipeline stage
|
||||
function get_custom_library() {
|
||||
curl -LO https://archive.ppy.sh/$2/$1.tar.gz
|
||||
}
|
||||
|
||||
# @pipeline
|
||||
prepare_dotnet() {
|
||||
curl -L https://dot.net/v1/dotnet-install.sh | bash
|
||||
bake_info "Successfully installed dotnet"
|
||||
}
|
||||
|
||||
# @pipeline
|
||||
fetch_source() {
|
||||
git clone https://github.com/ppy/osu.git --depth 1
|
||||
}
|
||||
|
||||
# @pipeline
|
||||
# @if(WS_PIPELINE_ARCH == amd64)
|
||||
# @timeout(720)
|
||||
# @retry(3, 5)
|
||||
# @parallel
|
||||
fetch_library() {
|
||||
get_custom_library ffmpeg amd64
|
||||
get_custom_library libesqlite3 amd64
|
||||
get_custom_library libbass amd64
|
||||
}
|
||||
|
||||
|
||||
# @pipeline
|
||||
# @fallible
|
||||
# @parallel
|
||||
fetch_osu_framework() {
|
||||
cd ..
|
||||
git clone https://github.com/ppy/osu-framework.git --depth 1
|
||||
}
|
||||
|
||||
# @pipeline
|
||||
# @timeout(600)
|
||||
restore_workload() {
|
||||
cd osu
|
||||
dotnet workload restore
|
||||
}
|
||||
|
||||
# @pipeline
|
||||
# @export
|
||||
set_environment() {
|
||||
export DOTNET_ENVIRONMENT=Production
|
||||
}
|
||||
|
||||
# @pipeline
|
||||
# @export
|
||||
build() {
|
||||
dotnet build osu.Desktop
|
||||
dotnet publish osu.Desktop -o osu-build --sc
|
||||
}
|
||||
|
||||
# @pipeline
|
||||
# @loop(3)
|
||||
test() {
|
||||
dotnet test
|
||||
}
|
||||
|
||||
# @pipeline
|
||||
# @parallel
|
||||
# @after(fetch_library)
|
||||
compress_library() {
|
||||
mkdir -p osu-native
|
||||
cp osu-build/*.so ../osu-native
|
||||
tar -cf osu-native osu-native.tar
|
||||
zstd osu.tar
|
||||
}
|
||||
|
||||
# @pipeline
|
||||
# @after(test)
|
||||
compress_artifact() {
|
||||
tar -cf osu osu.tar
|
||||
zstd osu.tar
|
||||
}
|
||||
|
||||
|
||||
|
||||
+1
-199
@@ -1,4 +1,3 @@
|
||||
对以上信息进行了补充
|
||||
我们正在实现一个基于Rust的CI/CD系统，名为'蜜饼工坊'(HoneyBiscuitWorkshop)，取材于明日方舟世界观中小刻爱吃的蜜饼和她的管理者火神大姐的工坊。
|
||||
这个CI/CD的最终用途如下：
|
||||
0. 一个合格的CICD当然可以DevOps
|
||||
@@ -23,205 +22,8 @@ LLM可以读取关键文件/定义的文件/预设规则/上次生成的报错
|
||||
9. 本地流水线快捷验证
|
||||
可以在不借助服务器的情况下快速验证流水线语法的正确性，在借助本机容器的情况下进行试构建
|
||||
10. 本地文件夹推送构建
|
||||
11. Vol1/2/3 用户/管理员/开发者文档
|
||||
可以推送本地的任何一个文件夹到服务器，其会根据预设规则/.workshop中的文件/LLM自生成进行构建，支持增量传输
|
||||
CI/CD采用Web+Server+Multi (Agent+Executor)设计。当然也有可能的CLI客户端用于上述目的。
|
||||
这个CI/CD可能会结合koishi, rustyvault（https://rustyvault.net，使用标准的Vault也可）等外部项目构建，当然我们希望减少对外部项目的依赖，尤其是数据库(如MySQL，意即，我们尽可能在SQLite解决问题)和消息队列(如RabbitMQ)，但是不阻止部署者使用这些。
|
||||
bake.sh的基础说明如下：
|
||||
#!/bin/bash
|
||||
# This is a helper function lying in all task functions
|
||||
load() {
|
||||
echo "loading..."
|
||||
log Hello!
|
||||
}
|
||||
|
||||
# This is a task function with some example decorators
|
||||
# @pipeline
|
||||
# @fallible
|
||||
# @if PIPELINE_BUILD_ENABLE
|
||||
# @retry 3 10s
|
||||
build() {
|
||||
load
|
||||
echo "Build process"
|
||||
}
|
||||
示例prebake.yml如下：
|
||||
# prebake.yaml.tmpl
|
||||
#
|
||||
# 术语说明：
|
||||
# 可选：该字段可以不给出
|
||||
# 默认为...：隐含“可选”
|
||||
# 留空：尚未定义，留作后续
|
||||
|
||||
# 版本号，用于格式兼容性检查
|
||||
version: "1.0"
|
||||
|
||||
# 构建环境定义
|
||||
environment:
|
||||
# 构建机类型
|
||||
builder: "docker" # 或 "firecracker", "custom", "baremetal"
|
||||
|
||||
# 构建机配置，注意以下只会出现与上述构建机类型对应的组。
|
||||
config:
|
||||
# baremetal:
|
||||
# 不需要
|
||||
|
||||
# docker:
|
||||
# 镜像名
|
||||
image: "rust:1.70-slim"
|
||||
# Dockerfile路径，相对于项目根目录，不能与镜像一同指定，两者必须指定其一
|
||||
dockerfile: "/Dockerfile"
|
||||
# Docker构建参数，可选，只能与dockerfile搭配使用
|
||||
build_args:
|
||||
- "RUST_VERSION=1.70"
|
||||
|
||||
# firecracker:
|
||||
# 留空
|
||||
|
||||
# custom:
|
||||
# 自定义构建机名称
|
||||
name: "my-custom-builder"
|
||||
# 设置脚本，相对于项目根目录
|
||||
setup_script: "setup-builder.sh"
|
||||
# 清洁脚本，相对于项目根目录
|
||||
cleanup_script: "cleanup-builder.sh"
|
||||
|
||||
# 硬件资源要求，满足最小要求的构建机可以被选中
|
||||
resources:
|
||||
cpu: 2 # CPU核心数，默认为1
|
||||
memory: "2G" # 内存大小，也可以写作2048MB,默认为"1G"
|
||||
disk: "10G" # 磁盘空间，也可以写作10240MB,默认为"1G"
|
||||
architecture: "x86_64" # 架构要求，只能是"noarch"或{"x86_64"/"amd64", "arm64"/"aarch64", "riscv64", "loongarch64"/"loong64"}中的一个或几个
|
||||
gpu: false # 是否需要GPU，默认为false
|
||||
gpu_type: "cuda" # GPU类型（如果需要），可以是"cuda", "rocm", "vulkan", "oneapi"中的一个或几个
|
||||
tags: # 自定义标签，只有满足标签的构建机可以被选中
|
||||
- "custom_tag"
|
||||
|
||||
# 网络配置
|
||||
network:
|
||||
enabled: true # 启用网络，对baremetal不适用，默认为true
|
||||
outbound: true # 允许出站连接，对baremetal不适用，默认为true
|
||||
dns_servers: # 指定构建机的DNS服务器，对baremetal不适用，custom应该自行处理DNS问题，可选
|
||||
- "8.8.8.8"
|
||||
- "1.1.1.1"
|
||||
proxies: # 代理配置，暂时留空，可选
|
||||
|
||||
# 依赖定义，安装依赖时按custom_pre-system-languages-custom顺序进行
|
||||
dependencies:
|
||||
# 系统包依赖，按软件包类型给出或any通配，若存在发行版则选择之，否则选中any，也没有则报警告但继续
|
||||
system:
|
||||
deb:
|
||||
packages: # 安装的软件包
|
||||
- "git"
|
||||
- "curl"
|
||||
- "build-essential"
|
||||
- "pkg-config"
|
||||
- "libssl-dev"
|
||||
repositories:
|
||||
- "ppa:custom/ppa" # 自定义PPA（Ubuntu）
|
||||
mirror: "https://mirrors.tuna.tsinghua.edu.cn" # 指定软件包镜像源，可选
|
||||
pacman:
|
||||
packages:
|
||||
- "git"
|
||||
- "curl"
|
||||
- "base-devel"
|
||||
- "pkg-config"
|
||||
- "openssl"
|
||||
repositories:
|
||||
- "https://mirrors.tuna.tsinghua.edu.cn/arch4edu/$arch" # 自定义仓库（Arch）
|
||||
mirror: "https://mirrors.tuna.tsinghua.edu.cn" # 指定软件包镜像源，可选
|
||||
rpm:
|
||||
# 略
|
||||
|
||||
# 语言特定依赖，可选，此处定义不稳定
|
||||
languages:
|
||||
rust:
|
||||
toolchain: "stable-x86_64-unknown-linux-gnu"
|
||||
|
||||
# 自定义依赖步骤，可选
|
||||
custom:
|
||||
- name: "install-custom-tool" # 名称
|
||||
command: "curl -fsSL https://example.com/install.sh | sh" # 命令
|
||||
environment: # 环境变量
|
||||
CUSTOM_VAR: "value"
|
||||
|
||||
- name: "build-native-dep"
|
||||
command: "make && make install"
|
||||
working_dir: "native-deps" # 工作目录，基于项目根目录
|
||||
timeout: 300 # 超时，默认为300秒
|
||||
|
||||
# 自定义依赖步骤，但是在依赖安装时最先进行，可选
|
||||
custom_pre:
|
||||
- name: "install-custom-tool"
|
||||
command: "curl -fsSL https://example.com/install.sh | sh"
|
||||
environment:
|
||||
CUSTOM_VAR: "value"
|
||||
|
||||
- name: "build-native-dep"
|
||||
command: "make && make install"
|
||||
working_dir: "native-deps"
|
||||
timeout: 300
|
||||
|
||||
# 环境变量配置
|
||||
environment_variables:
|
||||
# 构建环境变量，只在prebake阶段有效
|
||||
prebake:
|
||||
RUST_BACKTRACE: "full"
|
||||
CARGO_NET_GIT_FETCH_WITH_CLI: "true"
|
||||
NODE_ENV: "production"
|
||||
|
||||
# 运行时环境变量（会传递给bake阶段，与该阶段获取的环境一并作为.env保存）
|
||||
bake:
|
||||
DATABASE_URL: "postgresql://user:pass@localhost/db"
|
||||
LOG_LEVEL: "info"
|
||||
__ENV_FILE: ".env" # 特殊变量，指定项目中.env的位置并合并其中的值。该值本身不传入bake阶段
|
||||
|
||||
# 缓存配置
|
||||
cache:
|
||||
# 缓存目录
|
||||
directories:
|
||||
- path: "/usr/local/cargo"
|
||||
strategy: "always" # 在"always","readonly","clean"和"never"中选择一个，可以被流水线启动时配置覆盖，默认为"always"
|
||||
mode: "zstd" # 在"gzip","zstd","none"和"dir"中选择一个
|
||||
|
||||
- path: "/root/.npm"
|
||||
strategy: "clean"
|
||||
mode: "none"
|
||||
|
||||
- path: "/app/target"
|
||||
strategy: "always"
|
||||
mode: "dir"
|
||||
|
||||
# 缓存策略
|
||||
strategy:
|
||||
ttl_days: 30 # 缓存生存时间
|
||||
max_size_gb: 20 # 最大缓存大小
|
||||
cleanup_policy: "lru" # 清理策略：lru, fifo, size
|
||||
|
||||
# 安全配置
|
||||
security:
|
||||
# 留空
|
||||
|
||||
# 钩子脚本
|
||||
hooks:
|
||||
# 依赖安装前
|
||||
pre_dependencies:
|
||||
- command: "echo 'Starting dependency installation'"
|
||||
|
||||
# 依赖安装后
|
||||
post_dependencies:
|
||||
- command: "cargo fetch"
|
||||
working_dir: "/app"
|
||||
|
||||
# 环境准备完成
|
||||
ready:
|
||||
- command: "echo 'Environment ready for build'"
|
||||
|
||||
# 元数据
|
||||
metadata:
|
||||
description: "Rust project build environment"
|
||||
maintainer: "team@example.com"
|
||||
tags: ["rust", "web", "api"]
|
||||
project_type: "rust-cargo"
|
||||
|
||||
finalize.yml的语法尚在设计
|
||||
对于通信
|
||||
请对其进行（创新性？可行性？实用性？等等）评估和设计，不需要写代码，但作为示例写一点也可以。
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
book
|
||||
@@ -0,0 +1,5 @@
|
||||
[book]
|
||||
title = "HoneyBiscuitWorkshop Documentation"
|
||||
authors = ["Catty Steve"]
|
||||
language = "zh-CN"
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
# HoneyBiscuitWorkshop Documentation
|
||||
|
||||
- [中文](zh-CN/index.md)
|
||||
- [卷1 用户手册](zh-CN/vol1_user/index.md)
|
||||
- [卷2 管理员手册](zh-CN/vol2_admin/index.md)
|
||||
- [流水线隐参数](zh-CN/vol2_admin/pipeline_implicit_parameter.md)
|
||||
- [PIPELINE_UNCOVER_SECRET](zh-CN/vol2_admin/pipeline_uncover_secret.md)
|
||||
- [PIPELINE_BAREMETAL_PKGMAN](zh-CN/vol2_admin/pipeline_baremetal_pkgman.md)
|
||||
- [PIPELINE_BAREMETAL_ELEVATE](zh-CN/vol2_admin/pipeline_baremetal_elevate.md)
|
||||
- [PIPELINE_CONTAINER_PRIVILEGED](zh-CN/vol2_admin/pipeline_container_privileged.md)
|
||||
- [卷3 开发者手册](zh-CN/vol3_dev/index.md)
|
||||
@@ -0,0 +1 @@
|
||||
some
|
||||
@@ -0,0 +1,44 @@
|
||||
# PIPELINE_<PERMISSION_NAME>: 名称的字面意思描述
|
||||
|
||||
[一句话简介]
|
||||
|
||||
0. 预警/FOREWARN
|
||||
[警告读者可能的风险]
|
||||
|
||||
1. 概要/Synopsis
|
||||
[给出其定义与作用]
|
||||
|
||||
2. 场景/Scenario
|
||||
[给出其常见配置场景]
|
||||
|
||||
3. 描述/Description
|
||||
[给出其细节描述]
|
||||
|
||||
4. 授权/Authorize
|
||||
[给出权限的获得方法]
|
||||
|
||||
5. 撤回/Revoke
|
||||
[给出权限的撤回方法]
|
||||
|
||||
6. 生命周期/Lifecycle
|
||||
[给出权限的生命周期描述]
|
||||
|
||||
7. 审计/Audit
|
||||
[描述其对审计系统的影响，若无请忽略]
|
||||
|
||||
8. 参数/Options
|
||||
[给出可配置的参数，若无请忽略]
|
||||
|
||||
9. 建议/Suggestions
|
||||
[字面意思]
|
||||
|
||||
10. 常见问题/FAQ
|
||||
[给出用户提出的问题。此时不应该填充任何内容。]
|
||||
|
||||
11~90. 自定义字段
|
||||
|
||||
98. 作者/Author
|
||||
[字面意思，LLM的使用应该也在这里标注]
|
||||
|
||||
99. 参见/See Also
|
||||
[给出可能有关的页面]
|
||||
@@ -0,0 +1 @@
|
||||
meow?
|
||||
@@ -0,0 +1,184 @@
|
||||
# PIPELINE_BAREMETAL_ELEVATE：裸机环境完整提权
|
||||
|
||||
> [!WARNING]
|
||||
> 此权限允许裸机流水线以完整 root 权限运行，可执行任意系统级操作。
|
||||
> 滥用可能导致系统崩溃、数据永久丢失、安全入侵或合规违规。
|
||||
> **仅在绝对必要时使用，必须经过严格审批、2FA 验证并记录完整审计。**
|
||||
|
||||
## 1. 概要
|
||||
|
||||
PIPELINE_BAREMETAL_ELEVATE 是一个流水线隐参数，用于**在裸机环境下临时授予流水线完整的 root 权限**。当该参数生效时，流水线可以执行任何系统级操作，包括修改系统配置、安装软件、操作内核模块等。
|
||||
|
||||
**此参数默认不开启，需要管理员通过严格流程审批并授权。**
|
||||
|
||||
## 2. 场景
|
||||
|
||||
| 适用场景 | 说明 |
|
||||
|----------|------|
|
||||
| 内核模块编译与加载 | 需要安装 kernel headers 并动态加载模块 |
|
||||
| 系统级性能调优 | 修改 sysctl 参数、CPU 调频策略、IRQ 亲和性 |
|
||||
| 硬件设备配置 | 配置 PCIe 直通、GPU 驱动安装、FPGA 编程 |
|
||||
| 系统镜像定制 | 修改根文件系统、引导配置 |
|
||||
| 遗留软件兼容 | 某些老旧软件硬编码需要 root 运行 |
|
||||
|
||||
| 不适用场景 | 说明 |
|
||||
|------------|------|
|
||||
| 普通软件包安装 | 请用 PIPELINE_BAREMETAL_PKGMAN |
|
||||
| 语言级依赖 | 请用 DepsUser 阶段的包管理器 |
|
||||
| 容器化构建 | 不需要裸机提权 |
|
||||
| 日常构建 | 正常流水线不应使用 root |
|
||||
|
||||
## 3. 描述
|
||||
|
||||
### 技术本质
|
||||
|
||||
PIPELINE_BAREMETAL_ELEVATE 通过临时切换进程凭据实现完整提权：
|
||||
|
||||
1. 在 DepsSystem 阶段开始前，baker 进程的 uid 临时切换为 0 (root)
|
||||
2. 所有子进程继承 root 权限
|
||||
3. 可执行任何系统调用、访问任何文件、修改任何配置
|
||||
4. DepsSystem 阶段结束后自动降权回普通用户
|
||||
5. 提权期间的所有操作都被强制审计
|
||||
|
||||
### 边界
|
||||
|
||||
| 允许的操作 | 不允许的操作 |
|
||||
|------------|--------------|
|
||||
| 安装/卸载系统软件包 | 修改系统引导项（超出流水线生命周期） |
|
||||
| 修改系统配置文件 | 永久关闭 SELinux/AppArmor |
|
||||
| 加载/卸载内核模块 | 安装持久化后门或定时任务 |
|
||||
| 操作其他用户的文件 | 删除其他用户的流水线数据 |
|
||||
| 修改系统服务状态 | 修改审计系统配置 |
|
||||
| 访问硬件设备 | 覆盖系统关键二进制 |
|
||||
|
||||
### 与其他参数的关系
|
||||
|
||||
| 参数 | 关系 |
|
||||
|------|------|
|
||||
| PIPELINE_BAREMETAL_PKGMAN | ELEVATE 包含 PKGMAN 的所有能力，两者互斥使用 |
|
||||
| PIPELINE_UNCOVER_SECRET | 可同时使用，但风险叠加，需特别审批 |
|
||||
| PIPELINE_CONTAINER_PRIVILEGED | 适用于不同环境，无直接关联 |
|
||||
|
||||
## 4. 授权
|
||||
|
||||
| 方式 | 适用场景 | 操作示例 |
|
||||
|------|----------|----------|
|
||||
| 管理员 CLI + 2FA | 紧急生产问题 | |
|
||||
| 审批系统集成 | 企业级合规流程 | |
|
||||
| 双人授权 | 高安全环境 | |
|
||||
|
||||
**授权流程**：
|
||||
|
||||
1. 用户或管理员发起授权请求，必须填写详细理由
|
||||
2. 系统要求发起人完成 2FA 验证
|
||||
3. 需要至少一名其他管理员二次审批（可选，可配置）
|
||||
4. 授权确认后，系统生成短期令牌随流水线下发
|
||||
|
||||
**授权确认示例**：
|
||||
```
|
||||
🚨 高危操作授权请求 🚨
|
||||
|
||||
正在申请 PIPELINE_BAREMETAL_ELEVATE 权限
|
||||
|
||||
流水线: pipeline-12345
|
||||
申请人: user@example.com
|
||||
理由: 需要安装自定义内核模块并调优 NUMA 参数
|
||||
|
||||
风险确认：
|
||||
- 此权限允许完整 root 访问
|
||||
- 可能造成系统不稳定或数据丢失
|
||||
- 所有操作将被强制审计
|
||||
|
||||
2FA 验证: ******
|
||||
|
||||
请输入 "I UNDERSTAND THE RISKS" 确认:
|
||||
>
|
||||
```
|
||||
|
||||
## 5. 撤回
|
||||
|
||||
| 撤回方式 | 说明 |
|
||||
|----------|------|
|
||||
| 手动撤回 | |
|
||||
| 自动失效 | DepsSystem 阶段结束后自动降权 |
|
||||
| 流水线结束 | 流水线结束时强制回收权限 |
|
||||
| 强制回收 | 审计系统检测到异常行为时可立即终止流水线 |
|
||||
| 超时回收 | 超过配置的最大提权时长后自动降权 |
|
||||
|
||||
## 6. 生命周期
|
||||
|
||||
| 状态 | 说明 |
|
||||
|------|------|
|
||||
| PENDING | 等待审批 |
|
||||
| APPROVED | 已审批通过 |
|
||||
| ACTIVE | 提权生效中 |
|
||||
| EXPIRED | 超过最大时长或阶段结束 |
|
||||
| REVOKED | 被管理员强制撤回 |
|
||||
| AUDITED | 已完成审计归档 |
|
||||
|
||||
## 7. 审计
|
||||
|
||||
所有提权期间的操作都会被强制记录，且日志不可篡改：
|
||||
|
||||
| 字段 | 说明 |
|
||||
|------|------|
|
||||
| `event` | GRANTED / COMMAND_EXECUTED / FILE_ACCESS / EXPIRE |
|
||||
| `user` | 申请人 |
|
||||
| `approvers` | 审批人列表 |
|
||||
| `pipeline_id` | 关联流水线 ID |
|
||||
| `timestamp` | 操作时间 |
|
||||
| `command` | 执行的完整命令行 |
|
||||
| `working_dir` | 执行路径 |
|
||||
| `exit_code` | 命令执行结果 |
|
||||
| `accessed_files` | 访问的关键文件列表 |
|
||||
| `syscalls` | 关键系统调用记录 |
|
||||
|
||||
**日志示例**：
|
||||
```json
|
||||
{
|
||||
"timestamp": "",
|
||||
"event": "",
|
||||
"pipeline_id": "",
|
||||
"user": "",
|
||||
"command": ""
|
||||
}
|
||||
```
|
||||
|
||||
## 8. 参数
|
||||
|
||||
### 全局配置
|
||||
|
||||
```toml
|
||||
[security.baremetal_elevate]
|
||||
enabled = false # 默认关闭
|
||||
require_2fa = true # 必须 2FA
|
||||
require_dual_approval = true # 需要双人审批
|
||||
max_duration = "30m" # 最长提权时间
|
||||
allowed_stages = ["DepsSystem"] # 允许提权的阶段
|
||||
audit_log = true # 强制审计
|
||||
notify_channels = ["email", "slack"] # 通知渠道
|
||||
```
|
||||
|
||||
### 运行时参数
|
||||
|
||||
|
||||
|
||||
## 9. 建议
|
||||
|
||||
### 对用户
|
||||
|
||||
|
||||
|
||||
### 对管理员
|
||||
|
||||
|
||||
|
||||
## 10. 常见问题
|
||||
|
||||
|
||||
|
||||
## 98. 作者
|
||||
|
||||
|
||||
|
||||
## 99. 参见
|
||||
@@ -0,0 +1,161 @@
|
||||
# PIPELINE_BAREMETAL_PKGMAN：裸机环境包管理器权限
|
||||
|
||||
> [!WARNING]
|
||||
> 此权限允许裸机流水线在 DepsSystem 阶段临时使用系统包管理器（需要 root 权限的操作）。
|
||||
> 滥用可能导致系统环境被破坏或意外安装恶意软件包。
|
||||
> **非必要不使用，使用时必须经过管理员审批并记录审计。**
|
||||
|
||||
## 1. 概要
|
||||
|
||||
PIPELINE_BAREMETAL_PKGMAN 是一个流水线隐参数，用于**在裸机环境下临时授予 DepsSystem 阶段使用系统包管理器的权限**。当该参数生效时，流水线可以执行 apt-get、pacman、yum 等包管理器命令安装构建依赖，但**不授予完整的 root 权限**。
|
||||
|
||||
**此参数默认不开启，需要管理员显式授予。**
|
||||
|
||||
## 2. 场景
|
||||
|
||||
| 适用场景 | 说明 |
|
||||
|----------|------|
|
||||
| 系统依赖安装 | 构建过程需要 gcc、make、openssl-dev 等系统包 |
|
||||
| 基础环境准备 | 裸机环境缺少必要的构建工具链 |
|
||||
| 内核模块编译 | 需要安装 kernel headers 等特权操作 |
|
||||
| 性能调优工具 | 某些性能分析工具需要系统级安装 |
|
||||
|
||||
| 不适用场景 | 说明 |
|
||||
|------------|------|
|
||||
| 任意命令执行 | 请用 PIPELINE_BAREMETAL_ELEVATE |
|
||||
| 容器化构建 | 不需要裸机包管理器 |
|
||||
| 语言级依赖 | 请用 DepsUser 阶段的 cargo/pip/npm |
|
||||
|
||||
## 3. 描述
|
||||
|
||||
### 技术本质
|
||||
|
||||
PIPELINE_BAREMETAL_PKGMAN 通过 sudoers 白名单实现受限提权：
|
||||
|
||||
1. 系统在 DepsSystem 阶段开始前，为 baker 配置临时 sudo 规则
|
||||
2. 仅允许执行预设的包管理器命令列表
|
||||
3. 所有命令执行都被审计记录
|
||||
4. DepsSystem 阶段结束后自动移除 sudo 权限
|
||||
|
||||
### 边界
|
||||
|
||||
| 允许的操作 | 不允许的操作 |
|
||||
|------------|--------------|
|
||||
| apt-get install | 任意 shell 命令 |
|
||||
| pacman -S | 修改系统配置文件 |
|
||||
| yum install | 操作其他用户的文件 |
|
||||
| dnf install | 加载内核模块 |
|
||||
| zypper install | 安装持久化服务 |
|
||||
| 包管理器缓存更新 | 修改系统引导项 |
|
||||
|
||||
### 依赖关系
|
||||
|
||||
- 需要系统启用 `security.baremetal_pkgman.enabled`（默认开启）
|
||||
- 与其他隐参数（如 UNCOVER_SECRET）可同时使用，但需评估叠加风险
|
||||
- 与 BAREMETAL_ELEVATE 互斥（后者已包含包管理器权限）
|
||||
|
||||
## 4. 授权
|
||||
|
||||
| 方式 | 适用场景 | 操作示例 |
|
||||
|------|----------|----------|
|
||||
| 管理员 CLI | 临时授权 | |
|
||||
| 审批系统集成 | 企业流程 | |
|
||||
| 用户申请 | 常规需求 | |
|
||||
|
||||
**授权流程**：
|
||||
|
||||
1. 管理员或用户发起授权请求
|
||||
2. 系统验证请求者权限
|
||||
3. 授权确认后，参数随流水线下发
|
||||
|
||||
**授权确认示例**：
|
||||
```
|
||||
⚠️ 正在授予 PIPELINE_BAREMETAL_PKGMAN 权限
|
||||
|
||||
流水线: pipeline-12345
|
||||
申请人: user@example.com
|
||||
理由: 需要安装 gcc、make 等构建依赖
|
||||
|
||||
此权限允许：
|
||||
- 在 DepsSystem 阶段使用包管理器
|
||||
- 仅限预设命令列表
|
||||
- 不授予其他 root 权限
|
||||
|
||||
确认授予？(y/N)
|
||||
```
|
||||
|
||||
## 5. 撤回
|
||||
|
||||
| 撤回方式 | 说明 |
|
||||
|----------|------|
|
||||
| 手动撤回 | |
|
||||
| 自动失效 | DepsSystem 阶段结束后自动回收 |
|
||||
| 流水线结束 | 流水线结束时权限自动回收 |
|
||||
| 强制回收 | 审计系统检测到异常时可终止流水线 |
|
||||
|
||||
## 6. 生命周期
|
||||
|
||||
| 状态 | 说明 |
|
||||
|------|------|
|
||||
| PENDING | 等待审批 |
|
||||
| ACTIVE | 已授权，DepsSystem 阶段可用 |
|
||||
| EXPIRED | 阶段结束或流水线结束 |
|
||||
| REVOKED | 被管理员强制撤回 |
|
||||
|
||||
## 7. 审计
|
||||
|
||||
| 字段 | 说明 |
|
||||
|------|------|
|
||||
| `event` | GRANTED / COMMAND_EXECUTED / EXPIRED / REVOKED |
|
||||
| `user` | 申请人 |
|
||||
| `pipeline_id` | 关联流水线 ID |
|
||||
| `timestamp` | 操作时间 |
|
||||
| `command` | 执行的包管理器命令 |
|
||||
| `exit_code` | 命令执行结果 |
|
||||
|
||||
**日志示例**：
|
||||
```json
|
||||
{
|
||||
"timestamp": "",
|
||||
"event": "",
|
||||
"pipeline_id": "",
|
||||
"user": ""
|
||||
}
|
||||
```
|
||||
|
||||
## 8. 参数
|
||||
|
||||
### 全局配置
|
||||
|
||||
```toml
|
||||
[security.baremetal_pkgman]
|
||||
enabled = true
|
||||
require_approval = true
|
||||
default_timeout = "30m"
|
||||
allowed_commands = ["apt-get", "pacman", "yum", "dnf", "zypper"]
|
||||
audit_log = true
|
||||
```
|
||||
|
||||
### 运行时参数
|
||||
|
||||
|
||||
|
||||
## 9. 建议
|
||||
|
||||
### 对用户
|
||||
|
||||
|
||||
|
||||
### 对管理员
|
||||
|
||||
|
||||
|
||||
## 10. 常见问题
|
||||
|
||||
|
||||
|
||||
## 98. 作者
|
||||
|
||||
|
||||
|
||||
## 99. 参见
|
||||
@@ -0,0 +1,187 @@
|
||||
# PIPELINE_CONTAINER_PRIVILEGED：容器环境特权模式
|
||||
|
||||
> [!WARNING]
|
||||
> 此权限允许容器以 privileged 模式运行，使容器内的进程拥有接近宿主机的权限。
|
||||
> 滥用可能导致容器逃逸、宿主机被入侵或影响其他容器。
|
||||
> **仅在绝对必要时使用，必须经过严格审批并记录完整审计。**
|
||||
|
||||
## 1. 概要
|
||||
|
||||
PIPELINE_CONTAINER_PRIVILEGED 是一个流水线隐参数，用于**在容器环境下授予流水线 privileged 权限**。当该参数生效时，容器将获得所有 capabilities，并可以访问宿主机的所有设备，能够执行需要特权的操作如挂载文件系统、加载内核模块等。
|
||||
|
||||
**此参数默认不开启，需要管理员通过严格流程审批并授权。**
|
||||
|
||||
## 2. 场景
|
||||
|
||||
| 适用场景 | 说明 |
|
||||
|----------|------|
|
||||
| Docker in Docker | 需要在容器内运行 Docker 命令构建镜像 |
|
||||
| FUSE 文件系统 | 需要在容器内挂载用户态文件系统 |
|
||||
| 内核模块测试 | 需要在容器内加载/测试内核模块 |
|
||||
| 系统调用调试 | 需要 ptrace 或其他特权系统调用 |
|
||||
| 性能分析工具 | 需要访问 perf、systemtap 等性能工具 |
|
||||
| 硬件设备访问 | 需要直接访问 GPU、FPGA 或其他硬件设备 |
|
||||
|
||||
| 不适用场景 | 说明 |
|
||||
|------------|------|
|
||||
| 普通软件包安装 | 请用容器镜像预装或 DepsUser 阶段 |
|
||||
| 构建普通应用 | 不需要特权模式 |
|
||||
| 网络抓包调试 | 请用容器网络配置而非特权模式 |
|
||||
|
||||
## 3. 描述
|
||||
|
||||
### 技术本质
|
||||
|
||||
PIPELINE_CONTAINER_PRIVILEGED 通过修改容器运行时配置实现提权：
|
||||
|
||||
1. 容器启动时添加 `--privileged` 标志
|
||||
2. 容器内的 root 用户拥有宿主机 root 的完整 capabilities
|
||||
3. 容器可以访问宿主机所有设备（`/dev/*`）
|
||||
4. 可以执行通常被容器运行时限制的操作（如 mount）
|
||||
5. 权限仅在该容器生命周期内有效，容器销毁后自动回收
|
||||
|
||||
### 与 CAP_ADD 的区别
|
||||
|
||||
| 模式 | 说明 | 适用场景 |
|
||||
|------|------|----------|
|
||||
| PRIVILEGED | 全部 capabilities + 设备访问 | 需要完整系统级访问 |
|
||||
| CAP_ADD | 添加特定 capabilities | 仅需个别特权操作 |
|
||||
|
||||
### 边界
|
||||
|
||||
| 允许的操作 | 不允许的操作 |
|
||||
|------------|--------------|
|
||||
| 挂载文件系统 | 修改宿主机关键配置 |
|
||||
| 运行 Docker 命令 | 关闭宿主机审计系统 |
|
||||
| 加载内核模块 | 删除其他容器数据 |
|
||||
| 访问宿主机设备 | 修改宿主机网络配置 |
|
||||
| 执行 ptrace 调试 | 安装持久化后门 |
|
||||
| 配置网络命名空间 | 覆盖宿主机二进制 |
|
||||
|
||||
### 与其他参数的关系
|
||||
|
||||
| 参数 | 关系 |
|
||||
|------|------|
|
||||
| PIPELINE_CONTAINER_CAP_ADD | 互补关系，PRIVILEGED 是更粗粒度的权限 |
|
||||
| PIPELINE_UNCOVER_SECRET | 可同时使用，需评估叠加风险 |
|
||||
| PIPELINE_BAREMETAL_ELEVATE | 适用于不同环境，无直接关联 |
|
||||
|
||||
## 4. 授权
|
||||
|
||||
| 方式 | 适用场景 | 操作示例 |
|
||||
|------|----------|----------|
|
||||
| 管理员 CLI | 临时授权 | |
|
||||
| 审批系统集成 | 企业流程 | |
|
||||
| 用户申请 | 常规需求 | |
|
||||
|
||||
**授权流程**：
|
||||
|
||||
1. 管理员或用户发起授权请求
|
||||
2. 系统验证请求者权限并记录理由
|
||||
3. 授权确认后，参数随容器启动下发给运行时
|
||||
4. 容器销毁后权限自动回收
|
||||
|
||||
**授权确认示例**：
|
||||
```
|
||||
⚠️ 正在授予 PIPELINE_CONTAINER_PRIVILEGED 权限
|
||||
|
||||
容器: pipeline-12345-container
|
||||
申请人: user@example.com
|
||||
理由: 需要在 CI 中构建 Docker 镜像并运行容器测试
|
||||
|
||||
此权限允许：
|
||||
- 容器以 privileged 模式运行
|
||||
- 可访问宿主机所有设备
|
||||
- 可执行 mount 等特权操作
|
||||
- 仅在该容器生命周期内有效
|
||||
|
||||
风险提示：
|
||||
- 可能造成容器逃逸
|
||||
- 可能影响宿主机稳定性
|
||||
- 所有操作将被强制审计
|
||||
|
||||
确认授予？(y/N)
|
||||
```
|
||||
|
||||
## 5. 撤回
|
||||
|
||||
| 撤回方式 | 说明 |
|
||||
|----------|------|
|
||||
| 手动撤回 | |
|
||||
| 自动失效 | 容器停止时权限自动回收 |
|
||||
| 流水线结束 | 流水线结束时强制销毁容器 |
|
||||
| 强制回收 | 审计系统检测到异常时可终止容器 |
|
||||
| 超时回收 | 超过配置的最大时长后自动终止容器 |
|
||||
|
||||
## 6. 生命周期
|
||||
|
||||
| 状态 | 说明 |
|
||||
|------|------|
|
||||
| PENDING | 等待审批 |
|
||||
| ACTIVE | 容器以特权模式运行中 |
|
||||
| EXPIRED | 容器已停止 |
|
||||
| REVOKED | 被管理员强制终止 |
|
||||
|
||||
## 7. 审计
|
||||
|
||||
所有特权容器内的关键操作都会被记录：
|
||||
|
||||
| 字段 | 说明 |
|
||||
|------|------|
|
||||
| `event` | GRANTED / CONTAINER_START / CONTAINER_STOP / COMMAND_EXECUTED |
|
||||
| `user` | 申请人 |
|
||||
| `pipeline_id` | 关联流水线 ID |
|
||||
| `container_id` | 容器 ID |
|
||||
| `timestamp` | 操作时间 |
|
||||
| `command` | 执行的命令 |
|
||||
| `mounts` | 挂载的操作 |
|
||||
| `devices_accessed` | 访问的设备 |
|
||||
|
||||
**日志示例**：
|
||||
```json
|
||||
{
|
||||
"timestamp": "",
|
||||
"event": "",
|
||||
"pipeline_id": "",
|
||||
"container_id": "",
|
||||
"user": ""
|
||||
}
|
||||
```
|
||||
|
||||
## 8. 参数
|
||||
|
||||
### 全局配置
|
||||
|
||||
```toml
|
||||
[security.container_privileged]
|
||||
enabled = false # 默认关闭
|
||||
require_approval = true # 需要审批
|
||||
max_duration = "60m" # 最大运行时间
|
||||
allowed_images = [] # 允许特权运行的镜像白名单
|
||||
audit_log = true # 强制审计
|
||||
notify_on_start = true # 启动时通知管理员
|
||||
```
|
||||
|
||||
### 运行时参数
|
||||
|
||||
|
||||
|
||||
## 9. 建议
|
||||
|
||||
### 对用户
|
||||
|
||||
|
||||
|
||||
### 对管理员
|
||||
|
||||
|
||||
|
||||
## 10. 常见问题
|
||||
|
||||
|
||||
|
||||
## 98. 作者
|
||||
|
||||
|
||||
|
||||
## 99. 参见
|
||||
@@ -0,0 +1,12 @@
|
||||
# 流水线隐参数
|
||||
|
||||
流水线隐参数是 server 在运行时根据权限、环境、策略动态生成的参数，
|
||||
随流水线定义一起下发给执行组件。
|
||||
|
||||
常见隐参数：
|
||||
- `PIPELINE_BAREMETAL_PKGMAN`: 允许裸机环境使用包管理器（需 root）
|
||||
- `PIPELINE_CONTAINER_PRIVILEGED`: 允许容器以 privileged 模式运行
|
||||
- `PIPELINE_UNCOVER_SECRET`: 允许输出 secret 明文
|
||||
|
||||
这些参数不在 prebake.yml 中配置，由 server 管理，
|
||||
并随流水线生命周期自动生效和废弃。
|
||||
@@ -0,0 +1,198 @@
|
||||
# PIPELINE_UNCOVER_SECRET：临时暴露敏感信息
|
||||
|
||||
> [!WARNING]
|
||||
> 此权限允许流水线临时输出所有 secret 的明文，包括环境变量、配置文件、构建日志中的敏感信息。
|
||||
> 滥用可能导致凭据泄露、数据泄露或安全入侵。
|
||||
> **非必要不使用，使用时必须经过 2FA 验证并记录审计。**
|
||||
|
||||
## 1. 概要
|
||||
|
||||
PIPELINE_UNCOVER_SECRET 是一个流水线隐参数，用于**临时解除系统对敏感信息的默认保护**。当该参数生效时，原本会被遮蔽的 secret 值将以明文形式出现在日志、标准输出和构建产物中，便于调试、审计或教学。
|
||||
|
||||
**此参数默认不开启，需要用户通过 2FA 显式申请并获得临时授权。**
|
||||
|
||||
## 2. 场景
|
||||
|
||||
| 适用场景 | 说明 |
|
||||
|----------|------|
|
||||
| 调试构建失败 | 怀疑 secret 传递错误，需要确认环境变量实际值 |
|
||||
| 合规性审计 | 需要验证 secret 是否正确配置和使用 |
|
||||
| 教学演示 | 展示 CI/CD 流程中 secret 的完整生命周期 |
|
||||
| 遗留系统兼容 | 某些老旧脚本硬编码依赖 echo secret 的方式工作 |
|
||||
|
||||
|
||||
|
||||
| 不适用场景 | 说明 |
|
||||
|------------|------|
|
||||
| 日常构建 | 正常流水线不应暴露 secret |
|
||||
| 生产环境调试 | 请先在预发环境复现问题 |
|
||||
| 权限测试 | 请用专门的测试账号 |
|
||||
|
||||
## 3. 描述
|
||||
|
||||
### 技术本质
|
||||
|
||||
PIPELINE_UNCOVER_SECRET 通过以下机制实现：
|
||||
|
||||
1. 在流水线启动时，系统生成一个临时令牌并注入 baker 环境
|
||||
2. baker 在运行期间检测到该令牌，**暂停所有输出遮蔽逻辑**
|
||||
3. 所有敏感变量（包括环境变量、文件内容、构建日志）都以原始形式输出
|
||||
4. 权限到期后，系统自动恢复遮蔽保护
|
||||
|
||||
### 边界
|
||||
|
||||
| 允许的操作 | 不允许的操作 |
|
||||
|------------|--------------|
|
||||
| 查看 secret 明文 | 将 secret 持久化到外部系统 |
|
||||
| 输出 secret 到日志 | 关闭审计日志 |
|
||||
| 调试脚本中的 secret 使用 | 延长权限有效期 |
|
||||
|
||||
### 依赖关系
|
||||
|
||||
- 需要用户已配置 **2FA**
|
||||
- 需要系统启用 `security.uncover_secret.enabled`（默认开启）
|
||||
- 与其他隐参数（如 BAREMETAL_ELEVATE）互斥使用时需注意叠加风险
|
||||
|
||||
## 4. 授权
|
||||
|
||||
| 方式 | 适用场景 | 操作示例 |
|
||||
|------|----------|----------|
|
||||
| 用户自申请 | 调试/审计 | `hbw pipeline run --uncover-secret` |
|
||||
| 紧急授权 | 生产问题 | 需 2FA + 管理员二次确认 |
|
||||
|
||||
**授权流程**：
|
||||
|
||||
1. 用户发起请求时附加 `--uncover-secret` 标志
|
||||
2. 系统要求输入 2FA 验证码
|
||||
3. 验证通过后显示警告并要求确认
|
||||
4. 用户确认后，系统生成有效期 5 分钟的临时令牌
|
||||
|
||||
**授权确认示例**：
|
||||
```
|
||||
Request for PIPELINE_UNCOVER_SECRET
|
||||
2FA Code: ******
|
||||
|
||||
WARNING: You are about to expose ALL secrets in this pipeline.
|
||||
Sensitive data may appear in logs, outputs, and artifacts.
|
||||
This operation is audited. Continue? (y/N)
|
||||
```
|
||||
|
||||
## 5. 撤回
|
||||
|
||||
| 撤回方式 | 说明 |
|
||||
|----------|------|
|
||||
| 手动撤回 | 管理员可执行 `hbw admin revoke-uncover pipeline-12345` 强制终止 |
|
||||
| 自动失效 | 5 分钟有效期到达后自动恢复遮蔽 |
|
||||
| 流水线结束 | 无论是否到期，流水线结束时权限自动回收 |
|
||||
| 强制回收 | 审计系统检测到异常行为时可自动终止流水线并回收权限 |
|
||||
|
||||
## 6. 生命周期
|
||||
|
||||
```
|
||||
申请 → 2FA验证 → 确认 → 授权 → 使用 → 失效
|
||||
↑ ↑ ↑ ↑ ↑ ↑
|
||||
用户 系统 用户 系统 流水线 时间/事件
|
||||
```
|
||||
|
||||
| 状态 | 说明 |
|
||||
|------|------|
|
||||
| PENDING | 等待 2FA 验证 |
|
||||
| ACTIVE | 已授权，secret 可明文输出 |
|
||||
| EXPIRED | 超过 5 分钟有效期 |
|
||||
| REVOKED | 被管理员或系统强制撤回 |
|
||||
|
||||
## 7. 审计
|
||||
|
||||
所有涉及 PIPELINE_UNCOVER_SECRET 的操作都会被详细记录：
|
||||
|
||||
| 字段 | 说明 |
|
||||
|------|------|
|
||||
| `event` | GRANTED / USED / EXPIRED / REVOKED |
|
||||
| `user` | 操作人邮箱 |
|
||||
| `pipeline_id` | 关联流水线 ID |
|
||||
| `timestamp` | 操作时间 |
|
||||
| `auth_method` | 2FA |
|
||||
| `expires_at` | 授权到期时间 |
|
||||
| `accessed_secrets` | 授权期间访问过的 secret 列表 |
|
||||
|
||||
**日志示例**：
|
||||
```json
|
||||
{
|
||||
"timestamp": "2026-03-19T10:23:45Z",
|
||||
"event": "PIPELINE_UNCOVER_SECRET_GRANTED",
|
||||
"user": "developer@example.com",
|
||||
"pipeline_id": "pl-12345",
|
||||
"auth_method": "2FA",
|
||||
"expires_at": "2026-03-19T10:28:45Z"
|
||||
}
|
||||
|
||||
{
|
||||
"timestamp": "2026-03-19T10:25:12Z",
|
||||
"event": "SECRET_ACCESSED",
|
||||
"pipeline_id": "pl-12345",
|
||||
"variable": "DOCKER_PASSWORD",
|
||||
"access_type": "ENV_VAR_READ"
|
||||
}
|
||||
```
|
||||
|
||||
管理员可通过审计日志追溯：
|
||||
- 谁在什么时候申请了该权限
|
||||
- 授权给了哪个流水线
|
||||
- 哪些 secret 在授权期间被访问
|
||||
- 权限是否在有效期内
|
||||
|
||||
## 8. 参数
|
||||
|
||||
### 全局配置 (bakerd.toml)
|
||||
|
||||
```toml
|
||||
[security.uncover_secret]
|
||||
enabled = true # 是否允许使用该功能
|
||||
default_timeout = "5m" # 默认授权时长
|
||||
require_2fa = true # 是否强制 2FA
|
||||
audit_log = true # 是否记录审计日志
|
||||
max_concurrent_per_user = 1 # 每个用户同时最多可授权几个流水线
|
||||
```
|
||||
|
||||
### 运行时参数
|
||||
|
||||
```bash
|
||||
# 申请权限
|
||||
hbw pipeline run --uncover-secret
|
||||
|
||||
# 指定超时（需管理员权限）
|
||||
hbw pipeline run --uncover-secret --uncover-timeout 10m
|
||||
|
||||
# 带理由（便于审计）
|
||||
hbw pipeline run --uncover-secret --reason "调试数据库连接失败"
|
||||
```
|
||||
|
||||
## 9. 建议
|
||||
|
||||
### 对用户
|
||||
|
||||
- **非必要不使用**：能用常规调试手段解决的问题，不要依赖暴露 secret
|
||||
- **用完后立即结束流水线**：避免权限窗口被意外延长
|
||||
- **检查日志**：使用结束后检查是否有 secret 被意外记录到持久化存储中
|
||||
- **轮换凭据**：如果确认 secret 在授权期间被暴露，建议立即轮换相关凭据
|
||||
|
||||
### 对管理员
|
||||
|
||||
- **定期审计**：每周检查 UNCOVER_SECRET 使用记录，发现异常及时处理
|
||||
- **设置合理超时**：根据团队需求调整默认超时时间（建议不超过 15 分钟）
|
||||
- **培训用户**：确保团队成员理解该权限的风险和使用场景
|
||||
- **与审批系统集成**：对高风险操作增加二次审批流程
|
||||
|
||||
## 10. 常见问题
|
||||
|
||||
|
||||
|
||||
## 98. 作者
|
||||
|
||||
```yaml
|
||||
---
|
||||
authors:
|
||||
---
|
||||
```
|
||||
|
||||
## 99. 参见
|
||||
@@ -0,0 +1 @@
|
||||
what
|
||||
@@ -0,0 +1,192 @@
|
||||
# prebake.yaml.tmpl
|
||||
#
|
||||
# 术语说明：
|
||||
# 可选：该字段可以不给出
|
||||
# 默认为...：隐含“可选”
|
||||
# 留空：尚未定义，留作后续
|
||||
|
||||
# 版本号，用于格式兼容性检查
|
||||
version: "1.0"
|
||||
|
||||
# 构建环境定义
|
||||
environment:
|
||||
# 构建机类型：docker | firecracker | custom | baremetal
|
||||
builder: "docker"
|
||||
|
||||
# 构建机配置，与builder一致的被启用。
|
||||
baremetal: {}
|
||||
|
||||
docker:
|
||||
# 镜像名，与dockerfile互斥
|
||||
image: "rust:1.70-slim"
|
||||
|
||||
# Dockerfile路径，相对于项目根目录，与image互斥
|
||||
dockerfile: "/Dockerfile"
|
||||
# Docker构建参数，可选
|
||||
build_args:
|
||||
RUST_VERSION: "1.70"
|
||||
|
||||
firecracker: {}
|
||||
# TODO: 定义firecracker结构
|
||||
|
||||
custom:
|
||||
# 自定义构建机名称
|
||||
name: "my-custom-builder"
|
||||
# 设置脚本，相对于项目根目录
|
||||
setup_script: "setup-builder.sh"
|
||||
# 清洁脚本，相对于项目根目录
|
||||
cleanup_script: "cleanup-builder.sh"
|
||||
|
||||
# 硬件资源要求，满足最小要求的构建机可以被选中
|
||||
resources:
|
||||
cpu: 2 # CPU核心数，默认为1
|
||||
memory: "2G" # 内存大小，也可以写作2048MB,默认为"1G"
|
||||
disk: "10G" # 磁盘空间，也可以写作10240MB,默认为"1G"
|
||||
architecture: "x86_64" # 架构要求，只能是"noarch"或{"x86_64"/"amd64", "arm64"/"aarch64", "riscv64", "loongarch64"/"loong64"}中的一个或几个
|
||||
# TODO: 重新设计GPU字段
|
||||
# gpu: false # 是否需要GPU，默认为false
|
||||
# gpu_type: "cuda" # GPU类型（如果需要），可以是"cuda", "rocm", "vulkan", "oneapi"中的一个或几个
|
||||
tags: # 自定义标签，只有满足标签的构建机可以被选中
|
||||
- "custom_tag"
|
||||
|
||||
# 网络配置
|
||||
network:
|
||||
enabled: true # 启用网络，对baremetal不适用，默认为true
|
||||
outbound: true # 允许出站连接，对baremetal不适用，默认为true
|
||||
dns_servers: # 指定构建机的DNS服务器，对baremetal不适用，custom应该自行处理DNS问题，可选
|
||||
- "8.8.8.8"
|
||||
- "1.1.1.1"
|
||||
proxies: # 代理配置，暂时留空，可选
|
||||
|
||||
# 依赖定义，安装依赖时按system-user顺序进行
|
||||
dependencies:
|
||||
# 系统包依赖，按包管理器枚举给出，只应该给出与环境契合的字段。
|
||||
# 根据security.drop-after的配置，在此阶段一般具有root或免密root权限
|
||||
# 不适用于裸机
|
||||
system:
|
||||
apt:
|
||||
packages: # 安装的软件包
|
||||
- "git"
|
||||
- "curl"
|
||||
- "build-essential"
|
||||
- "pkg-config"
|
||||
- "libssl-dev"
|
||||
repositories: # 该字段按serde Value原样传递给插件，config.rs不做解析
|
||||
- url: "ppa:custom/ppa" # 自定义PPA（Ubuntu）
|
||||
- url: "https://download.docker.com/linux/ubuntu" # DEB822
|
||||
key: "https://download.docker.com/linux/ubuntu/gpg"
|
||||
mirror: "https://mirrors.tuna.tsinghua.edu.cn" # 指定软件包镜像源，可选
|
||||
pacman:
|
||||
packages:
|
||||
- "git"
|
||||
- "curl"
|
||||
- "base-devel"
|
||||
- "pkg-config"
|
||||
- "openssl"
|
||||
repositories:
|
||||
- server: "https://mirrors.tuna.tsinghua.edu.cn/arch4edu/$arch" # 自定义仓库（Arch）
|
||||
name: "arch4edu"
|
||||
keypackage: "arch4edu-keyring"
|
||||
mirror: "https://mirrors.tuna.tsinghua.edu.cn" # 指定软件包镜像源，可选
|
||||
dnf:
|
||||
# 略
|
||||
|
||||
# 用户依赖
|
||||
# 根据security.drop-after的配置，在此阶段一般不具有root权限
|
||||
# 例外：语言依赖会在system中产生隐式依赖
|
||||
user:
|
||||
# 以下字段按serde Value原样传递给插件，config.rs不做解析
|
||||
rust:
|
||||
type: "rustup" # 此时会向系统引入隐式的rustup依赖
|
||||
toolchain: "nightly-x86_64-unknown-linux-gnu"
|
||||
manifest: "Cargo.toml"
|
||||
lock: "Cargo.lock"
|
||||
python:
|
||||
type: "uv"
|
||||
python: "python3.14"
|
||||
index: "http://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
|
||||
directory: ".venv"
|
||||
env: # 优先，高级选项
|
||||
UV_PYTHON: "python3.14"
|
||||
UV_INDEX: "http://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
|
||||
UV_DIRECTORY: ".venv"
|
||||
manifest: "requirements.txt" # 或pyproject.toml
|
||||
nix: {}
|
||||
# TBD: 没用过lol
|
||||
# PR welcome
|
||||
custom: # 绝对的自定义，与上述全部配置互斥（custom独占）
|
||||
- command: "cargo fetch"
|
||||
working_dir: "/app"
|
||||
|
||||
# 环境变量配置
|
||||
envvars:
|
||||
# 构建环境变量，只在 prebake 阶段有效
|
||||
# 用于控制 prebake 阶段的行为（apt/cargo/rustc 等）
|
||||
prebake:
|
||||
RUST_BACKTRACE: "full" # Rust 调试
|
||||
CARGO_NET_GIT_FETCH_WITH_CLI: "true" # Cargo 配置
|
||||
NODE_ENV: "production" # Node 环境
|
||||
|
||||
# 运行时环境变量（会传递给 bake 阶段）
|
||||
# 这些变量会与项目中的 .env 文件合并后，作为最终环境
|
||||
bake:
|
||||
# 支持 secret 引用，格式: {{ secret.<key> }}
|
||||
DATABASE_URL: "postgresql://user:{{ secret.dbpassword }}@localhost/db"
|
||||
LOG_LEVEL: "info"
|
||||
|
||||
# 特殊变量：指定项目中 .env 文件的位置
|
||||
# 系统会读取该文件，并将其中的变量与上面定义的变量合并
|
||||
# 合并规则：prebake.yml 中定义的变量优先级更高，会覆盖 .env 中的同名变量
|
||||
# 此变量本身不会传递给 bake 阶段
|
||||
# .env不应该包含任何 secret ！
|
||||
__ENV_FILE: ".env"
|
||||
|
||||
# 注意：
|
||||
# - secret 只存在于内存中，不会写入磁盘
|
||||
# - 日志中会自动遮蔽 secret 的值
|
||||
# - 如需临时查看 secret，可使用 PIPELINE_UNCOVER_SECRET 权限（需 2FA）
|
||||
|
||||
|
||||
# 缓存配置
|
||||
cache:
|
||||
# 缓存目录
|
||||
directory:
|
||||
- path: "/usr/local/cargo"
|
||||
strategy: "always" # 在"always","readonly","clean"和"never"中选择一个，可以被流水线启动时配置覆盖，默认为"always"
|
||||
mode: "zstd" # 在"gzip","zstd","none"和"dir"中选择一个
|
||||
|
||||
- path: "/root/.npm"
|
||||
strategy: "clean"
|
||||
mode: "none"
|
||||
|
||||
- path: "/app/target"
|
||||
strategy: "always"
|
||||
mode: "dir"
|
||||
|
||||
# 缓存策略
|
||||
strategy:
|
||||
ttl_days: 30 # 缓存生存时间
|
||||
max_size_gb: 20 # 最大缓存大小
|
||||
cleanup_policy: "lru" # 清理策略：lru, fifo, size
|
||||
|
||||
# 安全配置
|
||||
security:
|
||||
#
|
||||
drop-after: ""
|
||||
# 留空
|
||||
|
||||
# 钩子脚本
|
||||
hooks:
|
||||
# 依赖安装前
|
||||
early:
|
||||
- command: "echo 'Starting dependency installation'"
|
||||
|
||||
# 依赖安装后
|
||||
late:
|
||||
- command: "cargo fetch"
|
||||
working_dir: "/app"
|
||||
|
||||
# 元数据
|
||||
metadata:
|
||||
description: "Rust project build environment"
|
||||
maintainer: "team@example.com"
|
||||
@@ -2015,6 +2015,16 @@ dependencies = [
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "privdrop"
|
||||
version = "0.5.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "70722a5a3728c9603c8d9469b64b8d1ee54dae6d74e24146da7f501b4c76540f"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"nix 0.30.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro-crate"
|
||||
version = "3.4.0"
|
||||
@@ -3389,6 +3399,7 @@ dependencies = [
|
||||
"log",
|
||||
"minijinja",
|
||||
"os_info",
|
||||
"privdrop",
|
||||
"regex",
|
||||
"semver",
|
||||
"serde",
|
||||
@@ -3,6 +3,22 @@ name = "workshop-executor"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
|
||||
[lib]
|
||||
name = "workshop_executor"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "workshop-executor"
|
||||
path = "src/main.rs"
|
||||
|
||||
[features]
|
||||
default = []
|
||||
integration-tests = []
|
||||
|
||||
[[test]]
|
||||
name = "engine_integration"
|
||||
path = "tests/engine_integration.rs"
|
||||
|
||||
[dependencies]
|
||||
anyhow = "1.0.100"
|
||||
bollard = { version = "0.20.0", features = ["buildkit", "chrono", "ssh"] }
|
||||
@@ -30,3 +46,4 @@ async-trait = "0.1.87"
|
||||
uuid = { version = "1.19.0", features = ["v4"] }
|
||||
libc = "0.2"
|
||||
os_info = "3.14.0"
|
||||
privdrop = "0.5.6"
|
||||
@@ -0,0 +1,194 @@
|
||||
use crate::cli::{Cli, TaskSpec};
|
||||
use config::Config;
|
||||
use serde::Serialize;
|
||||
use serde_json::Value;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use tokio::io::{AsyncBufReadExt, BufReader};
|
||||
use tokio::net::{UnixListener, UnixStream};
|
||||
use tokio::process::{Child, Command};
|
||||
use tokio::signal;
|
||||
use tokio::sync::RwLock;
|
||||
use tokio::time::{Duration, interval};
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
struct Heartbeat {
|
||||
pipeline_id: String,
|
||||
build_id: String,
|
||||
stage: String,
|
||||
timestamp: u64,
|
||||
status: String,
|
||||
subtasks: Vec<String>,
|
||||
}
|
||||
|
||||
pub async fn daemon(cli: &Cli) {
|
||||
todo!("Daemon should be refactored");
|
||||
// let socket = PathBuf::from(settings.get_string("socket").unwrap());
|
||||
// let agentsocket = settings.get_string("agentsocket").unwrap();
|
||||
// let _ = std::fs::remove_file(&socket);
|
||||
|
||||
// let heartbeat = Arc::new(RwLock::new(Heartbeat {
|
||||
// pipeline_id: cli.pipeline.clone(),
|
||||
// build_id: cli.build_id.clone(),
|
||||
// stage: String::new(),
|
||||
// timestamp: 0,
|
||||
// status: String::new(),
|
||||
// subtasks: Vec::new(),
|
||||
// }));
|
||||
|
||||
// // let heartbeat_clone = heartbeat.clone();
|
||||
// let heartbeat_task = {
|
||||
// let hb = heartbeat.clone();
|
||||
// tokio::spawn(async move {
|
||||
// let mut tick = interval(Duration::from_secs(1));
|
||||
// loop {
|
||||
// tick.tick().await;
|
||||
// let mut hb = hb.write().await;
|
||||
// hb.timestamp = std::time::SystemTime::now()
|
||||
// .duration_since(std::time::UNIX_EPOCH)
|
||||
// .unwrap()
|
||||
// .as_millis() as u64;
|
||||
|
||||
// // 只输出到stdout（模拟未来发送给Agent）
|
||||
// println!("[HEARTBEAT] {}", serde_json::to_string(&*hb).unwrap());
|
||||
// }
|
||||
// })
|
||||
// };
|
||||
|
||||
// let socket_task = {
|
||||
// let socket = socket.clone();
|
||||
// tokio::spawn(async move {
|
||||
// log::info!("Creating Unix socket at {}", socket.display());
|
||||
// let listener =
|
||||
// UnixListener::bind(&socket).expect("Failed to bind Unix socket");
|
||||
|
||||
// loop {
|
||||
// match listener.accept().await {
|
||||
// Ok((stream, _)) => {
|
||||
// tokio::spawn(handle_connection(stream));
|
||||
// }
|
||||
// Err(e) => {
|
||||
// log::error!("Accept error: {}", e);
|
||||
// break;
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// })
|
||||
// };
|
||||
|
||||
// let mut tasks: Vec<TaskSpec> = Vec::new();
|
||||
|
||||
// let execution_task = {
|
||||
// let hb = heartbeat.clone();
|
||||
// tokio::spawn(async move {
|
||||
// let mut hb = hb.write().await;
|
||||
// let mut children: Vec<(String, Child)> = vec![];
|
||||
|
||||
// let executable = std::env::current_exe().unwrap();
|
||||
|
||||
// for (index, task) in tasks.iter().enumerate() {
|
||||
// let mut command = task.command(&executable);
|
||||
// command.stdout(std::process::Stdio::piped());
|
||||
// command.stderr(std::process::Stdio::piped());
|
||||
// let child = command
|
||||
// .spawn()
|
||||
// .expect(format!("Failed to spawn child {}", index).as_str());
|
||||
|
||||
// let pid = child.id().unwrap_or(0).to_string();
|
||||
// children.push((pid.clone(), child));
|
||||
|
||||
// hb.subtasks.push(pid.clone());
|
||||
// }
|
||||
|
||||
// for (pid, mut child) in children {
|
||||
// let status = child.wait().await.expect("Failed to wait child");
|
||||
// log::info!("Subtask {} exited with: {}", pid, status);
|
||||
|
||||
// hb.subtasks.retain(|p| p != &pid);
|
||||
// }
|
||||
|
||||
// hb.stage = "Success".to_string();
|
||||
// })
|
||||
// };
|
||||
|
||||
// tokio::select! {
|
||||
// _ = heartbeat_task => {
|
||||
// log::error!("Heartbeat task died");
|
||||
// }
|
||||
// _ = execution_task => {
|
||||
// log::info!("All subtasks completed");
|
||||
// }
|
||||
// _ = socket_task => {
|
||||
// log::error!("Socket task died");
|
||||
// }
|
||||
// _ = signal::ctrl_c() => {
|
||||
// log::info!("Shutting down...");
|
||||
// }
|
||||
// }
|
||||
// for pid in heartbeat.read().await.subtasks.clone() {
|
||||
// let _ = Command::new("kill").arg("-TERM").arg(&pid).status().await;
|
||||
// }
|
||||
|
||||
// let _ = std::fs::remove_file(&socket);
|
||||
}
|
||||
|
||||
async fn handle_connection(stream: UnixStream) {
|
||||
let peer_addr = match stream.peer_addr() {
|
||||
Ok(addr) => format!("{:?}", addr),
|
||||
Err(_) => "unknown".to_string(),
|
||||
};
|
||||
|
||||
log::debug!("New connection from: {}", peer_addr);
|
||||
|
||||
let mut reader = BufReader::new(stream);
|
||||
let mut line = String::new();
|
||||
|
||||
loop {
|
||||
line.clear();
|
||||
match reader.read_line(&mut line).await {
|
||||
Ok(0) => {
|
||||
log::debug!("Connection closed by peer: {}", peer_addr);
|
||||
break;
|
||||
}
|
||||
Ok(_) => {
|
||||
// 简单打印原始JSON（预留字段解析空间）
|
||||
match serde_json::from_str::<Value>(&line) {
|
||||
Ok(json) => {
|
||||
// 只提取关键字段打印（字段可能变化）
|
||||
let msgtype = json
|
||||
.get("msgtype")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("unknown");
|
||||
let name = json
|
||||
.get("name")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("unnamed");
|
||||
|
||||
log::info!("[{}] Message from '{}': {}", msgtype, name, line.trim());
|
||||
|
||||
// TODO: 字段稳定后，可扩展为结构化打印
|
||||
// match msgtype {
|
||||
// "ResourceUsage" => print_resource(&json),
|
||||
// "Log" => print_log(&json),
|
||||
// _ => log::debug!("Unknown type: {}", msgtype),
|
||||
// }
|
||||
}
|
||||
Err(e) => {
|
||||
log::warn!(
|
||||
"Invalid JSON from {}: {} - Error: {}",
|
||||
peer_addr,
|
||||
line.trim(),
|
||||
e
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
log::error!("Failed to read from {}: {}", peer_addr, e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log::debug!("Connection handler for {} exiting", peer_addr);
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
use regex::Regex;
|
||||
use lazy_static::lazy_static;
|
||||
use regex::Regex;
|
||||
|
||||
lazy_static! {
|
||||
// 解析装饰器的正则表达式
|
||||
@@ -23,13 +23,12 @@ pub enum Decorator {
|
||||
Export,
|
||||
}
|
||||
|
||||
pub fn parse_decorator(line: &str) -> Option<Decorator>
|
||||
{
|
||||
pub fn parse_decorator(line: &str) -> Option<Decorator> {
|
||||
DECORATOR_REGEX.captures(line).and_then(|caps| {
|
||||
let name = caps.get(1)?.as_str();
|
||||
let args = match caps.get(3) {
|
||||
Some(value) => value.as_str(),
|
||||
None => ""
|
||||
None => "",
|
||||
};
|
||||
// dbg!(name, args);
|
||||
let decorator = match name {
|
||||
@@ -41,14 +40,16 @@ pub fn parse_decorator(line: &str) -> Option<Decorator>
|
||||
"timeout" => Decorator::Timeout(args.parse().unwrap()),
|
||||
"retry" => {
|
||||
let parts: Vec<&str> = args.split(',').collect();
|
||||
Decorator::Retry(parts[0].trim().parse().unwrap(), parts[1].trim().parse().unwrap())
|
||||
Decorator::Retry(
|
||||
parts[0].trim().parse().unwrap(),
|
||||
parts[1].trim().parse().unwrap(),
|
||||
)
|
||||
}
|
||||
"export" => Decorator::Export,
|
||||
_ => return None,
|
||||
};
|
||||
Some(decorator)
|
||||
}
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -57,14 +58,29 @@ mod tests{
|
||||
|
||||
#[test]
|
||||
fn test_parse_decorator() {
|
||||
assert_eq!(parse_decorator("# @if(true)"), Some(Decorator::If("true".to_string())));
|
||||
assert_eq!(
|
||||
parse_decorator("# @if(true)"),
|
||||
Some(Decorator::If("true".to_string()))
|
||||
);
|
||||
assert_eq!(parse_decorator("# @pipeline"), Some(Decorator::Pipeline));
|
||||
assert_eq!(parse_decorator("# @fallible"), Some(Decorator::Fallible));
|
||||
assert_eq!(parse_decorator("# @loop (10)"), Some(Decorator::Loop(10)));
|
||||
assert_eq!(parse_decorator("# @after (build)"), Some(Decorator::After("build".to_string())));
|
||||
assert_eq!(parse_decorator("# @timeout(5)"), Some(Decorator::Timeout(5)));
|
||||
assert_eq!(parse_decorator("# @retry (3, 2)"), Some(Decorator::Retry(3, 2)));
|
||||
assert_eq!(parse_decorator("# @export # comment"), Some(Decorator::Export));
|
||||
assert_eq!(
|
||||
parse_decorator("# @after (build)"),
|
||||
Some(Decorator::After("build".to_string()))
|
||||
);
|
||||
assert_eq!(
|
||||
parse_decorator("# @timeout(5)"),
|
||||
Some(Decorator::Timeout(5))
|
||||
);
|
||||
assert_eq!(
|
||||
parse_decorator("# @retry (3, 2)"),
|
||||
Some(Decorator::Retry(3, 2))
|
||||
);
|
||||
assert_eq!(
|
||||
parse_decorator("# @export # comment"),
|
||||
Some(Decorator::Export)
|
||||
);
|
||||
assert_eq!(parse_decorator("# fallible"), None);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
//! Engine module - execution engine for workshop-baker
|
||||
//!
|
||||
//! This module provides the core execution functionality for running commands
|
||||
//! in isolated environments with resource management.
|
||||
|
||||
pub mod cgroups;
|
||||
pub mod executor;
|
||||
pub mod pm;
|
||||
pub mod types;
|
||||
|
||||
// Re-export commonly used types for convenience
|
||||
pub use types::{
|
||||
Engine, EventSender, ExecutionContext, ExecutionError, ExecutionEvent, ExecutionResult,
|
||||
ResourceLimits, ResourceUsage, StreamType,
|
||||
};
|
||||
@@ -0,0 +1,470 @@
|
||||
use chrono::Utc;
|
||||
use std::path::PathBuf;
|
||||
use std::process::Stdio;
|
||||
use tokio::process::Command;
|
||||
|
||||
use super::pm::PackageManager;
|
||||
use super::types::{
|
||||
CgroupManager, EventSender, ExecutionContext, ExecutionError, ExecutionEvent, ExecutionResult,
|
||||
ResourceLimits,
|
||||
};
|
||||
|
||||
impl super::types::Engine {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
cgroup_manager: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_cgroup(limits: ResourceLimits, build_id: &str) -> Result<Self, String> {
|
||||
let cgroup_manager =
|
||||
CgroupManager::new(build_id).map_err(|e| format!("Failed to create cgroup: {}", e))?;
|
||||
|
||||
if let Some(mem) = limits.memory_bytes {
|
||||
cgroup_manager
|
||||
.set_memory_limit(mem)
|
||||
.map_err(|e| format!("Failed to set memory limit: {}", e))?;
|
||||
}
|
||||
|
||||
if let Some(weight) = limits.cpu_weight {
|
||||
cgroup_manager
|
||||
.set_cpu_weight(weight)
|
||||
.map_err(|e| format!("Failed to set cpu weight: {}", e))?;
|
||||
}
|
||||
|
||||
if let Some(max) = limits.cpu_max_us {
|
||||
cgroup_manager
|
||||
.set_cpu_max(max, limits.cpu_period_us)
|
||||
.map_err(|e| format!("Failed to set cpu max: {}", e))?;
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
cgroup_manager: Some(cgroup_manager),
|
||||
})
|
||||
}
|
||||
|
||||
fn apply_cgroup(&self, pid: u32) -> Result<(), ExecutionError> {
|
||||
if let Some(ref cgroup) = self.cgroup_manager {
|
||||
cgroup.add_process(pid).map_err(|e| {
|
||||
ExecutionError::IoError(format!("Failed to add process to cgroup: {}", e))
|
||||
})?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn execute_script(
|
||||
&self,
|
||||
script_path: &PathBuf,
|
||||
ctx: &ExecutionContext,
|
||||
event_tx: &EventSender,
|
||||
) -> Result<ExecutionResult, ExecutionError> {
|
||||
let abs_path = if script_path.is_absolute() {
|
||||
script_path.clone()
|
||||
} else {
|
||||
ctx.working_dir.join(script_path)
|
||||
};
|
||||
|
||||
let mut command = Command::new(ctx.shell.as_str());
|
||||
command.arg("-c");
|
||||
command.arg(abs_path);
|
||||
self.execute(command, ctx, event_tx).await
|
||||
}
|
||||
|
||||
pub async fn execute_command(
|
||||
&self,
|
||||
cmd: &str,
|
||||
ctx: &ExecutionContext,
|
||||
event_tx: &EventSender,
|
||||
) -> Result<ExecutionResult, ExecutionError> {
|
||||
let mut command = Command::new(ctx.shell.as_str());
|
||||
command.arg("-c");
|
||||
command.arg(cmd);
|
||||
self.execute(command, ctx, event_tx).await
|
||||
}
|
||||
|
||||
fn construct_command(&self, command: &mut Command, ctx: &ExecutionContext) {
|
||||
command
|
||||
.current_dir(&ctx.working_dir)
|
||||
.envs(&ctx.env_vars)
|
||||
.env("WS_TASKID", &ctx.task_id)
|
||||
.env("WS_PIPELINE", &ctx.pipeline_name)
|
||||
.env("WS_USERNAME", &ctx.username)
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped());
|
||||
}
|
||||
|
||||
pub async fn execute(
|
||||
&self,
|
||||
command: Command,
|
||||
ctx: &ExecutionContext,
|
||||
event_tx: &EventSender,
|
||||
) -> Result<ExecutionResult, ExecutionError> {
|
||||
let start_time = std::time::Instant::now();
|
||||
|
||||
if let Some(tx) = event_tx {
|
||||
let _ = tx
|
||||
.send(ExecutionEvent::TaskStarted {
|
||||
task_id: ctx.task_id.clone(),
|
||||
timestamp: Utc::now(),
|
||||
})
|
||||
.await;
|
||||
}
|
||||
let mut command = command;
|
||||
|
||||
self.construct_command(&mut command, ctx);
|
||||
|
||||
let child = command.spawn().map_err(|e| {
|
||||
ExecutionError::InvalidCommand(format!("Failed to spawn command: {}", e))
|
||||
})?;
|
||||
|
||||
let pid = child
|
||||
.id()
|
||||
.ok_or_else(|| ExecutionError::IoError("Failed to get PID".to_string()))?;
|
||||
self.apply_cgroup(pid)?;
|
||||
|
||||
let output = child
|
||||
.wait_with_output()
|
||||
.await
|
||||
.map_err(|e| ExecutionError::IoError(format!("Failed to wait for process: {}", e)))?;
|
||||
|
||||
let exit_code = output.status.code().unwrap_or(-1);
|
||||
let duration = start_time.elapsed();
|
||||
|
||||
// TODO: Timeout adaptation
|
||||
let result = ExecutionResult {
|
||||
task_id: ctx.task_id.clone(),
|
||||
exit_code,
|
||||
success: exit_code == 0,
|
||||
duration,
|
||||
stdout: String::from_utf8_lossy(&output.stdout).to_string(),
|
||||
stderr: String::from_utf8_lossy(&output.stderr).to_string(),
|
||||
resource_usage: None,
|
||||
};
|
||||
|
||||
if let Some(tx) = event_tx {
|
||||
let event = if result.success {
|
||||
ExecutionEvent::TaskCompleted {
|
||||
task_id: ctx.task_id.clone(),
|
||||
result: result.clone(),
|
||||
}
|
||||
} else {
|
||||
ExecutionEvent::TaskFailed {
|
||||
task_id: ctx.task_id.clone(),
|
||||
error: format!("Exit code: {}", exit_code),
|
||||
}
|
||||
};
|
||||
let _ = tx.send(event).await;
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub async fn install_dependency(
|
||||
&self,
|
||||
package: &[&str],
|
||||
pkgman: &dyn PackageManager,
|
||||
ctx: &ExecutionContext,
|
||||
) -> Result<ExecutionResult, ExecutionError> {
|
||||
let cmd = pkgman.install(package);
|
||||
|
||||
self.execute(cmd, ctx, &None).await
|
||||
}
|
||||
|
||||
pub async fn cleanup(&self) -> Result<(), String> {
|
||||
if let Some(ref cgroup) = self.cgroup_manager {
|
||||
cgroup
|
||||
.destroy()
|
||||
.map_err(|e| format!("Failed to destroy cgroup: {}", e))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for super::types::Engine {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::collections::VecDeque;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct CapturedCall {
|
||||
program: String,
|
||||
args: Vec<String>,
|
||||
env_vars: std::collections::HashMap<String, String>,
|
||||
working_dir: PathBuf,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct MockExecutor {
|
||||
calls: Arc<Mutex<VecDeque<CapturedCall>>>,
|
||||
}
|
||||
|
||||
impl MockExecutor {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
calls: Arc::new(Mutex::new(VecDeque::new())),
|
||||
}
|
||||
}
|
||||
|
||||
fn capture(
|
||||
&self,
|
||||
program: String,
|
||||
args: Vec<String>,
|
||||
env_vars: std::collections::HashMap<String, String>,
|
||||
working_dir: PathBuf,
|
||||
) {
|
||||
self.calls.lock().unwrap().push_back(CapturedCall {
|
||||
program,
|
||||
args,
|
||||
env_vars,
|
||||
working_dir,
|
||||
});
|
||||
}
|
||||
|
||||
fn pop(&self) -> Option<CapturedCall> {
|
||||
self.calls.lock().unwrap().pop_front()
|
||||
}
|
||||
}
|
||||
|
||||
struct TestExecutor {
|
||||
mock: MockExecutor,
|
||||
}
|
||||
|
||||
impl TestExecutor {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
mock: MockExecutor::new(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn execute_command(
|
||||
&self,
|
||||
cmd: String,
|
||||
ctx: &ExecutionContext,
|
||||
_event_tx: EventSender,
|
||||
) -> Result<ExecutionResult, ExecutionError> {
|
||||
let mut env_vars = ctx.env_vars.clone();
|
||||
env_vars.insert("WS_TASKID".to_string(), ctx.task_id.clone());
|
||||
env_vars.insert("WS_PIPELINE".to_string(), ctx.pipeline_name.clone());
|
||||
env_vars.insert("WS_USERNAME".to_string(), ctx.username.clone());
|
||||
|
||||
self.mock.capture(
|
||||
ctx.shell.clone(),
|
||||
vec!["-c".to_string(), cmd.clone()],
|
||||
env_vars,
|
||||
ctx.working_dir.clone(),
|
||||
);
|
||||
|
||||
Ok(ExecutionResult {
|
||||
task_id: ctx.task_id.clone(),
|
||||
exit_code: 0,
|
||||
success: true,
|
||||
duration: Duration::from_millis(10),
|
||||
stdout: String::new(),
|
||||
stderr: String::new(),
|
||||
resource_usage: None,
|
||||
})
|
||||
}
|
||||
|
||||
async fn execute_script(
|
||||
&self,
|
||||
script_path: &PathBuf,
|
||||
ctx: &ExecutionContext,
|
||||
_event_tx: EventSender,
|
||||
) -> Result<ExecutionResult, ExecutionError> {
|
||||
let abs_path = if script_path.is_absolute() {
|
||||
script_path.clone()
|
||||
} else {
|
||||
ctx.working_dir.join(script_path)
|
||||
};
|
||||
|
||||
self.mock.capture(
|
||||
ctx.shell.clone(),
|
||||
vec!["-c".to_string(), abs_path.to_string_lossy().into_owned()],
|
||||
ctx.env_vars.clone(),
|
||||
ctx.working_dir.clone(),
|
||||
);
|
||||
|
||||
Ok(ExecutionResult {
|
||||
task_id: ctx.task_id.clone(),
|
||||
exit_code: 0,
|
||||
success: true,
|
||||
duration: Duration::from_millis(10),
|
||||
stdout: String::new(),
|
||||
stderr: String::new(),
|
||||
resource_usage: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn create_test_context() -> ExecutionContext {
|
||||
ExecutionContext {
|
||||
task_id: "test-task-1".to_string(),
|
||||
pipeline_name: "test-pipeline".to_string(),
|
||||
username: "testuser".to_string(),
|
||||
working_dir: PathBuf::from("/tmp"),
|
||||
env_vars: std::collections::HashMap::new(),
|
||||
timeout: Some(Duration::from_secs(30)),
|
||||
cgroup_path: None,
|
||||
shell: "bash".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_command_captures_args() {
|
||||
let executor = TestExecutor::new();
|
||||
let ctx = create_test_context();
|
||||
|
||||
let result = executor
|
||||
.execute_command("echo hello".to_string(), &ctx, None)
|
||||
.await;
|
||||
|
||||
assert!(result.is_ok());
|
||||
|
||||
let call = executor.mock.pop();
|
||||
assert!(call.is_some());
|
||||
|
||||
let call = call.unwrap();
|
||||
assert_eq!(call.program, "bash");
|
||||
assert!(call.args.contains(&"-c".to_string()));
|
||||
assert!(call.args.contains(&"echo hello".to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_script_captures_path() {
|
||||
let executor = TestExecutor::new();
|
||||
let ctx = create_test_context();
|
||||
|
||||
let script_path = PathBuf::from("/tmp/test_script.sh");
|
||||
let result = executor.execute_script(&script_path, &ctx, None).await;
|
||||
|
||||
assert!(result.is_ok());
|
||||
|
||||
let call = executor.mock.pop().unwrap();
|
||||
assert!(call.args.contains(&"-c".to_string()));
|
||||
assert!(call.args.iter().any(|a| a.contains("test_script.sh")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_command_with_env_vars() {
|
||||
let executor = TestExecutor::new();
|
||||
let mut ctx = create_test_context();
|
||||
ctx.env_vars
|
||||
.insert("TEST_VAR".to_string(), "test_value".to_string());
|
||||
|
||||
executor
|
||||
.execute_command("echo $TEST_VAR".to_string(), &ctx, None)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let call = executor.mock.pop().unwrap();
|
||||
assert_eq!(
|
||||
call.env_vars.get("TEST_VAR"),
|
||||
Some(&"test_value".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_includes_ws_environment() {
|
||||
let executor = TestExecutor::new();
|
||||
let ctx = create_test_context();
|
||||
|
||||
executor
|
||||
.execute_command("ls".to_string(), &ctx, None)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let call = executor.mock.pop().unwrap();
|
||||
|
||||
assert!(call.env_vars.contains_key("WS_TASKID"));
|
||||
assert!(call.env_vars.contains_key("WS_PIPELINE"));
|
||||
assert!(call.env_vars.contains_key("WS_USERNAME"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_command_preserves_working_dir() {
|
||||
let executor = TestExecutor::new();
|
||||
let mut ctx = create_test_context();
|
||||
ctx.working_dir = PathBuf::from("/custom/path");
|
||||
|
||||
executor
|
||||
.execute_command("pwd".to_string(), &ctx, None)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let call = executor.mock.pop().unwrap();
|
||||
assert_eq!(call.working_dir, PathBuf::from("/custom/path"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_local_executor_new_creates_default() {
|
||||
let executor = super::super::types::Engine::new();
|
||||
assert!(executor.cgroup_manager.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execution_context_default() {
|
||||
let ctx = ExecutionContext::default();
|
||||
|
||||
assert!(ctx.task_id.is_empty());
|
||||
assert_eq!(ctx.shell, "bash");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execution_result_success() {
|
||||
let result = ExecutionResult {
|
||||
task_id: "test".to_string(),
|
||||
exit_code: 0,
|
||||
success: true,
|
||||
duration: Duration::from_secs(1),
|
||||
stdout: "output".to_string(),
|
||||
stderr: "".to_string(),
|
||||
resource_usage: None,
|
||||
};
|
||||
|
||||
assert!(result.success);
|
||||
assert_eq!(result.exit_code, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execution_result_failure() {
|
||||
let result = ExecutionResult {
|
||||
task_id: "test".to_string(),
|
||||
exit_code: 1,
|
||||
success: false,
|
||||
duration: Duration::from_secs(1),
|
||||
stdout: "".to_string(),
|
||||
stderr: "error".to_string(),
|
||||
resource_usage: None,
|
||||
};
|
||||
|
||||
assert!(!result.success);
|
||||
assert_eq!(result.exit_code, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execution_error_variants() {
|
||||
let _err1 = ExecutionError::InvalidCommand("test".to_string());
|
||||
let _err2 = ExecutionError::ScriptNotFound(PathBuf::from("/nonexistent"));
|
||||
let _err3 = ExecutionError::PackageManagerNotFound;
|
||||
let _err4 = ExecutionError::Timeout;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_resource_limits_default() {
|
||||
let limits = ResourceLimits::default();
|
||||
|
||||
assert!(limits.memory_bytes.is_none());
|
||||
assert!(limits.cpu_weight.is_none());
|
||||
assert_eq!(limits.cpu_period_us, 0);
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
use os_info::Info;
|
||||
use serde_yaml::Value;
|
||||
use tokio::process::Command;
|
||||
mod apt;
|
||||
mod dnf;
|
||||
mod pacman;
|
||||
|
||||
pub trait PackageManager {
|
||||
@@ -31,14 +31,14 @@ pub trait PackageManager {
|
||||
fn change_mirror(&self, repo: &str, osinfo: Option<Info>) -> Command;
|
||||
|
||||
/// add a custom repository
|
||||
fn add_repository(&self, repo: &str, osinfo: Option<Info>) -> Option<Command>;
|
||||
fn add_repository(&self, repo: &Value, osinfo: Option<Info>) -> Vec<Command>;
|
||||
}
|
||||
|
||||
pub fn all_managers() -> Vec<Box<dyn PackageManager>> {
|
||||
vec![
|
||||
Box::new(apt::Apt),
|
||||
Box::new(pacman::Pacman),
|
||||
Box::new(dnf::Dnf)
|
||||
// Box::new(dnf::Dnf),
|
||||
]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
use crate::engine::pm::PackageManager;
|
||||
use os_info::{Info, Version};
|
||||
use serde_yaml::Value;
|
||||
use tokio::process::Command;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Apt;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct AptRepositories {
|
||||
url: String,
|
||||
key: Option<String>,
|
||||
}
|
||||
|
||||
impl PackageManager for Apt {
|
||||
fn name(&self) -> &'static str {
|
||||
"apt"
|
||||
}
|
||||
|
||||
fn adopt(&self, distro: &str) -> bool {
|
||||
// 匹配所有主要的使用 apt 的发行版
|
||||
// 注意：os_info 返回的 Type 转字符串通常是全小写或特定格式，这里做宽松匹配
|
||||
let d = distro.to_lowercase();
|
||||
matches!(
|
||||
d.as_str(),
|
||||
"debian"
|
||||
| "ubuntu"
|
||||
| "linuxmint"
|
||||
| "mint"
|
||||
| "pop"
|
||||
| "pop!_os"
|
||||
| "kali"
|
||||
| "raspbian"
|
||||
| "aosc"
|
||||
| "deepin"
|
||||
| "uos"
|
||||
| "elementary"
|
||||
)
|
||||
}
|
||||
|
||||
fn binary(&self) -> &'static str {
|
||||
"apt"
|
||||
}
|
||||
|
||||
fn update(&self) -> Command {
|
||||
let mut cmd = Command::new("apt");
|
||||
cmd.arg("update");
|
||||
cmd
|
||||
}
|
||||
|
||||
fn install(&self, package: &[&str]) -> Command {
|
||||
let mut cmd = Command::new("apt");
|
||||
cmd.arg("install");
|
||||
cmd.arg("-y"); // 自动确认
|
||||
cmd.args(package);
|
||||
cmd
|
||||
}
|
||||
|
||||
fn change_mirror(&self, repo: &str, osinfo: Option<Info>) -> Command {
|
||||
|
||||
todo!();
|
||||
}
|
||||
|
||||
fn add_repository(&self, repo: &Value, osinfo: Option<Info>) -> Vec<Command> {
|
||||
|
||||
todo!();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_adopt_true_for_ubuntu_and_debian() {
|
||||
let apt = Apt;
|
||||
assert!(apt.adopt("ubuntu"));
|
||||
assert!(apt.adopt("debian"));
|
||||
assert!(!apt.adopt("fedora"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_command_contents() {
|
||||
let cmd = Apt.update();
|
||||
let cmd = cmd.as_std();
|
||||
assert_eq!(cmd.get_program().to_string_lossy(), "apt");
|
||||
let args: Vec<String> = cmd
|
||||
.get_args()
|
||||
.map(|a| a.to_string_lossy().into_owned())
|
||||
.collect();
|
||||
assert_eq!(args, vec!["update"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_install_command_contents() {
|
||||
let cmd = Apt.install(&["curl"]);
|
||||
let cmd = cmd.as_std();
|
||||
assert_eq!(cmd.get_program().to_string_lossy(), "apt");
|
||||
let args: Vec<String> = cmd
|
||||
.get_args()
|
||||
.map(|a| a.to_string_lossy().into_owned())
|
||||
.collect();
|
||||
assert_eq!(args, vec!["install", "-y", "curl"]);
|
||||
}
|
||||
}
|
||||
|
||||
/// 辅助函数：从 os_info 中提取 Codename
|
||||
fn extract_codename(info: Option<&Info>) -> Option<String> {
|
||||
let info = info?;
|
||||
|
||||
// 1. 直接获取 codename (os_info 较新版本支持)
|
||||
if let Some(codename) = info.codename() {
|
||||
if !codename.is_empty() {
|
||||
return Some(codename.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
// 2. 尝试从 version 中推断 (针对某些把 codename 放在 Custom 里的情况)
|
||||
match info.version() {
|
||||
Version::Custom(v) => {
|
||||
// 有些系统可能把 codename 放在这里，虽然不标准
|
||||
if !v.chars().next().map(|c| c.is_numeric()).unwrap_or(true) {
|
||||
return Some(v.clone());
|
||||
}
|
||||
}
|
||||
Version::Semantic(major, minor, _) => {
|
||||
// 对于 Ubuntu/Debian，如果没有 codename，有时可以用版本号兜底 (如 22.04)
|
||||
// 但 apt 源通常更喜欢 codename。如果实在没有，返回版本号字符串作为最后手段
|
||||
return Some(format!("{}.{}", major, minor));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
// 3. 如果 os_info 没提供，且是常见发行版，可以尝试根据 Type 猜测？
|
||||
// 不推荐硬编码猜测，因为版本太多。
|
||||
// 最好返回 None，让调用者决定是报错还是让用户手动指定。
|
||||
|
||||
None
|
||||
}
|
||||
@@ -0,0 +1,603 @@
|
||||
use crate::engine::pm::PackageManager;
|
||||
use os_info::Info;
|
||||
use serde::Deserialize;
|
||||
use serde_yaml::Value;
|
||||
use tokio::process::Command;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Pacman;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct PacmanRepo {
|
||||
name: String,
|
||||
server: String,
|
||||
cacheserver: Option<String>,
|
||||
siglevel: Option<SigLevel>,
|
||||
keyid: Option<String>,
|
||||
keypackage: Option<String>,
|
||||
}
|
||||
|
||||
struct SigLevel {
|
||||
raw: String,
|
||||
}
|
||||
|
||||
impl SigLevel {
|
||||
pub fn new(raw: String) -> Self {
|
||||
for token in raw.split_whitespace() {
|
||||
if !VALID_SIGLEVEL_TOKENS.contains(&token) {
|
||||
log::warn!("Invalid siglevel token: {}", token);
|
||||
}
|
||||
}
|
||||
Self { raw }
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for SigLevel {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{}", self.raw)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for SigLevel {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let s = String::deserialize(deserializer)?;
|
||||
Ok(SigLevel::new(s))
|
||||
}
|
||||
}
|
||||
|
||||
const VALID_SIGLEVEL_TOKENS: &[&str] = &[
|
||||
// Token
|
||||
"Never",
|
||||
"Optional",
|
||||
"Required",
|
||||
"TrustedOnly",
|
||||
"TrustAll",
|
||||
// Package prefix
|
||||
"PackageNever",
|
||||
"PackageOptional",
|
||||
"PackageRequired",
|
||||
"PackageTrustedOnly",
|
||||
"PackageTrustAll",
|
||||
// Database prefix
|
||||
"DatabaseNever",
|
||||
"DatabaseOptional",
|
||||
"DatabaseRequired",
|
||||
"DatabaseTrustedOnly",
|
||||
"DatabaseTrustAll",
|
||||
];
|
||||
|
||||
impl PackageManager for Pacman {
|
||||
fn name(&self) -> &'static str {
|
||||
"pacman"
|
||||
}
|
||||
|
||||
fn adopt(&self, distro: &str) -> bool {
|
||||
let d = distro.to_lowercase();
|
||||
matches!(
|
||||
d.as_str(),
|
||||
"arch"
|
||||
| "manjaro"
|
||||
| "endeavouros"
|
||||
| "garuda"
|
||||
| "cachyos"
|
||||
| "artix"
|
||||
| "mabox"
|
||||
| "nobara"
|
||||
| "instantos"
|
||||
| "archlinux"
|
||||
)
|
||||
}
|
||||
|
||||
fn binary(&self) -> &'static str {
|
||||
"pacman"
|
||||
}
|
||||
|
||||
fn update(&self) -> Command {
|
||||
let mut cmd = Command::new("pacman");
|
||||
cmd.arg("-Sy");
|
||||
cmd
|
||||
}
|
||||
|
||||
fn install(&self, package: &[&str]) -> Command {
|
||||
let mut cmd = Command::new("pacman");
|
||||
cmd.arg("-S");
|
||||
cmd.arg("--noconfirm");
|
||||
cmd.arg("--needed");
|
||||
cmd.args(package);
|
||||
cmd
|
||||
}
|
||||
|
||||
fn change_mirror(&self, repo: &str, _osinfo: Option<Info>) -> Command {
|
||||
let mirrorlist_path = "/etc/pacman.d/mirrorlist";
|
||||
|
||||
let content = format!(
|
||||
"# Generated by WSExecutor-PM\n\
|
||||
# Date: $(date)\n\
|
||||
# Original content overwritten.\n\
|
||||
\n\
|
||||
Server = {}\n",
|
||||
repo
|
||||
);
|
||||
|
||||
let mut cmd = Command::new("sh");
|
||||
cmd.arg("-c").arg(format!(
|
||||
"cat > {} << 'PACMAN_EOF'\n{}\nPACMAN_EOF",
|
||||
mirrorlist_path, content
|
||||
));
|
||||
|
||||
cmd
|
||||
}
|
||||
|
||||
fn add_repository(&self, repo: &Value, _osinfo: Option<Info>) -> Vec<Command> {
|
||||
let repos: Vec<PacmanRepo> = match serde_yaml::from_value(repo.clone()) {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
log::error!("Failed to parse pacman repository: {}", e);
|
||||
return Vec::new();
|
||||
}
|
||||
};
|
||||
let pacman_conf = "/etc/pacman.conf";
|
||||
let mut content = String::with_capacity(512);
|
||||
content.push_str("# Modified by HBW baker\n");
|
||||
let mut cmds = vec![];
|
||||
for repo in &repos {
|
||||
// Install keyring
|
||||
if repo.keyid.is_some() && repo.keypackage.is_some() {
|
||||
log::error!(
|
||||
"Both keyid and keypackage are set for repository {}",
|
||||
repo.name
|
||||
);
|
||||
log::warn!("Ignoring...");
|
||||
continue;
|
||||
}
|
||||
if let Some(keyid) = &repo.keyid {
|
||||
let mut cmd = Command::new("sh");
|
||||
cmd.arg("-c")
|
||||
.arg(format!("pacman-key --recv-keys {}", keyid));
|
||||
cmds.push(cmd);
|
||||
let mut cmd = Command::new("sh");
|
||||
cmd.arg("-c")
|
||||
.arg(format!("pacman-key --lsign-key {}", keyid));
|
||||
cmds.push(cmd);
|
||||
} else if let Some(keypackage) = &repo.keypackage {
|
||||
let mut cmd = Command::new("sh");
|
||||
cmd.arg("-c")
|
||||
.arg(format!("pacman -Sy {} --needed --noconfirm", keypackage));
|
||||
cmds.push(cmd);
|
||||
}
|
||||
|
||||
// Configure repository
|
||||
let mut block = format!("[{}]\n", &repo.name);
|
||||
block.push_str(&format!("Server = {}\n", &repo.server));
|
||||
|
||||
if let Some(cacheserver) = &repo.cacheserver {
|
||||
block.push_str(&format!("CacheServer = {}\n", cacheserver));
|
||||
}
|
||||
if let Some(siglevel) = &repo.siglevel {
|
||||
block.push_str(&format!("SigLevel = {}\n", siglevel));
|
||||
}
|
||||
content.push_str(&block);
|
||||
}
|
||||
|
||||
let mut cmd = Command::new("sh");
|
||||
cmd.arg("-c").arg(format!(
|
||||
"cat >> {} << 'PACMAN_EOF'\n{}\nPACMAN_EOF",
|
||||
pacman_conf, content
|
||||
));
|
||||
cmds.insert(0, cmd);
|
||||
|
||||
cmds
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_name() {
|
||||
let pacman = Pacman;
|
||||
assert_eq!(pacman.name(), "pacman");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_binary() {
|
||||
let pacman = Pacman;
|
||||
assert_eq!(pacman.binary(), "pacman");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_adopt_true_for_arch_based_distros() {
|
||||
let pacman = Pacman;
|
||||
assert!(pacman.adopt("arch"));
|
||||
assert!(pacman.adopt("Arch"));
|
||||
assert!(pacman.adopt("ARCH"));
|
||||
assert!(pacman.adopt("manjaro"));
|
||||
assert!(pacman.adopt("endeavouros"));
|
||||
assert!(pacman.adopt("garuda"));
|
||||
assert!(pacman.adopt("cachyos"));
|
||||
assert!(pacman.adopt("artix"));
|
||||
assert!(pacman.adopt("mabox"));
|
||||
assert!(pacman.adopt("nobara"));
|
||||
assert!(pacman.adopt("instantos"));
|
||||
assert!(pacman.adopt("archlinux"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_adopt_false_for_non_arch_distros() {
|
||||
let pacman = Pacman;
|
||||
assert!(!pacman.adopt("ubuntu"));
|
||||
assert!(!pacman.adopt("debian"));
|
||||
assert!(!pacman.adopt("fedora"));
|
||||
assert!(!pacman.adopt("opensuse"));
|
||||
assert!(!pacman.adopt("centos"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_command_contents() {
|
||||
let cmd = Pacman.update();
|
||||
let cmd = cmd.as_std();
|
||||
assert_eq!(cmd.get_program().to_string_lossy(), "pacman");
|
||||
let args: Vec<String> = cmd
|
||||
.get_args()
|
||||
.map(|a| a.to_string_lossy().into_owned())
|
||||
.collect();
|
||||
assert_eq!(args, vec!["-Sy"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_install_command_contents() {
|
||||
let cmd = Pacman.install(&["curl", "wget"]);
|
||||
let cmd = cmd.as_std();
|
||||
assert_eq!(cmd.get_program().to_string_lossy(), "pacman");
|
||||
let args: Vec<String> = cmd
|
||||
.get_args()
|
||||
.map(|a| a.to_string_lossy().into_owned())
|
||||
.collect();
|
||||
assert_eq!(args, vec!["-S", "--noconfirm", "--needed", "curl", "wget"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_install_single_package() {
|
||||
let cmd = Pacman.install(&["package"]);
|
||||
let cmd = cmd.as_std();
|
||||
let args: Vec<String> = cmd
|
||||
.get_args()
|
||||
.map(|a| a.to_string_lossy().into_owned())
|
||||
.collect();
|
||||
assert_eq!(args, vec!["-S", "--noconfirm", "--needed", "package"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_siglevel_valid_tokens() {
|
||||
let siglevel = SigLevel::new("Never".to_string());
|
||||
assert_eq!(siglevel.to_string(), "Never");
|
||||
|
||||
let siglevel = SigLevel::new("Optional Required".to_string());
|
||||
assert_eq!(siglevel.to_string(), "Optional Required");
|
||||
|
||||
let siglevel = SigLevel::new("PackageOptional DatabaseRequired".to_string());
|
||||
assert_eq!(siglevel.to_string(), "PackageOptional DatabaseRequired");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_siglevel_with_invalid_tokens() {
|
||||
// Invalid tokens are logged but still accepted
|
||||
let siglevel = SigLevel::new("Never InvalidToken".to_string());
|
||||
assert_eq!(siglevel.to_string(), "Never InvalidToken");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_siglevel_deserialize() {
|
||||
let yaml = "Never";
|
||||
let siglevel: SigLevel = serde_yaml::from_str(yaml).unwrap();
|
||||
assert_eq!(siglevel.to_string(), "Never");
|
||||
|
||||
let yaml = "PackageOptional DatabaseRequired";
|
||||
let siglevel: SigLevel = serde_yaml::from_str(yaml).unwrap();
|
||||
assert_eq!(siglevel.to_string(), "PackageOptional DatabaseRequired");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_siglevel_all_valid_tokens() {
|
||||
let all_tokens = VALID_SIGLEVEL_TOKENS.join(" ");
|
||||
let siglevel = SigLevel::new(all_tokens.clone());
|
||||
assert_eq!(siglevel.to_string(), all_tokens);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_change_mirror_command() {
|
||||
let cmd = Pacman.change_mirror("https://mirror.example.com/archlinux", None);
|
||||
let cmd = cmd.as_std();
|
||||
assert_eq!(cmd.get_program().to_string_lossy(), "sh");
|
||||
let args: Vec<String> = cmd
|
||||
.get_args()
|
||||
.map(|a| a.to_string_lossy().into_owned())
|
||||
.collect();
|
||||
let full_arg = args.join(" ");
|
||||
assert!(full_arg.contains("cat > /etc/pacman.d/mirrorlist"));
|
||||
assert!(full_arg.contains("https://mirror.example.com/archlinux"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_repository_single_repo() {
|
||||
let repo_yaml = serde_yaml::from_value(serde_yaml::Value::Sequence(vec![
|
||||
serde_yaml::Value::Mapping(
|
||||
[
|
||||
(
|
||||
serde_yaml::Value::String("name".to_string()),
|
||||
serde_yaml::Value::String("customrepo".to_string()),
|
||||
),
|
||||
(
|
||||
serde_yaml::Value::String("server".to_string()),
|
||||
serde_yaml::Value::String("https://repo.example.com".to_string()),
|
||||
),
|
||||
]
|
||||
.into_iter()
|
||||
.collect(),
|
||||
),
|
||||
]))
|
||||
.unwrap();
|
||||
|
||||
let cmds = Pacman.add_repository(&repo_yaml, None);
|
||||
// Should have at least one command (the one to append to pacman.conf)
|
||||
assert!(!cmds.is_empty());
|
||||
let last_cmd = cmds.last().unwrap().as_std();
|
||||
let args: Vec<String> = last_cmd
|
||||
.get_args()
|
||||
.map(|a| a.to_string_lossy().into_owned())
|
||||
.collect();
|
||||
let full_arg = args.join(" ");
|
||||
assert!(full_arg.contains("cat >> /etc/pacman.conf"));
|
||||
assert!(full_arg.contains("[customrepo]"));
|
||||
assert!(full_arg.contains("https://repo.example.com"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_repository_with_siglevel() {
|
||||
let repo_yaml = serde_yaml::from_value(serde_yaml::Value::Sequence(vec![
|
||||
serde_yaml::Value::Mapping(
|
||||
[
|
||||
(
|
||||
serde_yaml::Value::String("name".to_string()),
|
||||
serde_yaml::Value::String("customrepo".to_string()),
|
||||
),
|
||||
(
|
||||
serde_yaml::Value::String("server".to_string()),
|
||||
serde_yaml::Value::String("https://repo.example.com".to_string()),
|
||||
),
|
||||
(
|
||||
serde_yaml::Value::String("siglevel".to_string()),
|
||||
serde_yaml::Value::String("PackageOptional".to_string()),
|
||||
),
|
||||
]
|
||||
.into_iter()
|
||||
.collect(),
|
||||
),
|
||||
]))
|
||||
.unwrap();
|
||||
|
||||
let cmds = Pacman.add_repository(&repo_yaml, None);
|
||||
let last_cmd = cmds.last().unwrap().as_std();
|
||||
let args: Vec<String> = last_cmd
|
||||
.get_args()
|
||||
.map(|a| a.to_string_lossy().into_owned())
|
||||
.collect();
|
||||
let full_arg = args.join(" ");
|
||||
assert!(full_arg.contains("SigLevel = PackageOptional"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_repository_with_keyid() {
|
||||
let repo_yaml = serde_yaml::from_value(serde_yaml::Value::Sequence(vec![
|
||||
serde_yaml::Value::Mapping(
|
||||
[
|
||||
(
|
||||
serde_yaml::Value::String("name".to_string()),
|
||||
serde_yaml::Value::String("customrepo".to_string()),
|
||||
),
|
||||
(
|
||||
serde_yaml::Value::String("server".to_string()),
|
||||
serde_yaml::Value::String("https://repo.example.com".to_string()),
|
||||
),
|
||||
(
|
||||
serde_yaml::Value::String("keyid".to_string()),
|
||||
serde_yaml::Value::String("ABCDEF123456".to_string()),
|
||||
),
|
||||
]
|
||||
.into_iter()
|
||||
.collect(),
|
||||
),
|
||||
]))
|
||||
.unwrap();
|
||||
|
||||
let cmds = Pacman.add_repository(&repo_yaml, None);
|
||||
// Should have 3 commands: recv-keys, lsign-key, and append to conf
|
||||
assert_eq!(cmds.len(), 3);
|
||||
|
||||
let recv_cmd = cmds[1].as_std();
|
||||
let recv_args: Vec<String> = recv_cmd
|
||||
.get_args()
|
||||
.map(|a| a.to_string_lossy().into_owned())
|
||||
.collect();
|
||||
|
||||
assert!(
|
||||
recv_args
|
||||
.join(" ")
|
||||
.contains("pacman-key --recv-keys ABCDEF123456")
|
||||
);
|
||||
|
||||
let lsign_cmd = cmds[2].as_std();
|
||||
let lsign_args: Vec<String> = lsign_cmd
|
||||
.get_args()
|
||||
.map(|a| a.to_string_lossy().into_owned())
|
||||
.collect();
|
||||
assert!(
|
||||
lsign_args
|
||||
.join(" ")
|
||||
.contains("pacman-key --lsign-key ABCDEF123456")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_repository_with_keypackage() {
|
||||
let repo_yaml = serde_yaml::from_value(serde_yaml::Value::Sequence(vec![
|
||||
serde_yaml::Value::Mapping(
|
||||
[
|
||||
(
|
||||
serde_yaml::Value::String("name".to_string()),
|
||||
serde_yaml::Value::String("customrepo".to_string()),
|
||||
),
|
||||
(
|
||||
serde_yaml::Value::String("server".to_string()),
|
||||
serde_yaml::Value::String("https://repo.example.com".to_string()),
|
||||
),
|
||||
(
|
||||
serde_yaml::Value::String("keypackage".to_string()),
|
||||
serde_yaml::Value::String("gnupg".to_string()),
|
||||
),
|
||||
]
|
||||
.into_iter()
|
||||
.collect(),
|
||||
),
|
||||
]))
|
||||
.unwrap();
|
||||
|
||||
let cmds = Pacman.add_repository(&repo_yaml, None);
|
||||
// Should have 2 commands: install keypackage and append to conf
|
||||
assert_eq!(cmds.len(), 2);
|
||||
|
||||
let install_cmd = cmds[1].as_std();
|
||||
let install_args: Vec<String> = install_cmd
|
||||
.get_args()
|
||||
.map(|a| a.to_string_lossy().into_owned())
|
||||
.collect();
|
||||
assert!(
|
||||
install_args
|
||||
.join(" ")
|
||||
.contains("pacman -Sy gnupg --needed --noconfirm")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_repository_with_cacheserver() {
|
||||
let repo_yaml = serde_yaml::from_value(serde_yaml::Value::Sequence(vec![
|
||||
serde_yaml::Value::Mapping(
|
||||
[
|
||||
(
|
||||
serde_yaml::Value::String("name".to_string()),
|
||||
serde_yaml::Value::String("customrepo".to_string()),
|
||||
),
|
||||
(
|
||||
serde_yaml::Value::String("server".to_string()),
|
||||
serde_yaml::Value::String("https://repo.example.com".to_string()),
|
||||
),
|
||||
(
|
||||
serde_yaml::Value::String("cacheserver".to_string()),
|
||||
serde_yaml::Value::String("https://cache.example.com".to_string()),
|
||||
),
|
||||
]
|
||||
.into_iter()
|
||||
.collect(),
|
||||
),
|
||||
]))
|
||||
.unwrap();
|
||||
|
||||
let cmds = Pacman.add_repository(&repo_yaml, None);
|
||||
let last_cmd = cmds.last().unwrap().as_std();
|
||||
let args: Vec<String> = last_cmd
|
||||
.get_args()
|
||||
.map(|a| a.to_string_lossy().into_owned())
|
||||
.collect();
|
||||
let full_arg = args.join(" ");
|
||||
assert!(full_arg.contains("CacheServer = https://cache.example.com"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_repository_multiple_repos() {
|
||||
let repos_yaml = serde_yaml::from_value(serde_yaml::Value::Sequence(vec![
|
||||
serde_yaml::Value::Mapping(
|
||||
[
|
||||
(
|
||||
serde_yaml::Value::String("name".to_string()),
|
||||
serde_yaml::Value::String("repo1".to_string()),
|
||||
),
|
||||
(
|
||||
serde_yaml::Value::String("server".to_string()),
|
||||
serde_yaml::Value::String("https://repo1.example.com".to_string()),
|
||||
),
|
||||
]
|
||||
.into_iter()
|
||||
.collect(),
|
||||
),
|
||||
serde_yaml::Value::Mapping(
|
||||
[
|
||||
(
|
||||
serde_yaml::Value::String("name".to_string()),
|
||||
serde_yaml::Value::String("repo2".to_string()),
|
||||
),
|
||||
(
|
||||
serde_yaml::Value::String("server".to_string()),
|
||||
serde_yaml::Value::String("https://repo2.example.com".to_string()),
|
||||
),
|
||||
]
|
||||
.into_iter()
|
||||
.collect(),
|
||||
),
|
||||
]))
|
||||
.unwrap();
|
||||
|
||||
let cmds = Pacman.add_repository(&repos_yaml, None);
|
||||
let last_cmd = cmds.last().unwrap().as_std();
|
||||
let args: Vec<String> = last_cmd
|
||||
.get_args()
|
||||
.map(|a| a.to_string_lossy().into_owned())
|
||||
.collect();
|
||||
let full_arg = args.join(" ");
|
||||
assert!(full_arg.contains("[repo1]"));
|
||||
assert!(full_arg.contains("[repo2]"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_repository_both_keyid_and_keypackage_logs_error() {
|
||||
let repo_yaml = serde_yaml::from_value(serde_yaml::Value::Sequence(vec![
|
||||
serde_yaml::Value::Mapping(
|
||||
[
|
||||
(
|
||||
serde_yaml::Value::String("name".to_string()),
|
||||
serde_yaml::Value::String("customrepo".to_string()),
|
||||
),
|
||||
(
|
||||
serde_yaml::Value::String("server".to_string()),
|
||||
serde_yaml::Value::String("https://repo.example.com".to_string()),
|
||||
),
|
||||
(
|
||||
serde_yaml::Value::String("keyid".to_string()),
|
||||
serde_yaml::Value::String("ABCDEF123456".to_string()),
|
||||
),
|
||||
(
|
||||
serde_yaml::Value::String("keypackage".to_string()),
|
||||
serde_yaml::Value::String("gnupg".to_string()),
|
||||
),
|
||||
]
|
||||
.into_iter()
|
||||
.collect(),
|
||||
),
|
||||
]))
|
||||
.unwrap();
|
||||
|
||||
let cmds = Pacman.add_repository(&repo_yaml, None);
|
||||
// Should have no key-related commands when both are set (error case)
|
||||
assert!(cmds.is_empty() || cmds.len() == 1); // Only the conf append cmd if any
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_repository_invalid_yaml_returns_empty() {
|
||||
let invalid_yaml = serde_yaml::Value::String("invalid: [yaml".to_string());
|
||||
let cmds = Pacman.add_repository(&invalid_yaml, None);
|
||||
assert!(cmds.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
pub use super::cgroups::CgroupManager;
|
||||
pub use crate::error::{EXITCODE_EXECUTION_ERROR, EXITCODE_IO_ERROR, ExecutionError};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ResourceUsage {
|
||||
pub cpu_time_secs: f64,
|
||||
pub max_memory_kb: u64,
|
||||
pub io_read_kb: u64,
|
||||
pub io_write_kb: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ExecutionContext {
|
||||
pub task_id: String,
|
||||
pub pipeline_name: String,
|
||||
pub username: String,
|
||||
pub working_dir: PathBuf,
|
||||
pub env_vars: HashMap<String, String>,
|
||||
pub timeout: Option<Duration>,
|
||||
pub cgroup_path: Option<PathBuf>,
|
||||
pub shell: String,
|
||||
}
|
||||
|
||||
impl Default for ExecutionContext {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
task_id: String::new(),
|
||||
pipeline_name: String::new(),
|
||||
username: String::new(),
|
||||
working_dir: std::env::current_dir().unwrap_or_else(|_| PathBuf::from("/")),
|
||||
env_vars: HashMap::new(),
|
||||
timeout: None,
|
||||
cgroup_path: None,
|
||||
shell: String::from("bash"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct ResourceLimits {
|
||||
pub memory_bytes: Option<u64>,
|
||||
pub cpu_weight: Option<u64>,
|
||||
pub cpu_max_us: Option<u64>,
|
||||
pub cpu_period_us: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct ExecutionResult {
|
||||
pub task_id: String,
|
||||
pub exit_code: i32,
|
||||
pub success: bool,
|
||||
pub duration: Duration,
|
||||
pub stdout: String,
|
||||
pub stderr: String,
|
||||
pub resource_usage: Option<ResourceUsage>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub enum StreamType {
|
||||
Stdout,
|
||||
Stderr,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub enum ExecutionEvent {
|
||||
TaskStarted {
|
||||
task_id: String,
|
||||
timestamp: chrono::DateTime<chrono::Utc>,
|
||||
},
|
||||
OutputChunk {
|
||||
task_id: String,
|
||||
stream: StreamType,
|
||||
data: String,
|
||||
},
|
||||
TaskCompleted {
|
||||
task_id: String,
|
||||
result: ExecutionResult,
|
||||
},
|
||||
TaskFailed {
|
||||
task_id: String,
|
||||
error: String,
|
||||
},
|
||||
}
|
||||
|
||||
pub type EventSender = Option<mpsc::Sender<ExecutionEvent>>;
|
||||
|
||||
pub struct Engine {
|
||||
pub cgroup_manager: Option<CgroupManager>,
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
use std::path::PathBuf;
|
||||
use thiserror::Error;
|
||||
|
||||
pub const EXITCODE_OK: i32 = 0;
|
||||
pub const EXITCODE_GENERAL_ERROR: i32 = 1;
|
||||
pub const EXITCODE_INVALID_ARGS: i32 = 2;
|
||||
pub const EXITCODE_IO_ERROR: i32 = 3;
|
||||
pub const EXITCODE_PARSE_ERROR: i32 = 4;
|
||||
pub const EXITCODE_EXECUTION_ERROR: i32 = 5;
|
||||
pub const EXITCODE_TIMEOUT: i32 = 124;
|
||||
pub const EXITCODE_PRIV_DROP_FAILED: i32 = 201;
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum BuilderError {
|
||||
#[error("IO error: {0}")]
|
||||
IoError(#[from] std::io::Error),
|
||||
|
||||
#[error("Template error: {0}")]
|
||||
TemplateError(#[from] minijinja::Error),
|
||||
|
||||
#[error("Tempfile error: {0}")]
|
||||
TempfileError(#[from] tempfile::PersistError),
|
||||
}
|
||||
|
||||
impl BuilderError {
|
||||
pub fn exit_code(&self) -> i32 {
|
||||
match self {
|
||||
BuilderError::IoError(_) => EXITCODE_IO_ERROR,
|
||||
BuilderError::TemplateError(_) => EXITCODE_PARSE_ERROR,
|
||||
BuilderError::TempfileError(_) => EXITCODE_IO_ERROR,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub type Result<T> = std::result::Result<T, BuilderError>;
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum PrebakeError {
|
||||
#[error("Privilege drop failed: {0}")]
|
||||
PrivDropFailed(String),
|
||||
|
||||
#[error("User not found: {0}")]
|
||||
UserNotFound(String),
|
||||
|
||||
#[error("Invalid privilege state: current_uid={current_uid}, target_uid={target_uid}")]
|
||||
InvalidPrivilegeState { current_uid: u32, target_uid: u32 },
|
||||
|
||||
#[error("Invalid stage: current={current}, expected={expected}")]
|
||||
InvalidStage { current: String, expected: String },
|
||||
|
||||
#[error("Failed to validate prebake.yml.")]
|
||||
ValidateError(),
|
||||
|
||||
#[error("IO error: {0}")]
|
||||
IoError(#[from] std::io::Error),
|
||||
|
||||
#[error("YAML parse error: {0}")]
|
||||
YamlParseError(#[from] serde_yaml::Error),
|
||||
}
|
||||
|
||||
impl PrebakeError {
|
||||
pub fn exit_code(&self) -> i32 {
|
||||
match self {
|
||||
PrebakeError::PrivDropFailed(_) => EXITCODE_PRIV_DROP_FAILED,
|
||||
PrebakeError::UserNotFound(_) => EXITCODE_PRIV_DROP_FAILED,
|
||||
PrebakeError::InvalidPrivilegeState { .. } => EXITCODE_PRIV_DROP_FAILED,
|
||||
PrebakeError::ValidateError() => EXITCODE_PARSE_ERROR,
|
||||
PrebakeError::InvalidStage { .. } => EXITCODE_PRIV_DROP_FAILED,
|
||||
PrebakeError::IoError(_) => EXITCODE_IO_ERROR,
|
||||
PrebakeError::YamlParseError(_) => EXITCODE_PARSE_ERROR,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum ExecutionError {
|
||||
#[error("Invalid command: {0}")]
|
||||
InvalidCommand(String),
|
||||
|
||||
#[error("Execution failed: {0}")]
|
||||
ExecutionFailed(String),
|
||||
|
||||
#[error("Permission denied: uid={euid}, gid={egid}: {reason}")]
|
||||
PermissionDenied {
|
||||
euid: u32,
|
||||
egid: u32,
|
||||
reason: String,
|
||||
},
|
||||
|
||||
#[error("Timeout")]
|
||||
Timeout,
|
||||
|
||||
#[error("IO error: {0}")]
|
||||
IoError(String),
|
||||
|
||||
#[error("Script not found: {0}")]
|
||||
ScriptNotFound(PathBuf),
|
||||
|
||||
#[error("Package manager not found")]
|
||||
PackageManagerNotFound,
|
||||
}
|
||||
|
||||
impl ExecutionError {
|
||||
pub fn exit_code(&self) -> i32 {
|
||||
match self {
|
||||
ExecutionError::InvalidCommand(_) => EXITCODE_EXECUTION_ERROR,
|
||||
ExecutionError::ExecutionFailed(_) => EXITCODE_EXECUTION_ERROR,
|
||||
ExecutionError::PermissionDenied { .. } => EXITCODE_GENERAL_ERROR,
|
||||
ExecutionError::Timeout => EXITCODE_TIMEOUT,
|
||||
ExecutionError::IoError(_) => EXITCODE_IO_ERROR,
|
||||
ExecutionError::ScriptNotFound(_) => EXITCODE_INVALID_ARGS,
|
||||
ExecutionError::PackageManagerNotFound => EXITCODE_GENERAL_ERROR,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
// Library crate for workshop-executor
|
||||
// Selective exports: only modules used by external callers or tests
|
||||
|
||||
pub mod daemon;
|
||||
pub mod decorator;
|
||||
pub mod engine;
|
||||
pub mod error;
|
||||
pub mod monitor;
|
||||
pub mod parser;
|
||||
pub mod prebake;
|
||||
pub mod schedule;
|
||||
pub mod socket;
|
||||
pub mod variable;
|
||||
pub mod types;
|
||||
|
||||
// External crates re-exports
|
||||
pub use config;
|
||||
|
||||
// Re-export commonly used types
|
||||
pub use engine::{Engine, EventSender, ExecutionContext, ExecutionError, ExecutionResult};
|
||||
|
||||
// CLI types used by main (keep here since they're used in daemon)
|
||||
pub use clap::{Parser, Subcommand};
|
||||
pub use serde::Deserialize;
|
||||
pub use std::collections::HashMap;
|
||||
pub use std::path::PathBuf;
|
||||
|
||||
pub mod cli {
|
||||
pub use super::{Deserialize, HashMap, Parser, PathBuf, Subcommand};
|
||||
pub use clap::Command;
|
||||
|
||||
/// Honey Biscuit Workshop script executor
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(version, about, long_about=None)]
|
||||
pub struct Cli {
|
||||
/// Username of the pipeline executor
|
||||
#[arg(short, long)]
|
||||
pub username: String,
|
||||
|
||||
/// Pipeline ID of this pipeline
|
||||
#[arg(short, long)]
|
||||
pub pipeline: String,
|
||||
|
||||
/// Build ID of this build
|
||||
#[arg(short, long)]
|
||||
pub build_id: String,
|
||||
|
||||
/// Parse the script without executing
|
||||
#[arg(long, default_value = "false")]
|
||||
pub dry_run: bool,
|
||||
|
||||
#[command(subcommand)]
|
||||
pub command: Option<Commands>,
|
||||
}
|
||||
|
||||
#[derive(Subcommand, Debug)]
|
||||
pub enum Commands {
|
||||
/// Monitor a running task
|
||||
Monitor { group: String },
|
||||
/// Prepare a task
|
||||
Prebake { config: String },
|
||||
/// Run a task
|
||||
Bake {
|
||||
/// Path to the script to execute
|
||||
script: String,
|
||||
/// Path to bake_base.sh
|
||||
#[arg(short, long, default_value = "./bake_base.sh")]
|
||||
bake_base: String,
|
||||
},
|
||||
/// Finish a task
|
||||
Finalize { config: String },
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct TaskSpec {
|
||||
pub subcommand: String,
|
||||
pub arguments: Vec<String>,
|
||||
pub options: HashMap<String, String>,
|
||||
}
|
||||
|
||||
impl TaskSpec {
|
||||
pub fn command(&self, executable: &PathBuf) -> tokio::process::Command {
|
||||
let mut command = tokio::process::Command::new(executable);
|
||||
command.arg(&self.subcommand);
|
||||
for arg in &self.arguments {
|
||||
command.arg(arg);
|
||||
}
|
||||
for (flag, value) in self.options.iter() {
|
||||
if value.is_empty() {
|
||||
command.arg(format!("--{}", flag));
|
||||
} else {
|
||||
command.arg(format!("--{}={}", flag, value));
|
||||
}
|
||||
}
|
||||
command
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
use env_logger;
|
||||
use workshop_executor::{
|
||||
cli::{Cli, Commands, Parser},
|
||||
config, daemon,
|
||||
monitor::{self, monitor},
|
||||
prebake::{self, prebake},
|
||||
};
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
env_logger::init();
|
||||
let cli = Cli::parse();
|
||||
match &cli.command {
|
||||
Some(Commands::Monitor { group: _ }) => {
|
||||
unimplemented!("executor-monitor should be moved to daemon instead.")
|
||||
// log::info!("Starting monitor for group: {}", group);
|
||||
// let result = monitor(&group, Some(1000), settings).await;
|
||||
// result.unwrap();
|
||||
}
|
||||
Some(Commands::Prebake { config }) => {
|
||||
let result = prebake(config, &cli).await;
|
||||
result.unwrap();
|
||||
}
|
||||
Some(Commands::Bake {
|
||||
script: _,
|
||||
bake_base: _,
|
||||
}) => {
|
||||
// builder::bake(script, bake_base, username, pipeline, *dry_run);
|
||||
}
|
||||
Some(Commands::Finalize { config: _ }) => {
|
||||
// builder::finalize(config);
|
||||
}
|
||||
None => {
|
||||
// Running in daemon mode
|
||||
log::info!("Executor will be running in daemon mode.");
|
||||
|
||||
let result = daemon::daemon(&cli).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,10 @@
|
||||
mod cgroups;
|
||||
mod jobobject;
|
||||
use crate::socket::{establish_connection, get_socket_addr};
|
||||
use config::Config;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::time::Instant;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use crate::socket::{establish_connection, get_socket_addr};
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct ResourceUsageInfo {
|
||||
@@ -22,7 +22,6 @@ pub enum OsType {
|
||||
MacOS,
|
||||
}
|
||||
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Clone)]
|
||||
pub struct UsageStats {
|
||||
pub cpu_usage_percent: f64,
|
||||
@@ -40,7 +39,11 @@ fn default_internal_timestamp() -> Instant{
|
||||
Instant::now()
|
||||
}
|
||||
|
||||
pub async fn monitor(name: &String, interval_ms: Option<u64>, settings: Config) -> anyhow::Result<()> {
|
||||
pub async fn monitor(
|
||||
name: &String,
|
||||
interval_ms: Option<u64>,
|
||||
settings: Config,
|
||||
) -> anyhow::Result<()> {
|
||||
// let socket_addr = get_wsagent_socket_addr()?;
|
||||
let socket_addr = get_socket_addr(settings.get_string("socket")?)?;
|
||||
let interval = tokio::time::Duration::from_millis(interval_ms.unwrap_or(1000));
|
||||
@@ -62,7 +65,9 @@ pub async fn monitor(name: &String, interval_ms: Option<u64>, settings: Config)
|
||||
name: name.clone(),
|
||||
os_type: OsType::Linux, // TODO: 多系统支持?
|
||||
statistics: usage,
|
||||
timestamp: std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH)?.as_millis() as u64,
|
||||
timestamp: std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)?
|
||||
.as_millis() as u64,
|
||||
};
|
||||
|
||||
let usage_json = serde_json::to_string(&usage_info)?;
|
||||
@@ -75,7 +80,8 @@ pub async fn monitor(name: &String, interval_ms: Option<u64>, settings: Config)
|
||||
}
|
||||
|
||||
async fn send_usage_info<S>(stream: &mut S, usage_json: &str) -> anyhow::Result<()>
|
||||
where S: tokio::io::AsyncWrite + Unpin
|
||||
where
|
||||
S: tokio::io::AsyncWrite + Unpin,
|
||||
{
|
||||
stream.write(usage_json.as_bytes()).await?;
|
||||
stream.write_all(b"\n").await?;
|
||||
@@ -1,14 +1,19 @@
|
||||
use cgroups_rs;
|
||||
use cgroups_rs::{fs::{cpu::CpuController, memory::MemController, pid::PidController}};
|
||||
use log;
|
||||
use cgroups_rs::fs::Cgroup;
|
||||
use std::time::{Instant, Duration};
|
||||
use crate::monitor::UsageStats;
|
||||
use cgroups_rs;
|
||||
use cgroups_rs::fs::Cgroup;
|
||||
use cgroups_rs::fs::{cpu::CpuController, memory::MemController, pid::PidController};
|
||||
use log;
|
||||
use std::time::{Duration, Instant};
|
||||
#[cfg(target_os = "linux")]
|
||||
pub fn get_usage(name: &String, usage_prev: crate::monitor::UsageStats) -> anyhow::Result<crate::monitor::UsageStats> {
|
||||
pub fn get_usage(
|
||||
name: &String,
|
||||
usage_prev: crate::monitor::UsageStats,
|
||||
) -> anyhow::Result<crate::monitor::UsageStats> {
|
||||
let hierarchy = cgroups_rs::fs::hierarchies::auto();
|
||||
if !hierarchy.v2() {
|
||||
return Err(anyhow::anyhow!("Unsupported cgroup hierarchy, cgroups v2 required."));
|
||||
return Err(anyhow::anyhow!(
|
||||
"Unsupported cgroup hierarchy, cgroups v2 required."
|
||||
));
|
||||
}
|
||||
let cgroup = Cgroup::load(hierarchy, name);
|
||||
let mut cpu_usage_us: u64 = 0;
|
||||
@@ -20,8 +25,7 @@ pub fn get_usage(name: &String, usage_prev: crate::monitor::UsageStats) -> anyho
|
||||
if let Some(cpu) = cgroup.controller_of::<CpuController>() {
|
||||
let stats = parse_cpustat(&cpu.cpu().stat.as_str())?;
|
||||
cpu_usage_us = stats.usage_usec;
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
log::warn!("Failed to fetch CPU usage info!");
|
||||
}
|
||||
|
||||
@@ -30,14 +34,11 @@ pub fn get_usage(name: &String, usage_prev: crate::monitor::UsageStats) -> anyho
|
||||
let delta_wall = current - usage_prev._timestamp;
|
||||
if usage_prev._timestamp > current {
|
||||
log::warn!("Usage stats are from the future!");
|
||||
}
|
||||
else if usage_prev.cpu_usage_us == 0 {
|
||||
} else if usage_prev.cpu_usage_us == 0 {
|
||||
log::info!("Previous CPU usage is zero, probably a fresh start.");
|
||||
}
|
||||
else if delta_wall < Duration::from_millis(1) {
|
||||
} else if delta_wall < Duration::from_millis(1) {
|
||||
log::warn!("Usage stats are too close together!");
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
cpu_usage_percent = delta_usage as f64 / delta_wall.as_micros() as f64 * 100.0;
|
||||
// dbg!(&delta_usage, &delta_wall);
|
||||
}
|
||||
@@ -90,9 +91,14 @@ fn parse_cpustat(stat: &str) -> anyhow::Result<CgroupCpuStat> {
|
||||
.split_once(' ')
|
||||
.ok_or_else(|| anyhow::anyhow!("Invalid format:{}", line))?;
|
||||
|
||||
let value = value_str
|
||||
.parse::<u64>()
|
||||
.map_err(|e| anyhow::anyhow!("Cannot convert value {}({}) into integer: {}", key, value_str, e))?;
|
||||
let value = value_str.parse::<u64>().map_err(|e| {
|
||||
anyhow::anyhow!(
|
||||
"Cannot convert value {}({}) into integer: {}",
|
||||
key,
|
||||
value_str,
|
||||
e
|
||||
)
|
||||
})?;
|
||||
|
||||
match key {
|
||||
"usage_usec" => cpu_stat.usage_usec = value,
|
||||
@@ -105,7 +111,7 @@ fn parse_cpustat(stat: &str) -> anyhow::Result<CgroupCpuStat> {
|
||||
"throttled_usec" => cpu_stat.throttled_usec = value,
|
||||
"nr_bursts" => cpu_stat.nr_bursts = value,
|
||||
"burst_usec" => cpu_stat.burst_usec = value,
|
||||
_unknown_key => {}, // Ignore unknown keys
|
||||
_unknown_key => {} // Ignore unknown keys
|
||||
}
|
||||
}
|
||||
Ok(cpu_stat)
|
||||
@@ -4,8 +4,10 @@ use regex::Regex;
|
||||
use shlex::Shlex;
|
||||
|
||||
lazy_static! {
|
||||
static ref FUNCTION_REGEX: Regex = Regex::new(r"^\s*(function\s*)([a-zA-Z0-9_-]+)\s*(\(\))?").unwrap();
|
||||
static ref FUNCTION_REGEX_WITHOUT_KEYWORD: Regex = Regex::new(r"^\s*([a-zA-Z0-9_-]+)\s*(\(\))").unwrap();
|
||||
static ref FUNCTION_REGEX: Regex =
|
||||
Regex::new(r"^\s*(function\s*)([a-zA-Z0-9_-]+)\s*(\(\))?").unwrap();
|
||||
static ref FUNCTION_REGEX_WITHOUT_KEYWORD: Regex =
|
||||
Regex::new(r"^\s*([a-zA-Z0-9_-]+)\s*(\(\))").unwrap();
|
||||
static ref DECORATOR_HEADER_REGEX: Regex = Regex::new(r"\s*#\s+@").unwrap();
|
||||
}
|
||||
|
||||
@@ -23,7 +25,6 @@ impl Function {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
pub fn parse_script(text: String) -> Result<Vec<Function>, ()> {
|
||||
let mut functions: Vec<Function> = Vec::new();
|
||||
let mut current_decorators: Vec<Decorator> = Vec::new();
|
||||
@@ -38,7 +39,8 @@ pub fn parse_script(text: String) -> Result<Vec<Function>, ()>{
|
||||
current_body.push_str(line);
|
||||
current_body.push('\n');
|
||||
let tokens = Shlex::new(line).collect::<Vec<String>>();
|
||||
let brace_delta: i8 = tokens.iter().filter(|&token| token == "{").count() as i8 - tokens.iter().filter(|&token| token == "}").count() as i8;
|
||||
let brace_delta: i8 = tokens.iter().filter(|&token| token == "{").count() as i8
|
||||
- tokens.iter().filter(|&token| token == "}").count() as i8;
|
||||
brace_count += brace_delta;
|
||||
if brace_count == 0 {
|
||||
functions.push(Function {
|
||||
@@ -51,25 +53,27 @@ pub fn parse_script(text: String) -> Result<Vec<Function>, ()>{
|
||||
current_body.clear();
|
||||
in_function = false;
|
||||
}
|
||||
continue
|
||||
continue;
|
||||
}
|
||||
if DECORATOR_HEADER_REGEX.find(line).is_some() {
|
||||
if let Some(decorator) = parse_decorator(line) {
|
||||
current_decorators.push(decorator);
|
||||
}
|
||||
continue
|
||||
continue;
|
||||
}
|
||||
if FUNCTION_REGEX.find(line).is_some() || FUNCTION_REGEX_WITHOUT_KEYWORD.find(line).is_some() {
|
||||
if FUNCTION_REGEX.find(line).is_some()
|
||||
|| FUNCTION_REGEX_WITHOUT_KEYWORD.find(line).is_some()
|
||||
{
|
||||
let tokens = Shlex::new(line).collect::<Vec<String>>();
|
||||
// dbg!(&tokens);
|
||||
if tokens[0] != "function" {
|
||||
if FUNCTION_REGEX_WITHOUT_KEYWORD.find(line).is_none() {
|
||||
continue
|
||||
continue;
|
||||
}
|
||||
in_function = true;
|
||||
current_function_name = FUNCTION_REGEX_WITHOUT_KEYWORD.captures(line).unwrap()[1].to_string();
|
||||
}
|
||||
else{
|
||||
current_function_name =
|
||||
FUNCTION_REGEX_WITHOUT_KEYWORD.captures(line).unwrap()[1].to_string();
|
||||
} else {
|
||||
in_function = true;
|
||||
current_function_name = FUNCTION_REGEX.captures(line).unwrap()[2].to_string();
|
||||
}
|
||||
@@ -91,7 +95,7 @@ pub fn parse_script(text: String) -> Result<Vec<Function>, ()>{
|
||||
current_body.clear();
|
||||
in_function = false;
|
||||
}
|
||||
continue
|
||||
continue;
|
||||
}
|
||||
current_decorators.clear();
|
||||
in_function = false;
|
||||
@@ -204,6 +208,11 @@ mod tests {
|
||||
assert!(final_func.body.contains("最后函数"));
|
||||
|
||||
// 确保无效函数没有被解析
|
||||
assert!(functions.iter().find(|f| f.name == "another_invalid-function").is_none());
|
||||
assert!(
|
||||
functions
|
||||
.iter()
|
||||
.find(|f| f.name == "another_invalid-function")
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
pub mod config;
|
||||
pub mod env;
|
||||
pub mod security;
|
||||
pub mod stage;
|
||||
|
||||
use crate::cli::Cli;
|
||||
|
||||
pub use config::PrebakeConfig;
|
||||
pub use security::prebake_drop_privilege;
|
||||
|
||||
pub fn parse_prebake_config(yaml_content: &str) -> Result<PrebakeConfig, serde_yaml::Error> {
|
||||
serde_yaml::from_str(yaml_content)
|
||||
}
|
||||
|
||||
pub fn serialize_prebake_config(config: &PrebakeConfig) -> Result<String, serde_yaml::Error> {
|
||||
serde_yaml::to_string(config)
|
||||
}
|
||||
|
||||
pub async fn prebake(config_path: &str, cli: &Cli) -> anyhow::Result<()> {
|
||||
let config_content = std::fs::read_to_string(config_path)?;
|
||||
let config = parse_prebake_config(config_content.as_str())?;
|
||||
let result = config.validate();
|
||||
if let Err(errors) = result {
|
||||
for error in errors {
|
||||
log::error!("Error: {}", error);
|
||||
}
|
||||
// TODO: Raise an error
|
||||
}
|
||||
|
||||
let mut ctx = crate::ExecutionContext {
|
||||
task_id: format!("{}-{}-init", cli.pipeline, cli.build_id),
|
||||
pipeline_name: cli.pipeline.clone(),
|
||||
username: cli.username.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
// Wait for bakerd to finish Fetch-1
|
||||
|
||||
// Bootstrap stage
|
||||
ctx.task_id = format!("{}-{}-bootstrap", cli.pipeline, cli.build_id);
|
||||
// TODO: Pass real event_tx
|
||||
let result = stage::bootstrap::bootstrap(&config, &ctx, &None).await;
|
||||
result?;
|
||||
// TODO: Early security implementation
|
||||
|
||||
// EarlyHook stage
|
||||
ctx.task_id = format!("{}-{}-earlyhook", cli.pipeline, cli.build_id);
|
||||
let earlyhook = match &config.hooks {
|
||||
Some(hooks) => hooks.early.clone(),
|
||||
None => None,
|
||||
};
|
||||
let result = stage::hook::hook(earlyhook, &ctx, None).await;
|
||||
result?;
|
||||
|
||||
// DepsSystem stage
|
||||
ctx.task_id = format!("{}-{}-depssystem", cli.pipeline, cli.build_id);
|
||||
// TODO: Dependency stage
|
||||
// DepsUser stage
|
||||
ctx.task_id = format!("{}-{}-depsuser", cli.pipeline, cli.build_id);
|
||||
// TODO: Dependency stage
|
||||
|
||||
// LateHook stage
|
||||
ctx.task_id = format!("{}-{}-latehook", cli.pipeline, cli.build_id);
|
||||
let latehook = match &config.hooks {
|
||||
Some(hooks) => hooks.late.clone(),
|
||||
None => None,
|
||||
};
|
||||
let result = stage::hook::hook(latehook, &ctx, None).await;
|
||||
result?;
|
||||
|
||||
// Ready stage
|
||||
ctx.task_id = format!("{}-{}-ready", cli.pipeline, cli.build_id);
|
||||
// TODO: Security stage
|
||||
|
||||
// TODO: Ready here!
|
||||
todo!();
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
use semver::{Version, VersionReq};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_yaml::Value;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::str::FromStr;
|
||||
|
||||
use crate::types::architecture::Architecture;
|
||||
use crate::types::builderconfig::{
|
||||
BaremetalConfig, BuilderType, CustomConfig, DockerConfig, FirecrackerConfig,
|
||||
};
|
||||
use crate::types::cache::{CacheDirectory, CacheStrategy};
|
||||
use crate::types::command::CustomCommand;
|
||||
use crate::types::memsize::MemSize;
|
||||
|
||||
pub const VERSION_REQUIREMENT: &str = "^1.0";
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct PrebakeConfig {
|
||||
pub version: String,
|
||||
pub environment: Environment,
|
||||
#[serde(default)]
|
||||
pub dependencies: Option<Dependencies>,
|
||||
#[serde(default)]
|
||||
pub envvars: Option<EnvVars>,
|
||||
#[serde(default)]
|
||||
pub cache: Option<Cache>,
|
||||
#[serde(default)]
|
||||
pub security: Option<Security>,
|
||||
#[serde(default)]
|
||||
pub hooks: Option<Hooks>,
|
||||
#[serde(default)]
|
||||
pub metadata: Option<Metadata>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct Environment {
|
||||
pub builder: BuilderType,
|
||||
#[serde(default)]
|
||||
pub baremetal: Option<BaremetalConfig>,
|
||||
#[serde(default)]
|
||||
pub docker: Option<DockerConfig>,
|
||||
#[serde(default)]
|
||||
pub firecracker: Option<FirecrackerConfig>,
|
||||
#[serde(default)]
|
||||
pub custom: Option<CustomConfig>,
|
||||
#[serde(default)]
|
||||
pub resources: Option<ResourceRequirements>,
|
||||
#[serde(default)]
|
||||
pub network: Option<NetworkConfig>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct ResourceRequirements {
|
||||
#[serde(default = "default_cpu")]
|
||||
pub cpu: u32,
|
||||
#[serde(default = "default_memory")]
|
||||
pub memory: MemSize,
|
||||
#[serde(default = "default_disk")]
|
||||
pub disk: MemSize,
|
||||
#[serde(
|
||||
default,
|
||||
deserialize_with = "parse_architecture",
|
||||
serialize_with = "serialize_architecture"
|
||||
)]
|
||||
pub architecture: HashSet<Architecture>,
|
||||
// TODO: Design GPU config
|
||||
// #[serde(default = "default_gpu")]
|
||||
// pub gpu: bool,
|
||||
// #[serde(default)]
|
||||
// pub gpu_type: Option<Vec<GpuType>>,
|
||||
#[serde(default)]
|
||||
pub tags: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
fn default_cpu() -> u32 {
|
||||
1
|
||||
}
|
||||
fn default_memory() -> MemSize {
|
||||
MemSize::from_gib(1)
|
||||
}
|
||||
fn default_disk() -> MemSize {
|
||||
MemSize::from_gib(1)
|
||||
}
|
||||
|
||||
fn parse_architecture<'de, D>(deserializer: D) -> Result<HashSet<Architecture>, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
use serde::de::{Error, Visitor};
|
||||
use std::fmt;
|
||||
|
||||
struct ArchVisitor;
|
||||
|
||||
impl<'de> Visitor<'de> for ArchVisitor {
|
||||
type Value = HashSet<Architecture>;
|
||||
|
||||
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
|
||||
formatter.write_str("a string or an array of strings")
|
||||
}
|
||||
|
||||
fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
|
||||
where
|
||||
E: Error,
|
||||
{
|
||||
let arch = Architecture::from_str(v).map_err(E::custom)?;
|
||||
let mut set = HashSet::new();
|
||||
set.insert(arch);
|
||||
Ok(set)
|
||||
}
|
||||
|
||||
fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
|
||||
where
|
||||
A: serde::de::SeqAccess<'de>,
|
||||
{
|
||||
let mut set = HashSet::new();
|
||||
while let Some(s) = seq.next_element::<String>()? {
|
||||
let arch = Architecture::from_str(&s).map_err(A::Error::custom)?;
|
||||
set.insert(arch);
|
||||
}
|
||||
Ok(set)
|
||||
}
|
||||
}
|
||||
|
||||
deserializer.deserialize_any(ArchVisitor)
|
||||
}
|
||||
|
||||
fn serialize_architecture<S>(
|
||||
value: &HashSet<Architecture>,
|
||||
serializer: S,
|
||||
) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
use serde::ser::SerializeSeq;
|
||||
|
||||
let mut seq = serializer.serialize_seq(Some(value.len()))?;
|
||||
for arch in value {
|
||||
seq.serialize_element(arch.as_str())?;
|
||||
}
|
||||
seq.end()
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct NetworkConfig {
|
||||
#[serde(default = "default_enabled")]
|
||||
pub enabled: bool,
|
||||
#[serde(default = "default_outbound")]
|
||||
pub outbound: bool,
|
||||
#[serde(default)]
|
||||
pub dns_servers: Option<Vec<String>>,
|
||||
#[serde(default)]
|
||||
pub proxies: Option<Vec<ProxyConfig>>,
|
||||
}
|
||||
|
||||
fn default_enabled() -> bool {
|
||||
true
|
||||
}
|
||||
fn default_outbound() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct ProxyConfig {}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct Dependencies {
|
||||
#[serde(default)]
|
||||
pub system: Option<HashMap<String, SystemDependency>>,
|
||||
#[serde(default)]
|
||||
pub user: Option<HashMap<String, Value>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct SystemDependency {
|
||||
pub packages: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub repositories: Option<Value>,
|
||||
#[serde(default)]
|
||||
pub mirror: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct EnvVars {
|
||||
#[serde(default)]
|
||||
pub prebake: Option<HashMap<String, String>>,
|
||||
#[serde(default)]
|
||||
pub bake: Option<HashMap<String, String>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct Cache {
|
||||
pub directory: Vec<CacheDirectory>,
|
||||
#[serde(default)]
|
||||
pub strategy: Option<CacheStrategy>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct Security {} // TODO: Security module
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct Hooks {
|
||||
#[serde(default)]
|
||||
pub early: Option<Vec<CustomCommand>>,
|
||||
#[serde(default)]
|
||||
pub late: Option<Vec<CustomCommand>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct Metadata {
|
||||
#[serde(default)]
|
||||
pub description: Option<String>,
|
||||
#[serde(default)]
|
||||
pub maintainer: Option<String>,
|
||||
}
|
||||
|
||||
impl PrebakeConfig {
|
||||
pub fn validate(&self) -> Result<(), Vec<String>> {
|
||||
let mut errors = Vec::new();
|
||||
|
||||
let version_requirement =
|
||||
VersionReq::parse(VERSION_REQUIREMENT).expect("Invalid version requirement");
|
||||
match Version::parse(&self.version) {
|
||||
Ok(version) if version_requirement.matches(&version) => {}
|
||||
Ok(version) => errors.push(format!(
|
||||
"Version {} does not satisfy requirement {}",
|
||||
version, VERSION_REQUIREMENT
|
||||
)),
|
||||
Err(e) => errors.push(format!("Invalid version format: {}", e)),
|
||||
}
|
||||
|
||||
// TODO: Schema based validation
|
||||
|
||||
if errors.is_empty() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(errors)
|
||||
}
|
||||
}
|
||||
}
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
use crate::prebake::config::PrebakeConfig;
|
||||
use config::Config;
|
||||
use log::info;
|
||||
|
||||
pub async fn prepare_baremetal(_config: &PrebakeConfig, _settings: &Config) -> anyhow::Result<()> {
|
||||
info!("Nothing to be done for baremetal environment.");
|
||||
Ok(())
|
||||
}
|
||||
+44
-78
@@ -1,51 +1,24 @@
|
||||
use crate::prebake::*;
|
||||
use crate::prebake::config::PrebakeConfig;
|
||||
use crate::types::builderconfig::DockerConfig;
|
||||
use anyhow::Context;
|
||||
use bollard::{
|
||||
API_DEFAULT_VERSION, Docker, body_full, query_parameters::{BuildImageOptionsBuilder, CreateImageOptionsBuilder}, secret::{BuildInfo, CreateImageInfo, ProgressDetail}
|
||||
API_DEFAULT_VERSION, Docker, body_full,
|
||||
query_parameters::{BuildImageOptionsBuilder, CreateImageOptionsBuilder},
|
||||
secret::{BuildInfo, CreateImageInfo},
|
||||
};
|
||||
use config::Config;
|
||||
use futures_util::stream::StreamExt;
|
||||
// use futures_util::stream::TryStreamExt;
|
||||
use log::{debug, error, info, warn};
|
||||
use uuid::Uuid;
|
||||
use tar;
|
||||
use std::fs;
|
||||
use tar;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[derive(Debug)]
|
||||
enum ContainerBuildResult {
|
||||
PullResults(CreateImageInfo),
|
||||
BuildResults(BuildInfo),
|
||||
}
|
||||
|
||||
pub async fn prepare_environment(config: &PrebakeConfig, settings: &Config) -> anyhow::Result<()> {
|
||||
match config.environment.builder {
|
||||
BuilderType::Baremetal => {
|
||||
info!("Nothing to be done for baremetal environment.");
|
||||
}
|
||||
BuilderType::Docker => {
|
||||
let container = container(config, settings).await?;
|
||||
dbg!(container);
|
||||
}
|
||||
BuilderType::Firecracker => {
|
||||
error!("Firecracker is not supported yet.");
|
||||
}
|
||||
BuilderType::Custom => {
|
||||
custom(config, settings)?;
|
||||
}
|
||||
}
|
||||
todo!();
|
||||
}
|
||||
|
||||
async fn container(config: &PrebakeConfig, settings: &Config) -> anyhow::Result<String> {
|
||||
let container_options = match config.environment.config.as_ref() {
|
||||
Some(BuilderConfig::Docker(docker)) => docker,
|
||||
Some(_) => {
|
||||
// No way it is going to happen
|
||||
return Err(anyhow::anyhow!(
|
||||
"Unexpected builder configuration: {:?}",
|
||||
config.environment.builder
|
||||
));
|
||||
}
|
||||
pub async fn prepare_container(
|
||||
config: &PrebakeConfig,
|
||||
settings: &Config,
|
||||
) -> anyhow::Result<String> {
|
||||
let container_options = match config.environment.docker.as_ref() {
|
||||
Some(docker) => docker,
|
||||
None => {
|
||||
return Err(anyhow::anyhow!("Missing Docker configuration"));
|
||||
}
|
||||
@@ -74,17 +47,13 @@ async fn container(config: &PrebakeConfig, settings: &Config) -> anyhow::Result<
|
||||
};
|
||||
|
||||
if container_options.image.is_none() {
|
||||
// Using Dockerfile
|
||||
return container_build(container_options, container_connection, settings).await;
|
||||
} else {
|
||||
// Pulling from repo
|
||||
return container_pull(container_options, container_connection).await;
|
||||
}
|
||||
|
||||
// todo!();
|
||||
}
|
||||
|
||||
fn single_dockerfile_tar(dockerfile: &str) -> anyhow::Result<Vec<u8>> {
|
||||
fn create_dockerfile_tar(dockerfile: &str) -> anyhow::Result<Vec<u8>> {
|
||||
let mut tar_buffer = Vec::new();
|
||||
let mut tar_builder = tar::Builder::new(&mut tar_buffer);
|
||||
let dockerfile_content = fs::read_to_string(dockerfile)?;
|
||||
@@ -96,9 +65,7 @@ fn single_dockerfile_tar(dockerfile: &str) -> anyhow::Result<Vec<u8>> {
|
||||
header.set_cksum();
|
||||
tar_builder.append(&header, dockerfile_content.as_bytes())?;
|
||||
tar_builder.finish()?;
|
||||
|
||||
tar_builder.into_inner()?;
|
||||
|
||||
Ok(tar_buffer)
|
||||
}
|
||||
|
||||
@@ -108,10 +75,13 @@ async fn container_build(
|
||||
settings: &Config,
|
||||
) -> anyhow::Result<String> {
|
||||
info!("Building image at Prebake Stage 1...");
|
||||
// Build process may fail at this time, it is somehow acceptable.
|
||||
let dockerfile = container_options.dockerfile.as_ref().unwrap(); // No way to fail
|
||||
let tag = format!("HoneyBiscuitWorkshop/{}_{}", settings.get_string("pipeline").unwrap().as_str(), Uuid::new_v4());
|
||||
let tar_buffer = match single_dockerfile_tar(dockerfile) {
|
||||
let dockerfile = container_options.dockerfile.as_ref().unwrap();
|
||||
let tag = format!(
|
||||
"HoneyBiscuitWorkshop/{}_{}",
|
||||
settings.get_string("pipeline").unwrap().as_str(),
|
||||
Uuid::new_v4()
|
||||
);
|
||||
let tar_buffer = match create_dockerfile_tar(dockerfile) {
|
||||
Ok(buffer) => buffer,
|
||||
Err(error) => {
|
||||
if error.to_string().contains("No such file") {
|
||||
@@ -127,25 +97,32 @@ async fn container_build(
|
||||
.dockerfile("Dockerfile")
|
||||
.t(&tag)
|
||||
.build();
|
||||
let mut build_result = container_connection.build_image(build_options, None, Some(body_full(tar_buffer.into())));
|
||||
let mut build_result =
|
||||
container_connection.build_image(build_options, None, Some(body_full(tar_buffer.into())));
|
||||
while let Some(stream_item) = build_result.next().await {
|
||||
let build_info = stream_item.context("Unexpected build result.");
|
||||
dbg!(&build_info);
|
||||
debug!("{:?}", build_info);
|
||||
match build_info {
|
||||
Ok(BuildInfo { id, stream, error_detail, status, progress_detail, aux }) => {},
|
||||
Ok(BuildInfo {
|
||||
id: _,
|
||||
stream: _,
|
||||
error_detail: _,
|
||||
status: _,
|
||||
progress_detail: _,
|
||||
aux: _,
|
||||
}) => {}
|
||||
Err(error) => {
|
||||
let error_text = error.chain()
|
||||
let error_text = error
|
||||
.chain()
|
||||
.map(|err| err.to_string())
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ");
|
||||
error!("Failed to build image: {}", error_text);
|
||||
|
||||
return Err(error);
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
info!("Successfully built the image {}", tag);
|
||||
|
||||
Ok(tag)
|
||||
}
|
||||
|
||||
@@ -161,26 +138,21 @@ async fn container_pull(
|
||||
let pull_options = CreateImageOptionsBuilder::default()
|
||||
.from_image(image_name)
|
||||
.build();
|
||||
// TODO(@Catty2014): 2026-03-09 - Implement credentials
|
||||
// Credentials are required to pull from docker private registry.
|
||||
let mut pull_result = container_connection.create_image(Some(pull_options), None, None);
|
||||
while let Some(stream_item) = pull_result.next().await {
|
||||
let pull_info = stream_item.context("Unexpected pull result.");
|
||||
// dbg!(&pull_info);
|
||||
match pull_info {
|
||||
Ok(CreateImageInfo{ id, error_detail, status, progress_detail }) => {
|
||||
// Still unknown to continue or to fail
|
||||
Ok(CreateImageInfo {
|
||||
id,
|
||||
error_detail,
|
||||
status,
|
||||
progress_detail: _,
|
||||
}) => {
|
||||
if let Some(error_detail) = error_detail {
|
||||
warn!("Error when pulling: {:?}", error_detail);
|
||||
}
|
||||
|
||||
if let Some(status) = status {
|
||||
if status.contains("Downloading") || status.contains("Extracting") {
|
||||
// if let Some(progress_detail) = progress_detail {
|
||||
// info!("{}: {:?}", status, progress_detail);
|
||||
// } else {
|
||||
// info!("{}", status);
|
||||
// }
|
||||
} else if status.contains("Pull complete") {
|
||||
info!("Layer complete: {}", id.unwrap_or_default());
|
||||
} else if status.contains("Already exists") {
|
||||
@@ -191,18 +163,16 @@ async fn container_pull(
|
||||
info!("Warning: {}", status);
|
||||
} else if status.contains("Downloaded newer image") {
|
||||
info!("Downloaded newer image: {}", image_name);
|
||||
} else {
|
||||
// info!("Status: {}", status);
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
Err(error) => {
|
||||
let error_text = error.chain()
|
||||
let error_text = error
|
||||
.chain()
|
||||
.map(|err| err.to_string())
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ");
|
||||
error!("Failed to pull image: {}", error_text);
|
||||
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
@@ -210,7 +180,3 @@ async fn container_pull(
|
||||
info!("Successfully pulled image: {}", image_name);
|
||||
Ok(image_name.clone())
|
||||
}
|
||||
|
||||
fn custom(config: &PrebakeConfig, settings: &Config) -> anyhow::Result<()> {
|
||||
todo!();
|
||||
}
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
use crate::prebake::config::PrebakeConfig;
|
||||
use config::Config;
|
||||
|
||||
pub async fn prepare_custom(_config: &PrebakeConfig, _settings: &Config) -> anyhow::Result<()> {
|
||||
todo!("Custom builder is not implemented yet")
|
||||
}
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
use crate::prebake::config::PrebakeConfig;
|
||||
use config::Config;
|
||||
use log::error;
|
||||
|
||||
pub async fn prepare_firecracker(
|
||||
_config: &PrebakeConfig,
|
||||
_settings: &Config,
|
||||
) -> anyhow::Result<()> {
|
||||
error!("Firecracker is not supported yet.");
|
||||
Err(anyhow::anyhow!("Firecracker is not supported yet"))
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
pub mod baremetal;
|
||||
pub mod container;
|
||||
pub mod custom;
|
||||
pub mod firecracker;
|
||||
@@ -0,0 +1,33 @@
|
||||
目前的prebake流程如下：
|
||||
PBStage-init:
|
||||
此时ws-agent尚在获取流水线数据，PBStage0仅仅是等待其获取流水线定义文件(prebake.yml, bake.sh, finalize.yml)并尝试优先获取重要文件（如Dockerfile）。
|
||||
ws-agent在获取完数据后还会继续获取与配置缓存。此时PBStage与ws-agent是异步的。
|
||||
ws-agent获取数据可能随时结束，也可能随时出错，此时若不可恢复，则流水线生命周期终止。
|
||||
PBStage-env:
|
||||
ws-executor于主机运行，并开始根据environment配置环境。此时可能会拉取镜像，下载img文件或完成自定义环境准备。
|
||||
特殊地，若environment=container且dockerfile被指定，容器便会开始构建，但只要其依赖流水线的其它文件（且尚未被获取到）则PBStage1暂停于此，等待ws-agent的“获取结束”信号。这样做可以保证其依赖获取(FROM)字段能被处理，节省时间。
|
||||
PBStage-bootstrap:
|
||||
ws-executor被复制到目标环境并以root运行。其会与主机ws-executor通信并获取固定的环境初始化程序（有点像cloud-init），设置一般用户Vulcan和必要的免密权限，根据环境不同可能会使用systemd-run0/sudo/直接使用root，用户也可以使用模板或手动编写的程序覆盖此行为。若条件不可满足（如尝试在busybox中使用普通用户，但没有包管理器也没有用户管理手段）则流水线终止于此。该阶段还可能设置其它必要的信息。prebake环境变量也于此时注入。
|
||||
PBStage-earlysecurity:
|
||||
ws-executor@host,ws-executor@env开始按照配置文件设置必要的早期安全环境，不过目前大概除了创建cgroups组外什么都不会发生，暂时可以忽略此步骤
|
||||
PBStage-prehook:
|
||||
ws-executor在目标环境于Vulcan（或用户指定的，或者root，下略）运行，开始预烘焙钩子。
|
||||
PBStage-snapshot:
|
||||
这是一个伪阶段，executor@host会按配置（在可能的环境）创建目标环境的快照。若不可实现则自动忽略，此时dependency阶段失败将按照配置从bootstrap或dependency自己重新开始。
|
||||
目前我们尚不清楚容器/虚拟机快照能做到哪种地步。
|
||||
我们尚未设计这个字段retry-policy，其应该在prebake.yml的dependency中。
|
||||
PBStage-dependency:
|
||||
理论上，若存在缓存，则ws-executor@host必须等待ws-agent结束，ws-agent发射结束信号时，其确保获取了文件，同步到共享分区并（按配置）检查过。
|
||||
实际上可能会让依赖自行认领cache，不被认领的cache认为是bake cache从而允许忽略
|
||||
ws-executor在目标环境于Vulcan，从custom_pre到system到language到custom完成依赖安装动作
|
||||
用户可以配置可选的custom_*的清除脚本，使得custom+custom_clear是幂等的
|
||||
（如果不配置且存在不幂等的风险，用户应当自行设置合理的retry-poicy）
|
||||
PBStage-posthook:
|
||||
开始后依赖钩子
|
||||
PBStage-security:
|
||||
ws-executor@host,ws-executor@env开始按照配置文件设置必要的安全环境，不过目前大概什么都不会发生，暂时可以忽略此步骤
|
||||
PBStage-readyhook:
|
||||
开始完成时钩子
|
||||
PBStage-ready:
|
||||
完成bake环境变量的注入，标识准备结束，ws-executor@env会告知ws-executor@host可以开始bake了。
|
||||
这样编排好吗？可能有什么问题或遗漏吗？
|
||||
@@ -0,0 +1,221 @@
|
||||
use crate::error::PrebakeError;
|
||||
use crate::prebake::stage::PrebakeStage;
|
||||
use privdrop::PrivDrop;
|
||||
use std::ffi::CString;
|
||||
use std::mem::MaybeUninit;
|
||||
|
||||
const ROOT_UID: u32 = 0;
|
||||
|
||||
/// Resolves a username to its corresponding UID (User ID).
|
||||
///
|
||||
/// This function performs a thread-safe lookup of user information using the
|
||||
/// `getpwnam_r` system call, which is the reentrant version of `getpwnam`.
|
||||
/// It first attempts to find the user in the system password database,
|
||||
/// and if not found, falls back to parsing the input as a numeric UID.
|
||||
///
|
||||
/// # Security Model
|
||||
///
|
||||
/// This function is part of the privilege management system. It is used
|
||||
/// to resolve usernames to UIDs before privilege dropping operations,
|
||||
/// ensuring that processes can transition to the correct user identity.
|
||||
///
|
||||
/// # Parameters
|
||||
///
|
||||
/// * `username` - The username to look up. Can be either:
|
||||
/// - A textual username (e.g., `"vulcan"`, `"nobody"`)
|
||||
/// - A numeric UID string (e.g., `"1000"`) for fallback resolution
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Returns `Result<u32, PrebakeError>`:
|
||||
/// * `Ok(u32)` - The resolved UID on success
|
||||
/// * `Err(PrebakeError::UserNotFound)` - User does not exist in the system
|
||||
/// database and is not a valid numeric UID
|
||||
///
|
||||
/// # Error Handling
|
||||
///
|
||||
/// This function handles several error cases:
|
||||
/// 1. **Invalid username format**: If the username contains a null byte,
|
||||
/// `CString::new` will fail, returning `UserNotFound`.
|
||||
/// 2. **Buffer too small**: If `getpwnam_r` returns `ERANGE`, the buffer
|
||||
/// is dynamically resized and the lookup is retried.
|
||||
/// 3. **User not found**: If `getpwnam_r` returns `0` with a null pointer,
|
||||
/// the function attempts to parse `username` as a numeric UID.
|
||||
/// If that also fails, `UserNotFound` is returned.
|
||||
///
|
||||
/// # Thread Safety
|
||||
///
|
||||
/// This function is thread-safe because it uses `getpwnam_r` instead of
|
||||
/// the non-reentrant `getpwnam`. The `getpwnam_r` function stores its
|
||||
/// result in the caller-provided buffer, avoiding race conditions in
|
||||
/// multi-threaded contexts.
|
||||
///
|
||||
/// # Implementation Details
|
||||
///
|
||||
/// The function uses a dynamically growing buffer strategy:
|
||||
/// 1. Start with a 4KB buffer (`bufsize = 4096`)
|
||||
/// 2. If `getpwnam_r` returns `ERANGE` (buffer too small), double the buffer
|
||||
/// 3. Retry until the buffer is large enough or a non-ERANGE error occurs
|
||||
///
|
||||
/// If the user is not found in the system database, the function attempts
|
||||
/// to parse the `username` string directly as a numeric UID. This allows
|
||||
/// configurations to specify UIDs directly (e.g., `"0"` for root) without
|
||||
/// relying on the system's user database.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```rust,ignore
|
||||
/// // Resolve a username to UID
|
||||
/// match lookup_uid_by_name("vulcan") {
|
||||
/// Ok(uid) => println!("User vulcan has UID {}", uid),
|
||||
/// Err(e) => eprintln!("Failed to resolve user: {}", e),
|
||||
/// }
|
||||
///
|
||||
/// // Resolve a numeric UID string (fallback)
|
||||
/// match lookup_uid_by_name("1000") {
|
||||
/// Ok(uid) => println!("UID 1000 resolved to {}", uid),
|
||||
/// Err(e) => eprintln!("Invalid UID: {}", e),
|
||||
/// }
|
||||
/// ```
|
||||
///
|
||||
/// # See Also
|
||||
///
|
||||
/// * [`prebake_drop_privilege`] - Uses this function to resolve target user before dropping privileges
|
||||
/// * [`PrivDrop`](privdrop::PrivDrop) - The privilege dropping utility used after UID resolution
|
||||
pub fn lookup_uid_by_name(username: &str) -> Result<u32, PrebakeError> {
|
||||
let c_username = CString::new(username)
|
||||
.map_err(|_| PrebakeError::UserNotFound(format!("Invalid username: {}", username)))?;
|
||||
|
||||
let mut pwd = MaybeUninit::<libc::passwd>::uninit();
|
||||
let mut pwent = std::ptr::null_mut::<libc::passwd>();
|
||||
let mut bufsize: usize = 4096;
|
||||
let mut pwbuf = vec![0; bufsize];
|
||||
|
||||
loop {
|
||||
let ret = unsafe {
|
||||
libc::getpwnam_r(
|
||||
c_username.as_ptr(),
|
||||
pwd.as_mut_ptr(),
|
||||
pwbuf.as_mut_ptr() as *mut libc::c_char,
|
||||
pwbuf.len(),
|
||||
&mut pwent,
|
||||
)
|
||||
};
|
||||
|
||||
if ret == libc::ERANGE {
|
||||
bufsize *= 2;
|
||||
pwbuf.resize(bufsize, 0);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: originally unsafe {pwent} here, maybe to check and remove
|
||||
// let pwent = pwent ;
|
||||
if pwent.is_null() {
|
||||
if let Ok(uid) = username.parse::<u32>() {
|
||||
return Ok(uid);
|
||||
}
|
||||
return Err(PrebakeError::UserNotFound(format!(
|
||||
"User '{}' not found and not a valid numeric UID",
|
||||
username
|
||||
)));
|
||||
}
|
||||
|
||||
let uid = unsafe { (*pwent).pw_uid };
|
||||
Ok(uid)
|
||||
}
|
||||
|
||||
/// Drops process privileges if the current execution stage exceeds the allowed privilege boundary.
|
||||
///
|
||||
/// # Security Model
|
||||
/// This function acts as a **privilege gate**. It ensures that once the pipeline progresses beyond
|
||||
/// a certain stage (`stage_expected`), the process must no longer run with elevated (Root) privileges.
|
||||
///
|
||||
/// # Behavior
|
||||
/// - **Conditional Execution**: Only attempts to drop privileges if `stage_current > stage_expected`.
|
||||
/// - **Idempotent**: Safe to call multiple times. If the process is already running as the target user,
|
||||
/// it returns `Ok(())` immediately.
|
||||
/// - **State Validation**:
|
||||
/// - If running as Root: Attempts to switch to the target user.
|
||||
/// - If running as Target User: Returns success (no-op).
|
||||
/// - If running as Other Non-Root User: Returns `Err(InvalidPrivilegeState)` because privilege
|
||||
/// dropping is impossible from this state (you cannot switch users without Root).
|
||||
///
|
||||
/// # Logic Flow
|
||||
/// 1. Check if `stage_current` exceeds `stage_expected`. If not, return `Ok(())` immediately.
|
||||
/// 2. Resolve the target username to a UID.
|
||||
/// 3. Compare Current Effective UID vs. Target UID:
|
||||
/// - **Equal**: Log info and return `Ok(())`.
|
||||
/// - **Current is Root (0)**: Proceed to drop privileges using `PrivDrop`.
|
||||
/// - **Current is Neither**: Log error and return `Err(InvalidPrivilegeState)`.
|
||||
/// 4. Verify the new UID after dropping (sanity check) and log success.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `stage_current`: The pipeline stage currently being executed.
|
||||
/// * `stage_expected`: The last stage allowed to run with elevated privileges.
|
||||
/// * `user`: The target username to drop privileges to (e.g., "nobody", "builder").
|
||||
///
|
||||
/// # Returns
|
||||
/// - `Ok(())`: Privileges were successfully dropped, or were already correct.
|
||||
/// - `Err(PrebakeError)`:
|
||||
/// - `UserNotFound`: The target username does not exist.
|
||||
/// - `InvalidPrivilegeState`: Process is running as an unexpected non-root user.
|
||||
/// - `PrivDropFailed`: System call to change user failed.
|
||||
///
|
||||
/// # Caller Responsibility
|
||||
/// The caller must ensure that if a privilege drop is required, this function is invoked
|
||||
/// while the process still holds Root privileges. The caller is also responsible for mapping
|
||||
/// the returned `Err` to an appropriate exit code (e.g., 201) if termination is desired.
|
||||
pub fn prebake_drop_privilege(
|
||||
stage_current: PrebakeStage,
|
||||
stage_expected: PrebakeStage,
|
||||
user: &str,
|
||||
) -> Result<(), PrebakeError> {
|
||||
if stage_current > stage_expected {
|
||||
let current_uid = unsafe { libc::geteuid() };
|
||||
|
||||
let target_uid = lookup_uid_by_name(user)?;
|
||||
|
||||
if current_uid == target_uid {
|
||||
log::info!(
|
||||
"Already running as target user (UID {}), skipping privilege drop",
|
||||
target_uid
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if current_uid != ROOT_UID {
|
||||
log::error!(
|
||||
"Cannot drop privileges: current UID={} is neither Root (0) nor target UID={}",
|
||||
current_uid,
|
||||
target_uid
|
||||
);
|
||||
log::error!("Aborting pipeline.");
|
||||
return Err(PrebakeError::InvalidPrivilegeState {
|
||||
current_uid,
|
||||
target_uid,
|
||||
});
|
||||
}
|
||||
|
||||
PrivDrop::default().user(user).apply().map_err(|e| {
|
||||
log::error!(
|
||||
"Failed to drop privileges to {} at stage {} (after {}): {}",
|
||||
user,
|
||||
stage_current,
|
||||
stage_expected,
|
||||
e
|
||||
);
|
||||
log::error!("Aborting pipeline.");
|
||||
PrebakeError::PrivDropFailed(e.to_string())
|
||||
})?;
|
||||
|
||||
let new_uid = unsafe { libc::geteuid() };
|
||||
log::info!(
|
||||
"Successfully dropped privileges from Root to UID {} (user: {})",
|
||||
new_uid,
|
||||
user
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
pub mod bootstrap;
|
||||
pub mod environment;
|
||||
pub mod hook;
|
||||
pub mod depssystem;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub enum PrebakeStage {
|
||||
Init,
|
||||
Bootstrap,
|
||||
EarlyHook,
|
||||
DepsSystem,
|
||||
DepsUser,
|
||||
LateHook,
|
||||
Ready,
|
||||
}
|
||||
|
||||
impl PrebakeStage {
|
||||
pub const fn order(&self) -> u32 {
|
||||
match self {
|
||||
PrebakeStage::Init => 0,
|
||||
PrebakeStage::Bootstrap => 100,
|
||||
PrebakeStage::EarlyHook => 200,
|
||||
PrebakeStage::DepsSystem => 300,
|
||||
PrebakeStage::DepsUser => 400,
|
||||
PrebakeStage::LateHook => 500,
|
||||
PrebakeStage::Ready => 1000,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for PrebakeStage {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
PrebakeStage::Init => write!(f, "Init"),
|
||||
PrebakeStage::Bootstrap => write!(f, "Bootstrap"),
|
||||
PrebakeStage::EarlyHook => write!(f, "EarlyHook"),
|
||||
PrebakeStage::DepsSystem => write!(f, "DepsSystem"),
|
||||
PrebakeStage::DepsUser => write!(f, "DepsUser"),
|
||||
PrebakeStage::LateHook => write!(f, "LateHook"),
|
||||
PrebakeStage::Ready => write!(f, "Ready"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
use crate::ExecutionContext;
|
||||
use crate::engine::{Engine, EventSender, ExecutionError, ExecutionResult};
|
||||
use crate::prebake::config::PrebakeConfig;
|
||||
use std::path::PathBuf;
|
||||
|
||||
/// Executes the bootstrap stage.
|
||||
///
|
||||
/// This function is responsible for running the system bootstrap script,
|
||||
/// typically used to initialize the environment or perform pre-configuration
|
||||
/// operations required for system startup.
|
||||
///
|
||||
/// # Parameters
|
||||
///
|
||||
/// * `_config` - Prebake configuration parameters (not used currently)
|
||||
/// * `ctx` - Execution context containing state and information required
|
||||
/// for the execution environment
|
||||
/// * `event_tx` - Event sender for broadcasting execution events externally
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Returns `Result<ExecutionResult, ExecutionError>`:
|
||||
/// * `Ok(ExecutionResult)` - Result when script execution succeeds
|
||||
/// * `Err(ExecutionError)` - Error information when script execution fails
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```rust,ignore
|
||||
/// // Call bootstrap stage in prebake workflow
|
||||
/// let result = bootstrap(config, ctx, &event_tx).await;
|
||||
/// ```
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Possible errors:
|
||||
/// * Bootstrap script file does not exist
|
||||
/// * Insufficient permissions to execute script
|
||||
/// * Runtime errors during script execution
|
||||
pub async fn bootstrap(
|
||||
_config: &PrebakeConfig,
|
||||
ctx: &ExecutionContext,
|
||||
event_tx: &EventSender,
|
||||
) -> Result<ExecutionResult, ExecutionError> {
|
||||
let engine = Engine::new();
|
||||
let bootstrap_path = PathBuf::from("/bootstrap.sh");
|
||||
|
||||
// Check if bootstrap script exists before execution
|
||||
if !bootstrap_path.exists() {
|
||||
return Err(ExecutionError::ScriptNotFound(bootstrap_path));
|
||||
}
|
||||
|
||||
engine.execute_script(&bootstrap_path, ctx, &event_tx).await
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
use crate::ExecutionContext;
|
||||
use crate::engine::{Engine, EventSender, ExecutionError, ExecutionResult};
|
||||
use crate::prebake::config::PrebakeConfig;
|
||||
use std::path::PathBuf;
|
||||
|
||||
pub async fn depssystem(
|
||||
config: &PrebakeConfig,
|
||||
ctx: &ExecutionContext,
|
||||
event_tx: &EventSender,
|
||||
) -> Result<ExecutionResult, ExecutionError> {
|
||||
todo!();
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
use crate::ExecutionContext;
|
||||
use crate::engine::{Engine, EventSender, ExecutionError, ExecutionResult};
|
||||
use crate::prebake::config::PrebakeConfig;
|
||||
use std::path::PathBuf;
|
||||
@@ -0,0 +1,26 @@
|
||||
use crate::prebake::config::{PrebakeConfig};
|
||||
use crate::types::builderconfig::BuilderType;
|
||||
use config::Config;
|
||||
|
||||
pub use crate::prebake::env::baremetal::prepare_baremetal;
|
||||
pub use crate::prebake::env::container::prepare_container;
|
||||
pub use crate::prebake::env::custom::prepare_custom;
|
||||
pub use crate::prebake::env::firecracker::prepare_firecracker;
|
||||
|
||||
pub async fn prepare_environment(config: &PrebakeConfig, settings: &Config) -> anyhow::Result<()> {
|
||||
match config.environment.builder {
|
||||
BuilderType::Baremetal => {
|
||||
prepare_baremetal(config, settings).await?;
|
||||
}
|
||||
BuilderType::Docker => {
|
||||
let _container = prepare_container(config, settings).await?;
|
||||
}
|
||||
BuilderType::Firecracker => {
|
||||
prepare_firecracker(config, settings).await?;
|
||||
}
|
||||
BuilderType::Custom => {
|
||||
prepare_custom(config, settings).await?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
use crate::ExecutionContext;
|
||||
use crate::engine::{Engine, EventSender, ExecutionError};
|
||||
use crate::types::command::CustomCommand;
|
||||
use std::path::PathBuf;
|
||||
|
||||
pub async fn hook(
|
||||
hook: Option<Vec<CustomCommand>>,
|
||||
ctx: &ExecutionContext,
|
||||
event_tx: EventSender,
|
||||
) -> Result<(), ExecutionError> {
|
||||
let hook = match hook {
|
||||
Some(h) => h,
|
||||
None => {
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
|
||||
for (index, hook_cmd) in hook.iter().enumerate() {
|
||||
let engine = Engine::new();
|
||||
let mut hook_ctx = ctx.clone();
|
||||
|
||||
// Update working directory if provided
|
||||
if let Some(working_dir) = &hook_cmd.working_dir {
|
||||
log::debug!("Updating working directory to {}", working_dir);
|
||||
hook_ctx.working_dir = PathBuf::from(working_dir);
|
||||
}
|
||||
// Update environment variables if provided
|
||||
if let Some(environment) = &hook_cmd.environment {
|
||||
hook_ctx.env_vars.extend(environment.clone());
|
||||
}
|
||||
log::info!("Executing hook index {}.", index);
|
||||
|
||||
let result = engine
|
||||
.execute_command(&hook_cmd.command, &hook_ctx, &event_tx)
|
||||
.await;
|
||||
if let Err(e) = result {
|
||||
log::error!("Hook {} failed: {:?}", index, e);
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
use topological_sort::TopologicalSort;
|
||||
use crate::{decorator::Decorator, parser::Function};
|
||||
use topological_sort::TopologicalSort;
|
||||
|
||||
fn sort_function(functions: Vec<Function>) -> Vec<Function> {
|
||||
let mut ts = TopologicalSort::<String>::new();
|
||||
@@ -72,7 +72,11 @@ mod tests {
|
||||
#[test]
|
||||
fn test_simple_dependency() {
|
||||
let function1 = create_function("func1", vec![], "body1");
|
||||
let function2 = create_function("func2", vec![Decorator::After("func1".to_string())], "body2");
|
||||
let function2 = create_function(
|
||||
"func2",
|
||||
vec![Decorator::After("func1".to_string())],
|
||||
"body2",
|
||||
);
|
||||
let functions = vec![function1.clone(), function2.clone()];
|
||||
|
||||
let result = sort_function(functions);
|
||||
@@ -85,8 +89,16 @@ mod tests {
|
||||
#[test]
|
||||
fn test_multiple_dependencies() {
|
||||
let function1 = create_function("func1", vec![], "body1");
|
||||
let function2 = create_function("func2", vec![Decorator::After("func1".to_string())], "body2");
|
||||
let function3 = create_function("func3", vec![Decorator::After("func2".to_string())], "body3");
|
||||
let function2 = create_function(
|
||||
"func2",
|
||||
vec![Decorator::After("func1".to_string())],
|
||||
"body2",
|
||||
);
|
||||
let function3 = create_function(
|
||||
"func3",
|
||||
vec![Decorator::After("func2".to_string())],
|
||||
"body3",
|
||||
);
|
||||
let functions = vec![function1.clone(), function2.clone(), function3.clone()];
|
||||
|
||||
let result = sort_function(functions);
|
||||
@@ -100,13 +112,26 @@ mod tests {
|
||||
#[test]
|
||||
fn test_complex_dependencies() {
|
||||
let function1 = create_function("func1", vec![], "body1");
|
||||
let function2 = create_function("func2", vec![
|
||||
let function2 = create_function(
|
||||
"func2",
|
||||
vec![
|
||||
Decorator::After("func1".to_string()),
|
||||
Decorator::After("func3".to_string()),
|
||||
], "body2");
|
||||
let function3 = create_function("func3", vec![Decorator::After("func1".to_string())], "body3");
|
||||
],
|
||||
"body2",
|
||||
);
|
||||
let function3 = create_function(
|
||||
"func3",
|
||||
vec![Decorator::After("func1".to_string())],
|
||||
"body3",
|
||||
);
|
||||
let function4 = create_function("func4", vec![], "body4");
|
||||
let functions = vec![function1.clone(), function2.clone(), function3.clone(), function4.clone()];
|
||||
let functions = vec![
|
||||
function1.clone(),
|
||||
function2.clone(),
|
||||
function3.clone(),
|
||||
function4.clone(),
|
||||
];
|
||||
|
||||
let result = sort_function(functions);
|
||||
|
||||
@@ -130,15 +155,20 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_mixed_decorators() {
|
||||
let function1 = create_function("func1", vec![
|
||||
Decorator::Pipeline,
|
||||
Decorator::Timeout(1000),
|
||||
], "body1");
|
||||
let function2 = create_function("func2", vec![
|
||||
let function1 = create_function(
|
||||
"func1",
|
||||
vec![Decorator::Pipeline, Decorator::Timeout(1000)],
|
||||
"body1",
|
||||
);
|
||||
let function2 = create_function(
|
||||
"func2",
|
||||
vec![
|
||||
Decorator::After("func1".to_string()),
|
||||
Decorator::Retry(3, 100),
|
||||
Decorator::Fallible,
|
||||
], "body2");
|
||||
],
|
||||
"body2",
|
||||
);
|
||||
let functions = vec![function1.clone(), function2.clone()];
|
||||
|
||||
let result = sort_function(functions);
|
||||
@@ -152,10 +182,14 @@ mod tests {
|
||||
fn test_function_with_multiple_after_decorators() {
|
||||
let function1 = create_function("func1", vec![], "body1");
|
||||
let function2 = create_function("func2", vec![], "body2");
|
||||
let function3 = create_function("func3", vec![
|
||||
let function3 = create_function(
|
||||
"func3",
|
||||
vec![
|
||||
Decorator::After("func1".to_string()),
|
||||
Decorator::After("func2".to_string()),
|
||||
], "body3");
|
||||
],
|
||||
"body3",
|
||||
);
|
||||
let functions = vec![function1.clone(), function2.clone(), function3.clone()];
|
||||
|
||||
let result = sort_function(functions);
|
||||
@@ -175,17 +209,25 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_ignores_non_after_decorators() {
|
||||
let function1 = create_function("func1", vec![
|
||||
let function1 = create_function(
|
||||
"func1",
|
||||
vec![
|
||||
Decorator::Pipeline,
|
||||
Decorator::If("condition".to_string()),
|
||||
Decorator::Fallible,
|
||||
], "body1");
|
||||
let function2 = create_function("func2", vec![
|
||||
],
|
||||
"body1",
|
||||
);
|
||||
let function2 = create_function(
|
||||
"func2",
|
||||
vec![
|
||||
Decorator::Loop(5),
|
||||
Decorator::Timeout(1000),
|
||||
Decorator::Retry(3, 100),
|
||||
Decorator::Export,
|
||||
], "body2");
|
||||
],
|
||||
"body2",
|
||||
);
|
||||
let functions = vec![function1.clone(), function2.clone()];
|
||||
|
||||
let result = sort_function(functions);
|
||||
@@ -0,0 +1,5 @@
|
||||
pub mod memsize;
|
||||
pub mod architecture;
|
||||
pub mod builderconfig;
|
||||
pub mod command;
|
||||
pub mod cache;
|
||||
@@ -0,0 +1,58 @@
|
||||
use std::str::FromStr;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
pub enum Architecture {
|
||||
NoArch,
|
||||
X86_64,
|
||||
Amd64, // Same as X86_64, normalized when processing
|
||||
Arm64,
|
||||
Aarch64, // Same as Arm64, normalized when processing
|
||||
Riscv64,
|
||||
Loongarch64,
|
||||
Loong64, // Same as Loongarch64, normalized when processing, discarding old world Loongarch for simplicity
|
||||
}
|
||||
|
||||
impl Architecture {
|
||||
pub fn normalized(&self) -> Architecture {
|
||||
match self {
|
||||
Architecture::Amd64 => Architecture::X86_64,
|
||||
Architecture::Aarch64 => Architecture::Arm64,
|
||||
Architecture::Loong64 => Architecture::Loongarch64,
|
||||
other => other.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Architecture::NoArch => "noarch",
|
||||
Architecture::X86_64 => "x86_64",
|
||||
Architecture::Amd64 => "amd64",
|
||||
Architecture::Arm64 => "arm64",
|
||||
Architecture::Aarch64 => "aarch64",
|
||||
Architecture::Riscv64 => "riscv64",
|
||||
Architecture::Loongarch64 => "loongarch64",
|
||||
Architecture::Loong64 => "loong64",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for Architecture {
|
||||
type Err = String;
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
match s.to_lowercase().as_str() {
|
||||
"noarch" => Ok(Architecture::NoArch),
|
||||
"x86_64" | "amd64" => Ok(Architecture::Amd64),
|
||||
"arm64" | "aarch64" => Ok(Architecture::Aarch64),
|
||||
"riscv64" => Ok(Architecture::Riscv64),
|
||||
"loongarch64" | "loong64" => Ok(Architecture::Loong64),
|
||||
_ => Err(format!("Unknown architecture: {}", s)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Architecture {
|
||||
fn default() -> Self {
|
||||
Architecture::NoArch
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum BuilderType {
|
||||
Docker,
|
||||
Firecracker,
|
||||
Custom,
|
||||
Baremetal,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
#[serde(untagged)]
|
||||
pub enum BuilderConfig {
|
||||
Docker(DockerConfig),
|
||||
Firecracker(FirecrackerConfig),
|
||||
Custom(CustomConfig),
|
||||
Baremetal(BaremetalConfig),
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct DockerConfig {
|
||||
#[serde(default)]
|
||||
pub image: Option<String>,
|
||||
#[serde(default)]
|
||||
pub dockerfile: Option<String>,
|
||||
#[serde(default)]
|
||||
pub build_args: Option<HashMap<String, String>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct FirecrackerConfig {}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct CustomConfig {
|
||||
pub name: String,
|
||||
pub setup_script: String,
|
||||
pub cleanup_script: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct BaremetalConfig;
|
||||
@@ -0,0 +1,62 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct CacheDirectory {
|
||||
pub path: String,
|
||||
#[serde(default = "default_strategy")]
|
||||
pub strategy: CacheStrategyType,
|
||||
#[serde(default = "default_mode")]
|
||||
pub mode: CacheMode,
|
||||
}
|
||||
|
||||
fn default_strategy() -> CacheStrategyType {
|
||||
CacheStrategyType::Always
|
||||
}
|
||||
fn default_mode() -> CacheMode {
|
||||
CacheMode::Zstd
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum CacheStrategyType {
|
||||
Always,
|
||||
Readonly,
|
||||
Clean,
|
||||
Never,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum CacheMode {
|
||||
Gzip,
|
||||
Zstd,
|
||||
None,
|
||||
Dir,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct CacheStrategy {
|
||||
#[serde(default = "default_ttl_days", rename = "ttl_days")]
|
||||
pub ttl_days: u32,
|
||||
#[serde(default = "default_max_size_gb", rename = "max_size_gb")]
|
||||
pub max_size_gb: u32,
|
||||
#[serde(default = "default_cleanup_policy", rename = "cleanup_policy")]
|
||||
pub cleanup_policy: CleanupPolicy,
|
||||
}
|
||||
|
||||
fn default_ttl_days() -> u32 {
|
||||
30
|
||||
}
|
||||
fn default_max_size_gb() -> u32 {
|
||||
20
|
||||
}
|
||||
fn default_cleanup_policy() -> CleanupPolicy {
|
||||
CleanupPolicy::Lru
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum CleanupPolicy {
|
||||
Lru,
|
||||
Fifo,
|
||||
Size,
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct CustomCommand {
|
||||
pub name: String,
|
||||
pub command: String,
|
||||
#[serde(default)]
|
||||
pub environment: Option<HashMap<String, String>>,
|
||||
#[serde(default)]
|
||||
pub working_dir: Option<String>,
|
||||
#[serde(default = "default_timeout")]
|
||||
pub timeout: u32,
|
||||
}
|
||||
|
||||
fn default_timeout() -> u32 {
|
||||
300
|
||||
}
|
||||
@@ -0,0 +1,260 @@
|
||||
use serde::{Deserialize, Deserializer, Serialize, Serializer};
|
||||
use std::num::ParseFloatError;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct MemSize(pub u64);
|
||||
|
||||
impl MemSize {
|
||||
pub fn bytes(&self) -> u64 {
|
||||
self.0
|
||||
}
|
||||
|
||||
pub fn as_kb(&self) -> u64 {
|
||||
self.0 / 1024
|
||||
}
|
||||
|
||||
pub fn as_mb(&self) -> u64 {
|
||||
self.0 / 1024 / 1024
|
||||
}
|
||||
|
||||
pub fn as_gb(&self) -> u64 {
|
||||
self.0 / 1024 / 1024 / 1024
|
||||
}
|
||||
|
||||
pub const fn from_bytes(bytes: u64) -> Self {
|
||||
Self(bytes)
|
||||
}
|
||||
|
||||
pub const fn from_kb(kb: u64) -> Self {
|
||||
Self(kb * 1000)
|
||||
}
|
||||
|
||||
pub const fn from_mb(mb: u64) -> Self {
|
||||
Self(mb * 1000 * 1000)
|
||||
}
|
||||
|
||||
pub const fn from_gb(gb: u64) -> Self {
|
||||
Self(gb * 1000 * 1000 * 1000)
|
||||
}
|
||||
|
||||
pub const fn from_kib(kib: u64) -> Self {
|
||||
Self(kib * 1024)
|
||||
}
|
||||
|
||||
pub const fn from_mib(mib: u64) -> Self {
|
||||
Self(mib * 1024 * 1024)
|
||||
}
|
||||
|
||||
pub const fn from_gib(gib: u64) -> Self {
|
||||
Self(gib * 1024 * 1024 * 1024)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for MemSize {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
let s: String = String::deserialize(deserializer)?;
|
||||
parse_mem_size(&s).map_err(serde::de::Error::custom)
|
||||
}
|
||||
}
|
||||
|
||||
impl Serialize for MemSize {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: Serializer,
|
||||
{
|
||||
let bytes = self.0;
|
||||
|
||||
if bytes == 0 {
|
||||
return serializer.serialize_str("0B");
|
||||
}
|
||||
|
||||
// Define units in ascending order
|
||||
const UNITS: &[(u64, &str)] = &[
|
||||
(1, "B"),
|
||||
(1024, "KiB"),
|
||||
(1024 * 1024, "MiB"),
|
||||
(1024 * 1024 * 1024, "GiB"),
|
||||
(1024 * 1024 * 1024 * 1024, "TiB"),
|
||||
];
|
||||
|
||||
// Find the appropriate unit: the largest one where value >= 1
|
||||
// Start from the largest unit and work down
|
||||
let (divisor, unit) = UNITS
|
||||
.iter()
|
||||
.rev()
|
||||
.find(|&&(divisor, _)| bytes as f64 / divisor as f64 >= 1.0)
|
||||
.unwrap_or(&(1, "B"));
|
||||
|
||||
let value = bytes as f64 / *divisor as f64;
|
||||
|
||||
// Format with appropriate precision
|
||||
let formatted = if value.fract() == 0.0 {
|
||||
format!("{:.0}{}", value, unit)
|
||||
} else if value * 10.0 % 1.0 == 0.0 {
|
||||
format!("{:.1}{}", value, unit)
|
||||
} else {
|
||||
format!("{:.2}{}", value, unit)
|
||||
};
|
||||
|
||||
serializer.serialize_str(&formatted)
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses a human-readable size string into bytes.
|
||||
///
|
||||
/// Supported formats:
|
||||
/// - `500MB` → 500 * 1000 * 1000
|
||||
/// - `8192Ki` → 8192 * 1024
|
||||
/// - `0B` → 0
|
||||
/// - `333.33MiB` → ceil(333.33 * 1024 * 1024)
|
||||
/// - `-1B` → 0 (non-positive values are treated as 0)
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error if:
|
||||
/// - The unit is `b` (bits are not allowed)
|
||||
/// - The format is malformed (e.g., `2BB`, `7KiKiB`)
|
||||
/// - The number cannot be parsed
|
||||
/// - The unit is unrecognized
|
||||
pub fn parse_mem_size(s: &str) -> Result<MemSize, String> {
|
||||
let s = s.trim();
|
||||
if s.is_empty() {
|
||||
return Err("empty string".to_string());
|
||||
}
|
||||
|
||||
// Split into number and unit parts
|
||||
let (num_str, unit_raw) = split_number_and_unit(s)?;
|
||||
|
||||
// Parse the numeric part
|
||||
let num: f64 = num_str.parse().map_err(|e: ParseFloatError| e.to_string())?;
|
||||
|
||||
// Normalize unit: remove trailing 'B' if present, but preserve case for error detection
|
||||
let unit = if unit_raw.ends_with('B') || unit_raw.ends_with('b') {
|
||||
let without_b = &unit_raw[..unit_raw.len() - 1];
|
||||
// Check if the original unit ends with lowercase 'b' (bits) before stripping
|
||||
if unit_raw.ends_with('b') && !unit_raw.ends_with('B') {
|
||||
return Err(format!("bit unit 'b' is not allowed, use 'B' for bytes: {}", unit_raw));
|
||||
}
|
||||
without_b
|
||||
} else {
|
||||
unit_raw
|
||||
};
|
||||
|
||||
// Determine multiplier based on unit
|
||||
let multiplier = match unit {
|
||||
"" => 1, // bytes (no unit)
|
||||
"k" | "K" => 1000,
|
||||
"m" | "M" => 1000 * 1000,
|
||||
"g" | "G" => 1000 * 1000 * 1000,
|
||||
"t" | "T" => 1000 as u64 * 1000 * 1000 * 1000,
|
||||
"ki" | "kI" | "Ki" | "KI" => 1024,
|
||||
"mi" | "mI" | "Mi" | "MI" => 1024 * 1024,
|
||||
"gi" | "gI" | "Gi" | "GI" => 1024 * 1024 * 1024,
|
||||
"ti" | "tI" | "Ti" | "TI" => 1024 as u64 * 1024 * 1024 * 1024,
|
||||
_ => return Err(format!("unknown unit: {}", unit_raw)),
|
||||
};
|
||||
|
||||
// Calculate bytes
|
||||
let bytes_float = num * multiplier as f64;
|
||||
|
||||
// Handle non-positive values (treat as 0)
|
||||
if bytes_float <= 0.0 {
|
||||
return Ok(MemSize(0));
|
||||
}
|
||||
|
||||
// Ceil the result (e.g., 333.33 MiB → 333.33 * 1024² = 349,523,866.08 → 349,523,867)
|
||||
Ok(MemSize(bytes_float.ceil() as u64))
|
||||
}
|
||||
|
||||
/// Splits a size string into (number_part, unit_part).
|
||||
///
|
||||
/// # Examples
|
||||
/// - `"500MB"` → `("500", "MB")`
|
||||
/// - `"8192Ki"` → `("8192", "Ki")`
|
||||
/// - `"0B"` → `("0", "B")`
|
||||
fn split_number_and_unit(s: &str) -> Result<(&str, &str), String> {
|
||||
// Find the first character that is not a digit, dot, or minus sign
|
||||
let unit_start = s
|
||||
.find(|c: char| !c.is_ascii_digit() && c != '.' && c != '-')
|
||||
.unwrap_or(s.len());
|
||||
|
||||
if unit_start == 0 {
|
||||
return Err("missing numeric part".to_string());
|
||||
}
|
||||
|
||||
let num_part = &s[..unit_start];
|
||||
let unit_part = &s[unit_start..];
|
||||
|
||||
Ok((num_part, unit_part))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_valid_sizes() {
|
||||
assert_eq!(parse_mem_size("500MB").unwrap().bytes(), 500 * 1000 * 1000);
|
||||
assert_eq!(parse_mem_size("8192Ki").unwrap().bytes(), 8192 * 1024);
|
||||
assert_eq!(parse_mem_size("0B").unwrap().bytes(), 0);
|
||||
assert_eq!(parse_mem_size("-1B").unwrap().bytes(), 0);
|
||||
assert_eq!(parse_mem_size("333.33MiB").unwrap().bytes(), 349521839);
|
||||
assert_eq!(parse_mem_size("1G").unwrap().bytes(), 1_000_000_000);
|
||||
assert_eq!(parse_mem_size("2GB").unwrap().bytes(), 2_000_000_000);
|
||||
assert_eq!(parse_mem_size("1GiB").unwrap().bytes(), 1_073_741_824);
|
||||
assert_eq!(parse_mem_size("1.5GB").unwrap().bytes(), 1_500_000_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_invalid_sizes() {
|
||||
assert!(parse_mem_size("780Mb").is_err()); // bits not allowed
|
||||
assert!(parse_mem_size("2BB").is_err()); // malformed
|
||||
assert!(parse_mem_size("7KiKiB").is_err()); // malformed
|
||||
assert!(parse_mem_size("a_faulty_value").is_err());
|
||||
assert!(parse_mem_size("").is_err());
|
||||
assert!(parse_mem_size("GB").is_err()); // missing number
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_edge_cases() {
|
||||
assert_eq!(parse_mem_size("0").unwrap().bytes(), 0);
|
||||
assert_eq!(parse_mem_size("0KB").unwrap().bytes(), 0);
|
||||
assert_eq!(parse_mem_size("-0.5MB").unwrap().bytes(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_serialize_boundary() {
|
||||
// 1023 bytes → 1023B (not 0.999KiB)
|
||||
let size = MemSize(1023);
|
||||
let json = serde_json::to_string(&size).unwrap();
|
||||
assert_eq!(json, "\"1023B\"");
|
||||
|
||||
// 1024 bytes → 1KiB
|
||||
let size = MemSize(1024);
|
||||
let json = serde_json::to_string(&size).unwrap();
|
||||
assert_eq!(json, "\"1KiB\"");
|
||||
|
||||
// 1,048,575 bytes (1 MiB - 1) → should be in KiB, not MiB
|
||||
let size = MemSize(1024 * 1024 - 1);
|
||||
let json = serde_json::to_string(&size).unwrap();
|
||||
// 1,048,575 / 1024 = 1023.9990234375 KiB
|
||||
assert_eq!(json, "\"1024.00KiB\"");
|
||||
|
||||
// 1,048,576 bytes → 1MiB
|
||||
let size = MemSize(1024 * 1024);
|
||||
let json = serde_json::to_string(&size).unwrap();
|
||||
assert_eq!(json, "\"1MiB\"");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_serialize_tebibyte_edge() {
|
||||
// 1 TiB - 1 bytes
|
||||
let size = MemSize(1024 * 1024 * 1024 * 1024 - 1);
|
||||
let json = serde_json::to_string(&size).unwrap();
|
||||
// Should be in GiB, since it's less than 1 TiB
|
||||
assert!(json.contains("GiB"));
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
use std::collections::HashSet;
|
||||
use regex::Regex;
|
||||
use std::collections::HashSet;
|
||||
pub struct Variable {
|
||||
pub name: String,
|
||||
pub value: String,
|
||||
@@ -43,9 +43,7 @@ mod tests {
|
||||
|
||||
/// 辅助函数：将Variable列表转换为排序后的字符串集合（解决HashSet无序问题）
|
||||
fn sorted_var_names(vars: &[Variable]) -> BTreeSet<String> {
|
||||
vars.iter()
|
||||
.map(|v| v.name.clone())
|
||||
.collect::<BTreeSet<_>>()
|
||||
vars.iter().map(|v| v.name.clone()).collect::<BTreeSet<_>>()
|
||||
}
|
||||
|
||||
/// 测试1：基础场景（重复变量去重 + 多种合法格式）
|
||||
@@ -0,0 +1,80 @@
|
||||
fn lookup_user(
|
||||
user: &OsStr,
|
||||
fallback_to_ids_if_names_are_numeric: bool,
|
||||
) -> Result<UserIds, PrivDropError> {
|
||||
let username = CString::new(user.as_bytes())
|
||||
.map_err(|_| PrivDropError::from((ErrorKind::SysError, "Invalid username")))?;
|
||||
|
||||
let mut pwd = MaybeUninit::<libc::passwd>::uninit();
|
||||
let mut pwent = std::ptr::null_mut::<libc::passwd>();
|
||||
|
||||
// Start with the initial buffer size and increase if needed
|
||||
let mut bufsize = INITIAL_BUFFER_SIZE;
|
||||
let mut pwbuf = vec![0; bufsize];
|
||||
|
||||
let mut ret;
|
||||
loop {
|
||||
ret = unsafe {
|
||||
libc::getpwnam_r(
|
||||
username.as_ptr(),
|
||||
pwd.as_mut_ptr(),
|
||||
pwbuf.as_mut_ptr(),
|
||||
pwbuf.len(),
|
||||
&mut pwent,
|
||||
)
|
||||
};
|
||||
|
||||
// If we get ERANGE, the buffer was too small, double it and try again
|
||||
if ret == libc::ERANGE {
|
||||
bufsize *= 2;
|
||||
pwbuf.resize(bufsize, 0);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if ret != 0 || pwent.is_null() {
|
||||
if !fallback_to_ids_if_names_are_numeric {
|
||||
if ret != 0 && ret == libc::ENOENT {
|
||||
return Err(PrivDropError::from((ErrorKind::SysError, "User not found")));
|
||||
} else if ret != 0 {
|
||||
return Err(PrivDropError::from((
|
||||
ErrorKind::SysError,
|
||||
"Failed to look up user",
|
||||
)));
|
||||
} else {
|
||||
return Err(PrivDropError::from((ErrorKind::SysError, "User not found")));
|
||||
}
|
||||
}
|
||||
|
||||
// Try to parse the username as a numeric UID
|
||||
let user_str = user.to_str().ok_or_else(|| {
|
||||
PrivDropError::from((
|
||||
ErrorKind::SysError,
|
||||
"User not found and username is not valid UTF-8",
|
||||
))
|
||||
})?;
|
||||
|
||||
let uid = user_str.parse().map_err(|_| {
|
||||
PrivDropError::from((
|
||||
ErrorKind::SysError,
|
||||
"User not found and username is not a valid numeric ID",
|
||||
))
|
||||
})?;
|
||||
|
||||
return Ok(UserIds {
|
||||
uid: Some(uid),
|
||||
gid: None,
|
||||
group_list: None,
|
||||
});
|
||||
}
|
||||
|
||||
let uid = unsafe { *pwent }.pw_uid;
|
||||
let gid = unsafe { *pwent }.pw_gid;
|
||||
|
||||
Ok(UserIds {
|
||||
uid: Some(uid),
|
||||
gid: Some(gid),
|
||||
group_list: None,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
//! Integration tests for workshop-executor engine
|
||||
//!
|
||||
//! These tests execute real commands and verify results.
|
||||
//! Run with: cargo test --test engine_integration
|
||||
//!
|
||||
//! Note: These are TRUE integration tests - they run in a separate test binary
|
||||
//! and execute actual commands on the system.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
use workshop_executor::engine::{Engine, ExecutionContext};
|
||||
|
||||
fn create_executor() -> Engine {
|
||||
Engine::new()
|
||||
}
|
||||
|
||||
fn create_context() -> ExecutionContext {
|
||||
ExecutionContext {
|
||||
task_id: "test-integration-1".to_string(),
|
||||
pipeline_name: "test-pipeline".to_string(),
|
||||
username: "testuser".to_string(),
|
||||
working_dir: std::env::temp_dir(),
|
||||
env_vars: HashMap::new(),
|
||||
timeout: Some(Duration::from_secs(30)),
|
||||
cgroup_path: None,
|
||||
shell: "bash".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_echo_outputs_correct_message() {
|
||||
let executor = create_executor();
|
||||
let ctx = create_context();
|
||||
|
||||
use tokio::process::Command;
|
||||
let mut cmd = Command::new("echo");
|
||||
cmd.arg("hello world");
|
||||
|
||||
let result = executor.execute(cmd, &ctx, &None).await.unwrap();
|
||||
|
||||
assert!(result.success);
|
||||
assert_eq!(result.exit_code, 0);
|
||||
assert!(result.stdout.contains("hello world"));
|
||||
assert!(result.stderr.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_true_returns_zero_exit_code() {
|
||||
let executor = create_executor();
|
||||
let ctx = create_context();
|
||||
|
||||
use tokio::process::Command;
|
||||
let cmd = Command::new("true");
|
||||
|
||||
let result = executor.execute(cmd, &ctx, &None).await.unwrap();
|
||||
|
||||
assert!(result.success);
|
||||
assert_eq!(result.exit_code, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_false_returns_nonzero_exit_code() {
|
||||
let executor = create_executor();
|
||||
let ctx = create_context();
|
||||
|
||||
use tokio::process::Command;
|
||||
let cmd = Command::new("false");
|
||||
|
||||
let result = executor.execute(cmd, &ctx, &None).await.unwrap();
|
||||
|
||||
assert!(!result.success);
|
||||
assert_ne!(result.exit_code, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_ls_lists_directory() {
|
||||
let executor = create_executor();
|
||||
let ctx = create_context();
|
||||
|
||||
use tokio::process::Command;
|
||||
let mut cmd = Command::new("ls");
|
||||
cmd.arg("-la");
|
||||
cmd.arg("/tmp");
|
||||
|
||||
let result = executor.execute(cmd, &ctx, &None).await.unwrap();
|
||||
|
||||
assert!(result.success);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_date_command() {
|
||||
let executor = create_executor();
|
||||
let ctx = create_context();
|
||||
|
||||
use tokio::process::Command;
|
||||
let cmd = Command::new("date");
|
||||
|
||||
let result = executor.execute(cmd, &ctx, &None).await.unwrap();
|
||||
|
||||
assert!(result.success);
|
||||
assert!(!result.stdout.trim().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_invalid_command_returns_error() {
|
||||
let executor = create_executor();
|
||||
let ctx = create_context();
|
||||
|
||||
use tokio::process::Command;
|
||||
let cmd = Command::new("this_command_definitely_does_not_exist_12345");
|
||||
|
||||
let result = executor.execute(cmd, &ctx, &None).await;
|
||||
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_passes_custom_env_var() {
|
||||
let executor = create_executor();
|
||||
let mut ctx = create_context();
|
||||
ctx.env_vars
|
||||
.insert("MY_CUSTOM_VAR".to_string(), "my_value_123".to_string());
|
||||
|
||||
use tokio::process::Command;
|
||||
let cmd = Command::new("env");
|
||||
|
||||
let result = executor.execute(cmd, &ctx, &None).await.unwrap();
|
||||
|
||||
assert!(result.stdout.contains("MY_CUSTOM_VAR=my_value_123"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_sets_ws_taskid() {
|
||||
let executor = create_executor();
|
||||
let mut ctx = create_context();
|
||||
ctx.task_id = "task-abc-123".to_string();
|
||||
ctx.pipeline_name = "pipeline-xyz".to_string();
|
||||
|
||||
use tokio::process::Command;
|
||||
let cmd = Command::new("env");
|
||||
|
||||
let result = executor.execute(cmd, &ctx, &None).await.unwrap();
|
||||
|
||||
assert!(result.stdout.contains("WS_TASKID=task-abc-123"));
|
||||
assert!(result.stdout.contains("WS_PIPELINE=pipeline-xyz"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_sets_ws_user_info() {
|
||||
let executor = create_executor();
|
||||
let ctx = create_context();
|
||||
|
||||
use tokio::process::Command;
|
||||
let cmd = Command::new("env");
|
||||
|
||||
let result = executor.execute(cmd, &ctx, &None).await.unwrap();
|
||||
|
||||
assert!(result.stdout.contains("WS_USERNAME=testuser"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_respects_working_directory() {
|
||||
let executor = create_executor();
|
||||
let ctx = ExecutionContext {
|
||||
task_id: "test-context-1".to_string(),
|
||||
pipeline_name: "test-pipeline".to_string(),
|
||||
username: "testuser".to_string(),
|
||||
working_dir: PathBuf::from("/tmp"),
|
||||
env_vars: HashMap::new(),
|
||||
timeout: Some(Duration::from_secs(30)),
|
||||
cgroup_path: None,
|
||||
shell: "bash".to_string(),
|
||||
};
|
||||
|
||||
use tokio::process::Command;
|
||||
let cmd = Command::new("pwd");
|
||||
|
||||
let result = executor.execute(cmd, &ctx, &None).await.unwrap();
|
||||
|
||||
assert!(result.success);
|
||||
let trimmed = result.stdout.trim();
|
||||
assert!(trimmed == "/tmp" || trimmed.ends_with("tmp"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_captures_stderr() {
|
||||
let executor = create_executor();
|
||||
let ctx = create_context();
|
||||
|
||||
use tokio::process::Command;
|
||||
let mut cmd = Command::new("bash");
|
||||
cmd.arg("-c");
|
||||
cmd.arg("echo error >&2");
|
||||
|
||||
let result = executor.execute(cmd, &ctx, &None).await.unwrap();
|
||||
|
||||
assert!(result.success);
|
||||
assert!(result.stderr.contains("error"));
|
||||
}
|
||||
@@ -1,193 +0,0 @@
|
||||
use crate::{Cli, TaskSpec};
|
||||
use serde::Serialize;
|
||||
use serde_json::Value;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use tokio::io::{AsyncBufReadExt, BufReader};
|
||||
use tokio::net::{UnixListener, UnixStream};
|
||||
use tokio::process::{Child, Command};
|
||||
use tokio::signal;
|
||||
use tokio::sync::RwLock;
|
||||
use tokio::time::{Duration, interval};
|
||||
use config::Config;
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
struct Heartbeat {
|
||||
pipeline_id: String,
|
||||
build_id: String,
|
||||
stage: String,
|
||||
timestamp: u64,
|
||||
status: String,
|
||||
subtasks: Vec<String>,
|
||||
}
|
||||
|
||||
pub async fn daemon(cli: &Cli, settings: Config) {
|
||||
let socket = PathBuf::from(settings.get_string("socket").unwrap());
|
||||
let agentsocket = settings.get_string("agentsocket").unwrap();
|
||||
let _ = std::fs::remove_file(&socket);
|
||||
|
||||
let heartbeat = Arc::new(RwLock::new(Heartbeat {
|
||||
pipeline_id: cli.pipeline.clone(),
|
||||
build_id: cli.build_id.clone(),
|
||||
stage: String::new(),
|
||||
timestamp: 0,
|
||||
status: String::new(),
|
||||
subtasks: Vec::new(),
|
||||
}));
|
||||
|
||||
// let heartbeat_clone = heartbeat.clone();
|
||||
let heartbeat_task = {
|
||||
let hb = heartbeat.clone();
|
||||
tokio::spawn(async move {
|
||||
let mut tick = interval(Duration::from_secs(1));
|
||||
loop {
|
||||
tick.tick().await;
|
||||
let mut hb = hb.write().await;
|
||||
hb.timestamp = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_millis() as u64;
|
||||
|
||||
// 只输出到stdout（模拟未来发送给Agent）
|
||||
println!("[HEARTBEAT] {}", serde_json::to_string(&*hb).unwrap());
|
||||
}
|
||||
})
|
||||
};
|
||||
|
||||
let socket_task = {
|
||||
let socket = socket.clone();
|
||||
tokio::spawn(async move {
|
||||
log::info!("Creating Unix socket at {}", socket.display());
|
||||
let listener =
|
||||
UnixListener::bind(&socket).expect("Failed to bind Unix socket");
|
||||
|
||||
loop {
|
||||
match listener.accept().await {
|
||||
Ok((stream, _)) => {
|
||||
tokio::spawn(handle_connection(stream));
|
||||
}
|
||||
Err(e) => {
|
||||
log::error!("Accept error: {}", e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
};
|
||||
|
||||
let mut tasks: Vec<TaskSpec> = Vec::new();
|
||||
|
||||
let execution_task = {
|
||||
let hb = heartbeat.clone();
|
||||
tokio::spawn(async move {
|
||||
let mut hb = hb.write().await;
|
||||
let mut children: Vec<(String, Child)> = vec![];
|
||||
|
||||
let executable = std::env::current_exe().unwrap();
|
||||
|
||||
for (index, task) in tasks.iter().enumerate() {
|
||||
let mut command = task.command(&executable);
|
||||
command.stdout(std::process::Stdio::piped());
|
||||
command.stderr(std::process::Stdio::piped());
|
||||
let child = command
|
||||
.spawn()
|
||||
.expect(format!("Failed to spawn child {}", index).as_str());
|
||||
|
||||
let pid = child.id().unwrap_or(0).to_string();
|
||||
children.push((pid.clone(), child));
|
||||
|
||||
hb.subtasks.push(pid.clone());
|
||||
}
|
||||
|
||||
for (pid, mut child) in children {
|
||||
let status = child.wait().await.expect("Failed to wait child");
|
||||
log::info!("Subtask {} exited with: {}", pid, status);
|
||||
|
||||
hb.subtasks.retain(|p| p != &pid);
|
||||
}
|
||||
|
||||
hb.stage = "Success".to_string();
|
||||
})
|
||||
};
|
||||
|
||||
tokio::select! {
|
||||
_ = heartbeat_task => {
|
||||
log::error!("Heartbeat task died");
|
||||
}
|
||||
_ = execution_task => {
|
||||
log::info!("All subtasks completed");
|
||||
}
|
||||
_ = socket_task => {
|
||||
log::error!("Socket task died");
|
||||
}
|
||||
_ = signal::ctrl_c() => {
|
||||
log::info!("Shutting down...");
|
||||
}
|
||||
}
|
||||
for pid in heartbeat.read().await.subtasks.clone() {
|
||||
let _ = Command::new("kill").arg("-TERM").arg(&pid).status().await;
|
||||
}
|
||||
|
||||
let _ = std::fs::remove_file(&socket);
|
||||
}
|
||||
|
||||
async fn handle_connection(stream: UnixStream) {
|
||||
let peer_addr = match stream.peer_addr() {
|
||||
Ok(addr) => format!("{:?}", addr),
|
||||
Err(_) => "unknown".to_string(),
|
||||
};
|
||||
|
||||
log::debug!("New connection from: {}", peer_addr);
|
||||
|
||||
let mut reader = BufReader::new(stream);
|
||||
let mut line = String::new();
|
||||
|
||||
loop {
|
||||
line.clear();
|
||||
match reader.read_line(&mut line).await {
|
||||
Ok(0) => {
|
||||
log::debug!("Connection closed by peer: {}", peer_addr);
|
||||
break;
|
||||
}
|
||||
Ok(_) => {
|
||||
// 简单打印原始JSON（预留字段解析空间）
|
||||
match serde_json::from_str::<Value>(&line) {
|
||||
Ok(json) => {
|
||||
// 只提取关键字段打印（字段可能变化）
|
||||
let msgtype = json
|
||||
.get("msgtype")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("unknown");
|
||||
let name = json
|
||||
.get("name")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("unnamed");
|
||||
|
||||
log::info!("[{}] Message from '{}': {}", msgtype, name, line.trim());
|
||||
|
||||
// TODO: 字段稳定后，可扩展为结构化打印
|
||||
// match msgtype {
|
||||
// "ResourceUsage" => print_resource(&json),
|
||||
// "Log" => print_log(&json),
|
||||
// _ => log::debug!("Unknown type: {}", msgtype),
|
||||
// }
|
||||
}
|
||||
Err(e) => {
|
||||
log::warn!(
|
||||
"Invalid JSON from {}: {} - Error: {}",
|
||||
peer_addr,
|
||||
line.trim(),
|
||||
e
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
log::error!("Failed to read from {}: {}", peer_addr, e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log::debug!("Connection handler for {} exiting", peer_addr);
|
||||
}
|
||||
@@ -1,255 +0,0 @@
|
||||
pub mod cgroups;
|
||||
pub mod pm;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::process::Stdio;
|
||||
use std::time::Duration;
|
||||
use tokio::process::Command;
|
||||
use tokio::sync::mpsc;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use chrono::Utc;
|
||||
|
||||
pub use crate::engine::cgroups::CgroupManager;
|
||||
use crate::engine::pm::PackageManager;
|
||||
// pub use crate::engine::pm::{PackageManager};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ResourceUsage {
|
||||
pub cpu_time_secs: f64,
|
||||
pub max_memory_kb: u64,
|
||||
pub io_read_kb: u64,
|
||||
pub io_write_kb: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ExecutionContext {
|
||||
pub task_id: String,
|
||||
pub pipeline_name: String,
|
||||
pub username: String,
|
||||
pub euid: u32,
|
||||
pub egid: u32,
|
||||
pub working_dir: PathBuf,
|
||||
pub env_vars: HashMap<String, String>,
|
||||
pub timeout: Option<Duration>,
|
||||
pub cgroup_path: Option<PathBuf>,
|
||||
pub shell: String,
|
||||
}
|
||||
|
||||
impl Default for ExecutionContext {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
task_id: String::new(),
|
||||
pipeline_name: String::new(),
|
||||
username: String::new(),
|
||||
euid: unsafe { libc::geteuid() },
|
||||
egid: unsafe { libc::getegid() },
|
||||
working_dir: std::env::current_dir().unwrap_or_else(|_| PathBuf::from("/")),
|
||||
env_vars: HashMap::new(),
|
||||
timeout: None,
|
||||
cgroup_path: None,
|
||||
shell: String::from("bash"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct ResourceLimits {
|
||||
pub memory_bytes: Option<u64>,
|
||||
pub cpu_weight: Option<u64>,
|
||||
pub cpu_max_us: Option<u64>,
|
||||
pub cpu_period_us: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct ExecutionResult {
|
||||
pub task_id: String,
|
||||
pub exit_code: i32,
|
||||
pub success: bool,
|
||||
pub duration: Duration,
|
||||
pub stdout: String,
|
||||
pub stderr: String,
|
||||
pub resource_usage: Option<ResourceUsage>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub enum StreamType {
|
||||
Stdout,
|
||||
Stderr,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub enum ExecutionEvent {
|
||||
TaskStarted {
|
||||
task_id: String,
|
||||
timestamp: chrono::DateTime<chrono::Utc>,
|
||||
},
|
||||
OutputChunk {
|
||||
task_id: String,
|
||||
stream: StreamType,
|
||||
data: String,
|
||||
},
|
||||
TaskCompleted {
|
||||
task_id: String,
|
||||
result: ExecutionResult,
|
||||
},
|
||||
TaskFailed {
|
||||
task_id: String,
|
||||
error: String,
|
||||
},
|
||||
}
|
||||
|
||||
pub type EventSender = Option<mpsc::Sender<ExecutionEvent>>;
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub enum ExecutionError {
|
||||
InvalidCommand(String),
|
||||
ExecutionFailed(String),
|
||||
PermissionDenied { euid: u32, egid: u32, reason: String },
|
||||
Timeout,
|
||||
IoError(String),
|
||||
ScriptNotFound(PathBuf),
|
||||
PackageManagerNotFound,
|
||||
}
|
||||
|
||||
pub struct LocalExecutor {
|
||||
cgroup_manager: Option<CgroupManager>,
|
||||
}
|
||||
|
||||
impl LocalExecutor {
|
||||
pub fn new() -> Self {
|
||||
Self { cgroup_manager: None }
|
||||
}
|
||||
|
||||
pub fn with_cgroup(limits: ResourceLimits, build_id: &str) -> Result<Self, String> {
|
||||
let cgroup_manager = CgroupManager::new(build_id)
|
||||
.map_err(|e| format!("Failed to create cgroup: {}", e))?;
|
||||
|
||||
if let Some(mem) = limits.memory_bytes {
|
||||
cgroup_manager.set_memory_limit(mem)
|
||||
.map_err(|e| format!("Failed to set memory limit: {}", e))?;
|
||||
}
|
||||
|
||||
if let Some(weight) = limits.cpu_weight {
|
||||
cgroup_manager.set_cpu_weight(weight)
|
||||
.map_err(|e| format!("Failed to set cpu weight: {}", e))?;
|
||||
}
|
||||
|
||||
if let Some(max) = limits.cpu_max_us {
|
||||
cgroup_manager.set_cpu_max(max, limits.cpu_period_us)
|
||||
.map_err(|e| format!("Failed to set cpu max: {}", e))?;
|
||||
}
|
||||
|
||||
Ok(Self { cgroup_manager: Some(cgroup_manager) })
|
||||
}
|
||||
|
||||
fn apply_cgroup(&self, pid: u32) -> Result<(), ExecutionError> {
|
||||
if let Some(ref cgroup) = self.cgroup_manager {
|
||||
cgroup.add_process(pid)
|
||||
.map_err(|e| ExecutionError::IoError(format!("Failed to add process to cgroup: {}", e)))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn execute_script(&self, script_path: &PathBuf, ctx: &ExecutionContext, event_tx: EventSender) -> Result<ExecutionResult, ExecutionError> {
|
||||
let abs_path = if script_path.is_absolute() {
|
||||
script_path.clone()
|
||||
} else {
|
||||
ctx.working_dir.join(script_path)
|
||||
};
|
||||
|
||||
let mut command = Command::new(ctx.shell.as_str());
|
||||
command.arg("-c");
|
||||
command.arg(abs_path);
|
||||
self.execute(command, ctx, event_tx).await
|
||||
}
|
||||
|
||||
pub async fn execute_command(&self, cmd: String, ctx: &ExecutionContext, event_tx: EventSender) -> Result<ExecutionResult, ExecutionError> {
|
||||
let mut command = Command::new(ctx.shell.as_str());
|
||||
command.arg("-c");
|
||||
command.arg(cmd);
|
||||
self.execute(command, ctx, event_tx).await
|
||||
}
|
||||
|
||||
pub async fn execute(&self, cmd: Command, ctx: &ExecutionContext, event_tx: EventSender) -> Result<ExecutionResult, ExecutionError> {
|
||||
let start_time = std::time::Instant::now();
|
||||
|
||||
if let Some(ref tx) = event_tx {
|
||||
let _ = tx.send(ExecutionEvent::TaskStarted {
|
||||
task_id: ctx.task_id.clone(),
|
||||
timestamp: Utc::now(),
|
||||
}).await;
|
||||
}
|
||||
|
||||
let mut command = cmd;
|
||||
command.current_dir(&ctx.working_dir);
|
||||
command.envs(&ctx.env_vars);
|
||||
command.env("WS_TASKID", &ctx.task_id);
|
||||
command.env("WS_PIPELINE", &ctx.pipeline_name);
|
||||
command.env("WS_EUID", ctx.euid.to_string());
|
||||
command.env("WS_EGID", ctx.egid.to_string());
|
||||
command.env("WS_USERNAME", &ctx.username);
|
||||
command.stdin(Stdio::null());
|
||||
command.stdout(Stdio::piped());
|
||||
command.stderr(Stdio::piped());
|
||||
|
||||
let child = command.spawn()
|
||||
.map_err(|e| ExecutionError::InvalidCommand(format!("Failed to spawn command: {}", e)))?;
|
||||
|
||||
let pid = child.id().ok_or_else(|| ExecutionError::IoError("Failed to get PID".to_string()))?;
|
||||
self.apply_cgroup(pid)?;
|
||||
|
||||
let output = child.wait_with_output()
|
||||
.await
|
||||
.map_err(|e| ExecutionError::IoError(format!("Failed to wait for process: {}", e)))?;
|
||||
|
||||
let exit_code = output.status.code().unwrap_or(-1);
|
||||
let duration = start_time.elapsed();
|
||||
|
||||
let result = ExecutionResult {
|
||||
task_id: ctx.task_id.clone(),
|
||||
exit_code,
|
||||
success: exit_code == 0,
|
||||
duration,
|
||||
stdout: String::from_utf8_lossy(&output.stdout).to_string(),
|
||||
stderr: String::from_utf8_lossy(&output.stderr).to_string(),
|
||||
resource_usage: None,
|
||||
};
|
||||
|
||||
if let Some(ref tx) = event_tx {
|
||||
let event = if result.success {
|
||||
ExecutionEvent::TaskCompleted {
|
||||
task_id: ctx.task_id.clone(),
|
||||
result: result.clone(),
|
||||
}
|
||||
} else {
|
||||
ExecutionEvent::TaskFailed {
|
||||
task_id: ctx.task_id.clone(),
|
||||
error: format!("Exit code: {}", exit_code),
|
||||
}
|
||||
};
|
||||
let _ = tx.send(event).await;
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
pub async fn install_dependency(&self, package: &[&str], pkgman: &dyn PackageManager, ctx: &ExecutionContext) -> Result<ExecutionResult, ExecutionError> {
|
||||
let cmd = pkgman.install(package);
|
||||
|
||||
self.execute(cmd, ctx, None).await
|
||||
}
|
||||
|
||||
pub async fn cleanup(&self) -> Result<(), String> {
|
||||
if let Some(ref cgroup) = self.cgroup_manager {
|
||||
cgroup.destroy()
|
||||
.map_err(|e| format!("Failed to destroy cgroup: {}", e))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for LocalExecutor {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
@@ -1,219 +0,0 @@
|
||||
use crate::engine::pm::PackageManager;
|
||||
use os_info::{Info, Version};
|
||||
use tokio::process::Command;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Apt;
|
||||
|
||||
impl PackageManager for Apt {
|
||||
fn name(&self) -> &'static str {
|
||||
"apt"
|
||||
}
|
||||
|
||||
fn adopt(&self, distro: &str) -> bool {
|
||||
// 匹配所有主要的使用 apt 的发行版
|
||||
// 注意：os_info 返回的 Type 转字符串通常是全小写或特定格式，这里做宽松匹配
|
||||
let d = distro.to_lowercase();
|
||||
matches!(
|
||||
d.as_str(),
|
||||
"debian"
|
||||
| "ubuntu"
|
||||
| "linuxmint"
|
||||
| "mint"
|
||||
| "pop"
|
||||
| "pop!_os"
|
||||
| "kali"
|
||||
| "raspbian"
|
||||
| "aosc"
|
||||
| "deepin"
|
||||
| "uos"
|
||||
| "elementary"
|
||||
)
|
||||
}
|
||||
|
||||
fn binary(&self) -> &'static str {
|
||||
"apt"
|
||||
}
|
||||
|
||||
fn update(&self) -> Command {
|
||||
let mut cmd = Command::new("apt");
|
||||
cmd.arg("update");
|
||||
cmd
|
||||
}
|
||||
|
||||
fn install(&self, package: &[&str]) -> Command {
|
||||
let mut cmd = Command::new("apt");
|
||||
cmd.arg("install");
|
||||
cmd.arg("-y"); // 自动确认
|
||||
cmd.args(package);
|
||||
cmd
|
||||
}
|
||||
|
||||
fn change_mirror(&self, repo: &str, osinfo: Option<Info>) -> Command {
|
||||
// 目标仓库 URL (例如: https://mirrors.tuna.tsinghua.edu.cn/debian)
|
||||
// 我们需要构建一个 sed 命令来替换 /etc/apt/sources.list 中的旧 URL
|
||||
|
||||
let codename = extract_codename(osinfo.as_ref()).unwrap_or_else(|| {
|
||||
log::warn!("无法自动获取版本代号，将尝试替换所有行，请谨慎使用。");
|
||||
String::new()
|
||||
});
|
||||
|
||||
// 构造 sed 表达式
|
||||
// 逻辑：找到包含当前 codename 或 generic 的行，替换掉其中的 http/https 域名部分
|
||||
// 为了安全，我们通常建议备份，但这里只生成命令，用户执行前可自行备份或使用 --backup
|
||||
|
||||
// 简单的策略：替换 sources.list 中第一个出现的 http/https 开头直到第一个空格的部分
|
||||
// 更稳健的策略是针对特定发行版模板，但通用 sed 更灵活
|
||||
|
||||
let mut cmd = Command::new("sed");
|
||||
|
||||
// 注意：sed -i 需要权限，通常由用户 sudo 运行此命令
|
||||
cmd.arg("-i.bak"); // 自动备份为 .bak
|
||||
|
||||
if codename.is_empty() {
|
||||
// 如果不知道代号，尝试宽泛替换 (风险较高，仅替换常见源)
|
||||
// 匹配 http://... 或 https://... 直到空格
|
||||
cmd.arg("-E");
|
||||
cmd.arg(format!(
|
||||
r"s|^https?://[^\s]+|{}|g",
|
||||
repo.trim_end_matches('/')
|
||||
));
|
||||
} else {
|
||||
// 如果知道代号，尽量只替换包含该代号的行，减少误伤
|
||||
// 匹配： deb http://old-url codename ... -> deb new-repo codename ...
|
||||
cmd.arg("-E");
|
||||
cmd.arg(format!(
|
||||
r"s|(deb\s+https?://)[^\s]+(\s+{}\s)|\1{}{}\2|g",
|
||||
codename,
|
||||
repo.trim_end_matches('/'),
|
||||
if codename.is_empty() { "" } else { "" } // 占位，实际逻辑在 regex 中
|
||||
));
|
||||
// 修正上面的正则逻辑，使其更准确：
|
||||
// 捕获组1: "deb http://"
|
||||
// 匹配中间: [^\s]+ (旧域名)
|
||||
// 捕获组2: " codename " (包含代号的部分)
|
||||
// 替换为: \1 + 新域名 + \2
|
||||
|
||||
// 重新构建更准确的 sed 参数
|
||||
cmd = Command::new("sed");
|
||||
cmd.arg("-i.bak");
|
||||
cmd.arg("-E");
|
||||
|
||||
let pattern = format!(
|
||||
r"s#(deb\s+https?://)[^\s]+(\s+{}(\s|$))#\1{}\2#g",
|
||||
codename,
|
||||
repo.trim_end_matches('/')
|
||||
);
|
||||
cmd.arg(pattern);
|
||||
}
|
||||
|
||||
cmd.arg("/etc/apt/sources.list");
|
||||
|
||||
// 提示：对于 /etc/apt/sources.list.d/ 下的文件，此命令不会处理，通常需要额外逻辑
|
||||
// 但作为基础实现，修改主文件是最常见的操作
|
||||
cmd
|
||||
}
|
||||
|
||||
fn add_repository(&self, repo: &str, osinfo: Option<Info>) -> Option<Command> {
|
||||
// 策略 1: 优先尝试使用 add-apt-repository (来自 software-properties-common)
|
||||
// 这个命令会自动处理 GPG key 和 list 文件
|
||||
let mut cmd_ppa = Command::new("add-apt-repository");
|
||||
cmd_ppa.arg("-y");
|
||||
cmd_ppa.arg(repo);
|
||||
|
||||
// 我们无法在这里完美检测 add-apt-repository 是否存在而不执行
|
||||
// 但我们可以假设如果它是 Ubuntu/Debian 标准环境，可能有用。
|
||||
// 为了稳妥，我们返回一个 sh -c 脚本，先检查命令是否存在，不存在则 fallback 到手动写入
|
||||
|
||||
let codename = extract_codename(osinfo.as_ref())?;
|
||||
|
||||
// 构建一个 shell 脚本来处理逻辑
|
||||
let script = format!(
|
||||
r#"
|
||||
if command -v add-apt-repository >/dev/null 2>&1; then
|
||||
add-apt-repository -y "{repo}"
|
||||
else
|
||||
echo "deb {repo} {codename} main" | tee /etc/apt/sources.list.d/custom-repo.list
|
||||
fi
|
||||
"#,
|
||||
repo = repo,
|
||||
codename = codename
|
||||
);
|
||||
|
||||
let mut cmd = Command::new("sh");
|
||||
cmd.arg("-c").arg(script);
|
||||
|
||||
Some(cmd)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_adopt_true_for_ubuntu_and_debian() {
|
||||
let apt = Apt;
|
||||
assert!(apt.adopt("ubuntu"));
|
||||
assert!(apt.adopt("debian"));
|
||||
assert!(!apt.adopt("fedora"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_command_contents() {
|
||||
let cmd = Apt.update();
|
||||
let cmd = cmd.as_std();
|
||||
assert_eq!(cmd.get_program().to_string_lossy(), "apt");
|
||||
let args: Vec<String> = cmd
|
||||
.get_args()
|
||||
.map(|a| a.to_string_lossy().into_owned())
|
||||
.collect();
|
||||
assert_eq!(args, vec!["update"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_install_command_contents() {
|
||||
let cmd = Apt.install(&["curl"]);
|
||||
let cmd = cmd.as_std();
|
||||
assert_eq!(cmd.get_program().to_string_lossy(), "apt");
|
||||
let args: Vec<String> = cmd
|
||||
.get_args()
|
||||
.map(|a| a.to_string_lossy().into_owned())
|
||||
.collect();
|
||||
assert_eq!(args, vec!["install", "-y", "curl"]);
|
||||
}
|
||||
}
|
||||
|
||||
/// 辅助函数：从 os_info 中提取 Codename
|
||||
fn extract_codename(info: Option<&Info>) -> Option<String> {
|
||||
let info = info?;
|
||||
|
||||
// 1. 直接获取 codename (os_info 较新版本支持)
|
||||
if let Some(codename) = info.codename() {
|
||||
if !codename.is_empty() {
|
||||
return Some(codename.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
// 2. 尝试从 version 中推断 (针对某些把 codename 放在 Custom 里的情况)
|
||||
match info.version() {
|
||||
Version::Custom(v) => {
|
||||
// 有些系统可能把 codename 放在这里，虽然不标准
|
||||
if !v.chars().next().map(|c| c.is_numeric()).unwrap_or(true) {
|
||||
return Some(v.clone());
|
||||
}
|
||||
}
|
||||
Version::Semantic(major, minor, _) => {
|
||||
// 对于 Ubuntu/Debian，如果没有 codename，有时可以用版本号兜底 (如 22.04)
|
||||
// 但 apt 源通常更喜欢 codename。如果实在没有，返回版本号字符串作为最后手段
|
||||
return Some(format!("{}.{}", major, minor));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
// 3. 如果 os_info 没提供，且是常见发行版，可以尝试根据 Type 猜测？
|
||||
// 不推荐硬编码猜测，因为版本太多。
|
||||
// 最好返回 None，让调用者决定是报错还是让用户手动指定。
|
||||
|
||||
None
|
||||
}
|
||||
@@ -1,133 +0,0 @@
|
||||
use crate::engine::pm::PackageManager;
|
||||
use os_info::Info;
|
||||
use tokio::process::Command;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Dnf;
|
||||
|
||||
impl PackageManager for Dnf {
|
||||
fn name(&self) -> &'static str {
|
||||
"dnf"
|
||||
}
|
||||
|
||||
fn adopt(&self, distro: &str) -> bool {
|
||||
let d = distro.to_lowercase();
|
||||
matches!(
|
||||
d.as_str(),
|
||||
"fedora" | "rhel" | "centos" | "almalinux" | "rocky"
|
||||
)
|
||||
}
|
||||
|
||||
fn binary(&self) -> &'static str {
|
||||
"dnf"
|
||||
}
|
||||
|
||||
fn update(&self) -> Command {
|
||||
let mut cmd = Command::new("dnf");
|
||||
cmd.arg("makecache");
|
||||
cmd
|
||||
}
|
||||
|
||||
fn install(&self, package: &[&str]) -> Command {
|
||||
let mut cmd = Command::new("dnf");
|
||||
cmd.arg("install");
|
||||
cmd.arg("-y"); // auto-confirm
|
||||
cmd.args(package);
|
||||
cmd
|
||||
}
|
||||
|
||||
fn change_mirror(&self, repo: &str, _osinfo: Option<Info>) -> Command {
|
||||
// Change baseurl in yum repos under /etc/yum.repos.d
|
||||
// This is a best-effort: replace baseurl with the provided repo URL
|
||||
let mut cmd = Command::new("sed");
|
||||
cmd.arg("-i.bak");
|
||||
cmd.arg(format!("s#baseurl=.*#{}#g", repo));
|
||||
// The target files
|
||||
cmd.arg("/etc/yum.repos.d/*.repo");
|
||||
cmd
|
||||
}
|
||||
|
||||
fn add_repository(&self, repo: &str, _osinfo: Option<Info>) -> Option<Command> {
|
||||
// Append a minimal repo file under /etc/yum.repos.d
|
||||
let script = format!(
|
||||
"printf '%s' '[custom]\nname=Custom Repo\nbaseurl={}%\nenabled=1\n' >> /etc/yum.repos.d/custom.repo",
|
||||
repo
|
||||
);
|
||||
|
||||
// Run via bash -lc to avoid escaping issues
|
||||
let mut cmd = Command::new("bash");
|
||||
cmd.arg("-lc").arg(script);
|
||||
Some(cmd)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_adopt_fedora_family() {
|
||||
let dnf = Dnf;
|
||||
assert!(dnf.adopt("Fedora"));
|
||||
assert!(dnf.adopt("RHEL"));
|
||||
assert!(dnf.adopt("centos"));
|
||||
assert!(dnf.adopt("AlmaLinux"));
|
||||
assert!(dnf.adopt("Rocky"));
|
||||
assert!(!dnf.adopt("ubuntu"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_command() {
|
||||
let cmd = Dnf.update();
|
||||
let cmd = cmd.as_std();
|
||||
assert_eq!(cmd.get_program().to_string_lossy(), "dnf");
|
||||
let args: Vec<String> = cmd
|
||||
.get_args()
|
||||
.map(|a| a.to_string_lossy().into_owned())
|
||||
.collect();
|
||||
assert_eq!(args, vec!["makecache"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_install_command() {
|
||||
let cmd = Dnf.install(&["git", "curl"]);
|
||||
let cmd = cmd.as_std();
|
||||
assert_eq!(cmd.get_program().to_string_lossy(), "dnf");
|
||||
let args: Vec<String> = cmd
|
||||
.get_args()
|
||||
.map(|a| a.to_string_lossy().into_owned())
|
||||
.collect();
|
||||
assert_eq!(args, vec!["install", "-y", "git", "curl"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_change_mirror_command_basic() {
|
||||
// Provide a dummy repo url and verify the command program and a couple of args
|
||||
let cmd = Dnf.change_mirror("https://example.repo/", None);
|
||||
let cmd = cmd.as_std();
|
||||
assert_eq!(cmd.get_program().to_string_lossy(), "sed");
|
||||
let args: Vec<String> = cmd
|
||||
.get_args()
|
||||
.map(|a| a.to_string_lossy().into_owned())
|
||||
.collect();
|
||||
// Should include -i.bak and the target path
|
||||
assert!(args.contains(&"-i.bak".to_string()));
|
||||
assert!(args.iter().any(|s| s.contains("/etc/yum.repos.d/*.repo")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_repository_command_exists() {
|
||||
let maybe_cmd = Dnf.add_repository("https://example.repo/", None);
|
||||
assert!(maybe_cmd.is_some());
|
||||
let cmd = maybe_cmd.unwrap();
|
||||
let cmd = cmd.as_std();
|
||||
// The command should be a shell invocation executing a script
|
||||
assert_eq!(cmd.get_program().to_string_lossy(), "bash");
|
||||
let args: Vec<String> = cmd
|
||||
.get_args()
|
||||
.map(|a| a.to_string_lossy().into_owned())
|
||||
.collect();
|
||||
// Ensure the script contains the repo URL
|
||||
assert!(args.iter().any(|s| s.contains("https://example.repo/")));
|
||||
}
|
||||
}
|
||||
@@ -1,124 +0,0 @@
|
||||
use crate::engine::pm::PackageManager;
|
||||
use os_info::Info;
|
||||
use tokio::process::Command;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Pacman;
|
||||
|
||||
impl PackageManager for Pacman {
|
||||
fn name(&self) -> &'static str {
|
||||
"pacman"
|
||||
}
|
||||
|
||||
fn adopt(&self, distro: &str) -> bool {
|
||||
let d = distro.to_lowercase();
|
||||
matches!(
|
||||
d.as_str(),
|
||||
"arch"
|
||||
| "manjaro"
|
||||
| "endeavouros"
|
||||
| "garuda"
|
||||
| "cachyos"
|
||||
| "artix"
|
||||
| "mabox"
|
||||
| "nobara"
|
||||
| "instantos"
|
||||
| "archlinux"
|
||||
)
|
||||
}
|
||||
|
||||
fn binary(&self) -> &'static str {
|
||||
"pacman"
|
||||
}
|
||||
|
||||
fn update(&self) -> Command {
|
||||
let mut cmd = Command::new("pacman");
|
||||
cmd.arg("-Sy"); // 仅刷新数据库，不升级包（通常换源后需要这一步）
|
||||
cmd
|
||||
}
|
||||
|
||||
fn install(&self, package: &[&str]) -> Command {
|
||||
let mut cmd = Command::new("pacman");
|
||||
cmd.arg("-S"); // 同步/安装
|
||||
cmd.arg("--noconfirm"); // 自动确认
|
||||
cmd.arg("--needed");
|
||||
cmd.args(package);
|
||||
cmd
|
||||
}
|
||||
|
||||
fn change_mirror(&self, repo: &str, osinfo: Option<Info>) -> Command {
|
||||
let mirrorlist_path = "/etc/pacman.d/mirrorlist";
|
||||
|
||||
let content = format!(
|
||||
"# Generated by WSExecutor-PM\n\
|
||||
# Date: $(date)\n\
|
||||
# Original content overwritten.\n\
|
||||
\n\
|
||||
Server = {}\n",
|
||||
repo
|
||||
);
|
||||
|
||||
// echo "content" | sudo tee /path/to/file > /dev/null
|
||||
let safe_content = content.replace("'", "'\"'\"'"); // 转义单引号: ' -> '\''
|
||||
|
||||
let mut cmd = Command::new("sh");
|
||||
cmd.arg("-c").arg(format!(
|
||||
"printf '%s' '{}' | sudo tee {} > /dev/null",
|
||||
safe_content, mirrorlist_path
|
||||
));
|
||||
|
||||
cmd
|
||||
}
|
||||
|
||||
fn add_repository(&self, repo: &str, osinfo: Option<Info>) -> Option<Command> {
|
||||
let pacman_conf = "/etc/pacman.conf";
|
||||
let content = format!(
|
||||
"[custom]\nServer = {}\nSigLevel = Optional TrustAll\n",
|
||||
repo,
|
||||
);
|
||||
let safe_content = content.replace("'", "'\"'\"'");
|
||||
|
||||
let mut cmd = Command::new("sh");
|
||||
cmd.arg("-c")
|
||||
.arg(format!("echo '{}' >> {}", safe_content, pacman_conf));
|
||||
|
||||
Some(cmd)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_adopt_arch_like() {
|
||||
let pac = Pacman;
|
||||
assert!(pac.adopt("arch"));
|
||||
assert!(pac.adopt("manjaro"));
|
||||
assert!(!pac.adopt("ubuntu"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_command_contents() {
|
||||
let cmd = Pacman.update();
|
||||
let cmd = cmd.as_std();
|
||||
assert_eq!(cmd.get_program().to_string_lossy(), "pacman");
|
||||
let args: Vec<String> = cmd
|
||||
.get_args()
|
||||
.map(|a| a.to_string_lossy().into_owned())
|
||||
.collect();
|
||||
assert_eq!(args, vec!["-Sy"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_install_command_contents() {
|
||||
let cmd = Pacman.install(&["vim"]);
|
||||
let cmd = cmd.as_std();
|
||||
assert_eq!(cmd.get_program().to_string_lossy(), "pacman");
|
||||
let args: Vec<String> = cmd
|
||||
.get_args()
|
||||
.map(|a| a.to_string_lossy().into_owned())
|
||||
.collect();
|
||||
assert_eq!(args, vec!["-S", "--noconfirm", "--needed", "vim"]);
|
||||
}
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Error, Debug, Clone)]
|
||||
pub enum ExecutionError {
|
||||
#[error("IO error: {0}")]
|
||||
IoError(String),
|
||||
|
||||
#[error("Parse error: {0}")]
|
||||
ParseError(String),
|
||||
|
||||
#[error("Variable error: {0}")]
|
||||
VariableError(String),
|
||||
|
||||
#[error("Execution error: {0}")]
|
||||
ExecutionError(String),
|
||||
|
||||
#[error("Circular dependency detected")]
|
||||
CircularDependency,
|
||||
|
||||
#[error("Dependency not satisfied: {0}")]
|
||||
DependencyNotSatisfied(String),
|
||||
|
||||
#[error("Parallel execution error: {0}")]
|
||||
ParallelExecutionError(String),
|
||||
|
||||
#[error("Timeout error: {0}")]
|
||||
TimeoutError(String),
|
||||
}
|
||||
|
||||
#[derive(Error, Debug, Clone)]
|
||||
pub enum BuilderError {
|
||||
#[error("IO error: {0}")]
|
||||
IoError(String),
|
||||
|
||||
#[error("Template error: {0}")]
|
||||
TemplateError(String),
|
||||
}
|
||||
|
||||
impl From<std::io::Error> for BuilderError {
|
||||
fn from(err: std::io::Error) -> Self {
|
||||
BuilderError::IoError(err.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<minijinja::Error> for BuilderError {
|
||||
fn from(err: minijinja::Error) -> Self {
|
||||
BuilderError::TemplateError(err.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<tempfile::PersistError> for BuilderError {
|
||||
fn from(err: tempfile::PersistError) -> Self {
|
||||
BuilderError::IoError(err.to_string())
|
||||
}
|
||||
}
|
||||
@@ -1,123 +0,0 @@
|
||||
mod builder;
|
||||
mod daemon;
|
||||
mod decorator;
|
||||
mod engine;
|
||||
mod error;
|
||||
mod monitor;
|
||||
mod parser;
|
||||
mod schedule;
|
||||
mod variable;
|
||||
mod socket;
|
||||
mod prebake;
|
||||
use clap::{Parser, Subcommand};
|
||||
use config::Config;
|
||||
use env_logger;
|
||||
use serde::Deserialize;
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use crate::daemon::daemon;
|
||||
use crate::monitor::monitor;
|
||||
use crate::prebake::prebake;
|
||||
|
||||
/// Honey Biscuit Workshop script executor
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(version, about, long_about=None)]
|
||||
struct Cli {
|
||||
/// Username of the pipeline executor
|
||||
#[arg(short, long)]
|
||||
username: String,
|
||||
|
||||
/// Pipeline ID of this pipeline
|
||||
#[arg(short, long)]
|
||||
pipeline: String,
|
||||
|
||||
/// Build ID of this build
|
||||
#[arg(short, long)]
|
||||
build_id: String,
|
||||
|
||||
/// Parse the script without executing
|
||||
#[arg(long, default_value = "false")]
|
||||
dry_run: bool,
|
||||
|
||||
#[command(subcommand)]
|
||||
command: Option<Commands>,
|
||||
}
|
||||
|
||||
#[derive(Subcommand, Debug)]
|
||||
enum Commands {
|
||||
/// Monitor a running task
|
||||
Monitor { group: String },
|
||||
/// Prepare a task
|
||||
Prebake { config: String },
|
||||
/// Run a task
|
||||
Bake {
|
||||
/// Path to the script to execute
|
||||
script: String,
|
||||
/// Path to bake_base.sh
|
||||
#[arg(short, long, default_value = "./bake_base.sh")]
|
||||
bake_base: String,
|
||||
},
|
||||
/// Finish a task
|
||||
Finalize { config: String },
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct TaskSpec {
|
||||
// pub stage: String,
|
||||
pub subcommand: String,
|
||||
pub arguments: Vec<String>,
|
||||
pub options: HashMap<String, String>,
|
||||
}
|
||||
|
||||
impl TaskSpec {
|
||||
fn command(&self, executable: &PathBuf) -> tokio::process::Command {
|
||||
let mut command = tokio::process::Command::new(executable);
|
||||
command.arg(&self.subcommand);
|
||||
for arg in &self.arguments {
|
||||
command.arg(arg);
|
||||
}
|
||||
for (flag, value) in self.options.iter() {
|
||||
if value.is_empty() {
|
||||
command.arg(format!("--{}", flag));
|
||||
} else {
|
||||
command.arg(format!("--{}={}", flag, value));
|
||||
}
|
||||
}
|
||||
command
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
env_logger::init();
|
||||
let cli = Cli::parse();
|
||||
let settings = Config::builder()
|
||||
.set_default("pipeline", cli.pipeline.clone()).unwrap()
|
||||
.add_source(config::Environment::with_prefix("WSEXECUTOR_"))
|
||||
.build()
|
||||
.unwrap();
|
||||
match &cli.command {
|
||||
Some(Commands::Monitor { group }) => {
|
||||
log::info!("Starting monitor for group: {}", group);
|
||||
let result = monitor(group, Some(1000), settings).await;
|
||||
result.unwrap();
|
||||
}
|
||||
Some(Commands::Prebake { config }) => {
|
||||
let result = prebake(config, &settings).await;
|
||||
result.unwrap();
|
||||
}
|
||||
Some(Commands::Bake { script, bake_base }) => {
|
||||
// builder::bake(script, bake_base, username, pipeline, *dry_run);
|
||||
}
|
||||
Some(Commands::Finalize { config }) => {
|
||||
// builder::finalize(config);
|
||||
}
|
||||
None => {
|
||||
// Running in daemon mode
|
||||
log::info!("Executor will be running in daemon mode.");
|
||||
|
||||
let result = daemon(&cli, settings).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,518 +0,0 @@
|
||||
mod environment;
|
||||
use serde::{Deserialize, Serialize};
|
||||
// use serde_with::{serde_as, DisplayFromStr, NoneAsEmptyString};
|
||||
use std::collections::HashMap;
|
||||
// use std::path::PathBuf;
|
||||
use semver::{Version, VersionReq};
|
||||
use config::Config;
|
||||
|
||||
use crate::prebake::environment::prepare_environment;
|
||||
|
||||
const VERSION_REQUIREMENT: &str = "^1.0";
|
||||
|
||||
// 顶层配置结构
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct PrebakeConfig {
|
||||
pub version: String,
|
||||
pub environment: Environment,
|
||||
#[serde(default)]
|
||||
pub dependencies: Option<Dependencies>,
|
||||
#[serde(default)]
|
||||
pub environment_variables: Option<EnvironmentVariables>,
|
||||
#[serde(default)]
|
||||
pub cache: Option<Cache>,
|
||||
#[serde(default)]
|
||||
pub security: Option<Security>,
|
||||
#[serde(default)]
|
||||
pub hooks: Option<Hooks>,
|
||||
#[serde(default)]
|
||||
pub metadata: Option<Metadata>,
|
||||
}
|
||||
|
||||
// 构建环境配置
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct Environment {
|
||||
pub builder: BuilderType,
|
||||
#[serde(default)]
|
||||
pub config: Option<BuilderConfig>,
|
||||
#[serde(default)]
|
||||
pub resources: Option<ResourceRequirements>,
|
||||
#[serde(default)]
|
||||
pub network: Option<NetworkConfig>,
|
||||
}
|
||||
|
||||
// 构建器类型枚举
|
||||
#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum BuilderType {
|
||||
Docker,
|
||||
Firecracker,
|
||||
Custom,
|
||||
Baremetal,
|
||||
}
|
||||
|
||||
// 构建器配置 - 使用枚举来处理不同类型的配置
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
#[serde(untagged)]
|
||||
pub enum BuilderConfig {
|
||||
Docker(DockerConfig),
|
||||
Firecracker(FirecrackerConfig),
|
||||
Custom(CustomConfig),
|
||||
Baremetal(BaremetalConfig),
|
||||
}
|
||||
|
||||
// 为每种构建器类型实现From trait以便转换
|
||||
impl From<DockerConfig> for BuilderConfig {
|
||||
fn from(config: DockerConfig) -> Self {
|
||||
BuilderConfig::Docker(config)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<FirecrackerConfig> for BuilderConfig {
|
||||
fn from(config: FirecrackerConfig) -> Self {
|
||||
BuilderConfig::Firecracker(config)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<CustomConfig> for BuilderConfig {
|
||||
fn from(config: CustomConfig) -> Self {
|
||||
BuilderConfig::Custom(config)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<BaremetalConfig> for BuilderConfig {
|
||||
fn from(_: BaremetalConfig) -> Self {
|
||||
BuilderConfig::Baremetal(BaremetalConfig {})
|
||||
}
|
||||
}
|
||||
|
||||
// Docker配置
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct DockerConfig {
|
||||
#[serde(default)]
|
||||
pub image: Option<String>,
|
||||
#[serde(default)]
|
||||
pub dockerfile: Option<String>,
|
||||
#[serde(default)]
|
||||
pub build_args: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
// Firecracker配置（目前为空）
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct FirecrackerConfig {
|
||||
// 留空，根据文档说明
|
||||
}
|
||||
|
||||
// 自定义构建器配置
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct CustomConfig {
|
||||
pub name: String,
|
||||
pub setup_script: String,
|
||||
pub cleanup_script: String,
|
||||
}
|
||||
|
||||
// Baremetal配置（空结构体）
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct BaremetalConfig {}
|
||||
|
||||
// 资源要求
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct ResourceRequirements {
|
||||
#[serde(default = "default_cpu")]
|
||||
pub cpu: u32,
|
||||
|
||||
#[serde(default = "default_memory")]
|
||||
pub memory: String,
|
||||
|
||||
#[serde(default = "default_disk")]
|
||||
pub disk: String,
|
||||
|
||||
pub architecture: Architecture,
|
||||
|
||||
#[serde(default = "default_gpu")]
|
||||
pub gpu: bool,
|
||||
|
||||
#[serde(default)]
|
||||
pub gpu_type: Option<Vec<GpuType>>,
|
||||
|
||||
#[serde(default)]
|
||||
pub tags: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
fn default_cpu() -> u32 {
|
||||
1
|
||||
}
|
||||
fn default_memory() -> String {
|
||||
"1G".to_string()
|
||||
}
|
||||
fn default_disk() -> String {
|
||||
"1G".to_string()
|
||||
}
|
||||
fn default_gpu() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
// 架构类型
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
#[serde(untagged)]
|
||||
pub enum Architecture {
|
||||
Single(String),
|
||||
Multiple(Vec<String>),
|
||||
}
|
||||
|
||||
impl Default for Architecture {
|
||||
fn default() -> Self {
|
||||
Architecture::Single("x86_64".to_string())
|
||||
}
|
||||
}
|
||||
|
||||
// GPU类型枚举
|
||||
#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum GpuType {
|
||||
Cuda,
|
||||
Rocm,
|
||||
Vulkan,
|
||||
Oneapi,
|
||||
}
|
||||
|
||||
// 网络配置
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct NetworkConfig {
|
||||
#[serde(default = "default_enabled")]
|
||||
pub enabled: bool,
|
||||
|
||||
#[serde(default = "default_outbound")]
|
||||
pub outbound: bool,
|
||||
|
||||
#[serde(default)]
|
||||
pub dns_servers: Option<Vec<String>>,
|
||||
|
||||
#[serde(default)]
|
||||
pub proxies: Option<Vec<ProxyConfig>>,
|
||||
}
|
||||
|
||||
fn default_enabled() -> bool {
|
||||
true
|
||||
}
|
||||
fn default_outbound() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct ProxyConfig {
|
||||
// 代理配置结构，根据需求扩展
|
||||
}
|
||||
|
||||
// 依赖配置
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct Dependencies {
|
||||
#[serde(default)]
|
||||
pub system: Option<SystemDependencies>,
|
||||
|
||||
#[serde(default)]
|
||||
pub languages: Option<HashMap<String, LanguageDependency>>,
|
||||
|
||||
#[serde(default)]
|
||||
pub custom: Option<Vec<CustomDependency>>,
|
||||
|
||||
#[serde(default, rename = "custom_pre")]
|
||||
pub custom_pre: Option<Vec<CustomDependency>>,
|
||||
}
|
||||
|
||||
// 系统依赖
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct SystemDependencies {
|
||||
#[serde(default)]
|
||||
pub deb: Option<PackageManagerConfig>,
|
||||
|
||||
#[serde(default)]
|
||||
pub pacman: Option<PackageManagerConfig>,
|
||||
|
||||
#[serde(default)]
|
||||
pub rpm: Option<PackageManagerConfig>,
|
||||
// 可以添加其他包管理器
|
||||
}
|
||||
|
||||
// 包管理器配置
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct PackageManagerConfig {
|
||||
pub packages: Vec<String>,
|
||||
|
||||
#[serde(default)]
|
||||
pub repositories: Option<Vec<String>>,
|
||||
|
||||
#[serde(default)]
|
||||
pub mirror: Option<String>,
|
||||
}
|
||||
|
||||
// 语言依赖
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct LanguageDependency {
|
||||
pub toolchain: String,
|
||||
|
||||
#[serde(default)]
|
||||
pub version: Option<String>,
|
||||
|
||||
#[serde(default)]
|
||||
pub features: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
// 自定义依赖
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct CustomDependency {
|
||||
pub name: String,
|
||||
pub command: String,
|
||||
|
||||
#[serde(default)]
|
||||
pub environment: Option<HashMap<String, String>>,
|
||||
|
||||
#[serde(default)]
|
||||
pub working_dir: Option<String>,
|
||||
|
||||
#[serde(default = "default_timeout")]
|
||||
pub timeout: u32,
|
||||
}
|
||||
|
||||
fn default_timeout() -> u32 {
|
||||
300
|
||||
}
|
||||
|
||||
// 环境变量配置
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct EnvironmentVariables {
|
||||
#[serde(default)]
|
||||
pub prebake: Option<HashMap<String, String>>,
|
||||
|
||||
#[serde(default)]
|
||||
pub bake: Option<HashMap<String, String>>,
|
||||
}
|
||||
|
||||
// 缓存配置
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct Cache {
|
||||
pub directories: Vec<CacheDirectory>,
|
||||
|
||||
#[serde(default)]
|
||||
pub strategy: Option<CacheStrategy>,
|
||||
}
|
||||
|
||||
// 缓存目录
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct CacheDirectory {
|
||||
pub path: String,
|
||||
|
||||
#[serde(default = "default_strategy")]
|
||||
pub strategy: CacheStrategyType,
|
||||
|
||||
#[serde(default = "default_mode")]
|
||||
pub mode: CacheMode,
|
||||
}
|
||||
|
||||
fn default_strategy() -> CacheStrategyType {
|
||||
CacheStrategyType::Always
|
||||
}
|
||||
fn default_mode() -> CacheMode {
|
||||
CacheMode::Zstd
|
||||
}
|
||||
|
||||
// 缓存策略类型
|
||||
#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum CacheStrategyType {
|
||||
Always,
|
||||
Readonly,
|
||||
Clean,
|
||||
Never,
|
||||
}
|
||||
|
||||
// 缓存模式
|
||||
#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum CacheMode {
|
||||
Gzip,
|
||||
Zstd,
|
||||
None,
|
||||
Dir,
|
||||
}
|
||||
|
||||
// 缓存策略
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct CacheStrategy {
|
||||
#[serde(default = "default_ttl_days", rename = "ttl_days")]
|
||||
pub ttl_days: u32,
|
||||
|
||||
#[serde(default = "default_max_size_gb", rename = "max_size_gb")]
|
||||
pub max_size_gb: u32,
|
||||
|
||||
#[serde(default = "default_cleanup_policy", rename = "cleanup_policy")]
|
||||
pub cleanup_policy: CleanupPolicy,
|
||||
}
|
||||
|
||||
fn default_ttl_days() -> u32 {
|
||||
30
|
||||
}
|
||||
fn default_max_size_gb() -> u32 {
|
||||
20
|
||||
}
|
||||
fn default_cleanup_policy() -> CleanupPolicy {
|
||||
CleanupPolicy::Lru
|
||||
}
|
||||
|
||||
// 清理策略
|
||||
#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum CleanupPolicy {
|
||||
Lru,
|
||||
Fifo,
|
||||
Size,
|
||||
}
|
||||
|
||||
// 安全配置（目前为空）
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct Security {
|
||||
// 根据文档留空，后续扩展
|
||||
}
|
||||
|
||||
// 钩子脚本
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct Hooks {
|
||||
#[serde(default, rename = "pre_dependencies")]
|
||||
pub pre_dependencies: Option<Vec<HookCommand>>,
|
||||
|
||||
#[serde(default, rename = "post_dependencies")]
|
||||
pub post_dependencies: Option<Vec<HookCommand>>,
|
||||
|
||||
#[serde(default)]
|
||||
pub ready: Option<Vec<HookCommand>>,
|
||||
}
|
||||
|
||||
// 钩子命令
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct HookCommand {
|
||||
pub command: String,
|
||||
|
||||
#[serde(default)]
|
||||
pub working_dir: Option<String>,
|
||||
|
||||
#[serde(default)]
|
||||
pub environment: Option<HashMap<String, String>>,
|
||||
|
||||
#[serde(default = "default_hook_timeout")]
|
||||
pub timeout: u32,
|
||||
}
|
||||
|
||||
fn default_hook_timeout() -> u32 {
|
||||
300
|
||||
}
|
||||
|
||||
// 元数据
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct Metadata {
|
||||
#[serde(default)]
|
||||
pub description: Option<String>,
|
||||
|
||||
#[serde(default)]
|
||||
pub maintainer: Option<String>,
|
||||
|
||||
#[serde(default)]
|
||||
pub tags: Option<Vec<String>>,
|
||||
|
||||
#[serde(default, rename = "project_type")]
|
||||
pub project_type: Option<String>,
|
||||
}
|
||||
|
||||
// 反序列化函数
|
||||
pub fn parse_prebake_config(yaml_content: &str) -> Result<PrebakeConfig, serde_yaml::Error> {
|
||||
serde_yaml::from_str(yaml_content)
|
||||
}
|
||||
|
||||
// 序列化函数
|
||||
pub fn serialize_prebake_config(config: &PrebakeConfig) -> Result<String, serde_yaml::Error> {
|
||||
serde_yaml::to_string(config)
|
||||
}
|
||||
|
||||
// 验证函数示例
|
||||
impl PrebakeConfig {
|
||||
pub fn validate(&self) -> Result<(), Vec<String>> {
|
||||
let mut errors = Vec::new();
|
||||
|
||||
let version_requirement =
|
||||
VersionReq::parse(VERSION_REQUIREMENT).expect("Invalid version requirement");
|
||||
match Version::parse(&self.version) {
|
||||
Ok(version) if version_requirement.matches(&version) => {},
|
||||
Ok(version) => errors.push(format!(
|
||||
"Version {} does not satisfy requirement {}",
|
||||
version, VERSION_REQUIREMENT
|
||||
)),
|
||||
Err(e) => errors.push(format!("Invalid version format: {}", e)),
|
||||
}
|
||||
|
||||
match (&self.environment.builder, &self.environment.config) {
|
||||
(BuilderType::Docker, Some(BuilderConfig::Docker(docker))) => {
|
||||
if docker.image.is_some() && docker.dockerfile.is_some() {
|
||||
errors.push("At most one of image and dockerfile can be specified".to_string());
|
||||
}
|
||||
if docker.image.is_none() && docker.dockerfile.is_none() {
|
||||
errors.push("At least one of image or dockerfile must be specified".to_string());
|
||||
}
|
||||
}
|
||||
(BuilderType::Firecracker, Some(BuilderConfig::Firecracker(firecracker))) => {
|
||||
errors.push("Firecracker is not supported".to_string());
|
||||
}
|
||||
(BuilderType::Custom, Some(BuilderConfig::Custom(custom))) => {
|
||||
if custom.name.is_empty() {
|
||||
errors.push("Custom builder name cannot be empty".to_string());
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
// TODO: 完成强劲的Validator口牙
|
||||
|
||||
if errors.is_empty() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(errors)
|
||||
}
|
||||
}
|
||||
|
||||
// 辅助方法：获取构建器类型对应的配置
|
||||
pub fn get_builder_config(&self) -> Option<&BuilderConfig> {
|
||||
self.environment.config.as_ref()
|
||||
}
|
||||
|
||||
// 辅助方法：获取Docker配置（如果存在）
|
||||
pub fn get_docker_config(&self) -> Option<&DockerConfig> {
|
||||
match &self.environment.config {
|
||||
Some(BuilderConfig::Docker(docker)) => Some(docker),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn prebake(config: &String, settings: &Config) -> anyhow::Result<()> {
|
||||
let config = std::fs::read_to_string(config)?;
|
||||
let config = parse_prebake_config(config.as_str())?;
|
||||
let result = config.validate();
|
||||
if result.is_err() {
|
||||
for error in result.err().unwrap() {
|
||||
println!("Error: {}", error);
|
||||
}
|
||||
}
|
||||
|
||||
// Begin Stage 1 Prebake
|
||||
// Stage 1 Prebake means preparation that do not need project file.
|
||||
|
||||
let result = prepare_environment(&config, settings).await;
|
||||
// if result.is_err() {
|
||||
// for error in result.err().unwrap() {
|
||||
// println!("Error: {}", error);
|
||||
// }
|
||||
// }
|
||||
dbg!(&result);
|
||||
result.unwrap();
|
||||
|
||||
|
||||
todo!();
|
||||
}
|
||||
Reference in New Issue
Block a user