Compare commits
22 Commits
84b3b7d8e8
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
| df5026cdbe | |||
| d1ad08ef3a | |||
| cf2968e720 | |||
| 28abc5d207 | |||
| 6e7b96cd66 | |||
| 3e80825d58 | |||
| c97eafab29 | |||
| f737b6a24d | |||
| 681b7d1333 | |||
| f07588a716 | |||
| b0530fb858 | |||
| fd18b547e5 | |||
| cf317b55c6 | |||
| 20e2ab4224 | |||
| 50b42e654c | |||
| a64e47d772 | |||
| 8fd807f174 | |||
| e8ce2eec9e | |||
| 9e45bdbbfb | |||
| 5ce8b72900 | |||
| 2c9d32e954 | |||
| 901005639b |
+20
-14
@@ -5,27 +5,33 @@ on:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'workshop-baker/**'
|
||||
- 'workshop-engine/**'
|
||||
- '.github/workflows/ci.yml'
|
||||
pull_request:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'workshop-baker/**'
|
||||
- 'workshop-engine/**'
|
||||
- '.github/workflows/ci.yml'
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
jobs:
|
||||
check-and-lint:
|
||||
name: Check & Lint
|
||||
ci:
|
||||
name: ${{ matrix.crate }}
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
crate: [workshop-baker, workshop-engine]
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: ./workshop-baker
|
||||
working-directory: ./${{ matrix.crate }}
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
@@ -38,19 +44,19 @@ jobs:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
target
|
||||
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
|
||||
${{ matrix.crate }}/target
|
||||
key: ${{ runner.os }}-cargo-${{ matrix.crate }}-${{ hashFiles(format('{0}/Cargo.lock', matrix.crate)) }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cargo-
|
||||
${{ runner.os }}-cargo-${{ matrix.crate }}-
|
||||
|
||||
- name: Check compilation
|
||||
run: cargo check --all-features
|
||||
|
||||
- name: Run tests
|
||||
run: cargo test --all-features
|
||||
|
||||
- name: Check formatting
|
||||
run: cargo fmt -- --check
|
||||
|
||||
- name: Check code (cargo check)
|
||||
run: cargo check --all-features
|
||||
|
||||
- name: Run Clippy lints
|
||||
run: cargo clippy --all-features -- -D warnings
|
||||
|
||||
- name: Run tests
|
||||
run: cargo test --all-features
|
||||
|
||||
+3
-1
@@ -9,7 +9,7 @@ archived
|
||||
*.bak
|
||||
*.clean
|
||||
session*
|
||||
llm.env
|
||||
.secret
|
||||
|
||||
# System test artifacts
|
||||
tests/results/
|
||||
@@ -18,3 +18,5 @@ tests/**/__pycache__/
|
||||
tests/.pytest_cache/
|
||||
tests/llm/.cache/
|
||||
*.pyc
|
||||
.staging
|
||||
.pytest_cache
|
||||
|
||||
@@ -1,82 +1,72 @@
|
||||
# PROJECT KNOWLEDGE BASE
|
||||
|
||||
**Generated:** 2026-04-22
|
||||
**Commit:** 7b3b71a
|
||||
**Generated:** 2026-05-19
|
||||
**Commit:** 28abc5d
|
||||
**Branch:** master
|
||||
|
||||
## OVERVIEW
|
||||
HoneyBiscuitWorkshop is a Rust-based CI/CD system ("蜜饼工坊") with LLM-powered pipeline auto-configuration. 9 independent Cargo crates, no workspace.
|
||||
HoneyBiscuitWorkshop ("蜜饼工坊") — Rust CI/CD system with decorator-driven shell pipeline DSL and LLM-powered configuration. 3 independent Cargo crates, no workspace.
|
||||
|
||||
## STRUCTURE
|
||||
```
|
||||
./
|
||||
├── workshop-baker/ # Main orchestrator (CLI + daemon)
|
||||
├── workshop-agent/ # HTTP agent server
|
||||
├── workshop-pipeline/ # Config library (prebake/finalize)
|
||||
├── workshop-llm-detector/ # LLM repo analyzer
|
||||
├── workshop-vault/ # Secret management client
|
||||
├── workshop-cert/ # TLS certificate generator
|
||||
├── workshop-deviceid/ # Device ID library
|
||||
├── workshop-builder-native/ # Stub (unused)
|
||||
├── workshop-helper-mac/ # MAC address utility
|
||||
├── workshop-baker/ # Main orchestrator (CLI + pipeline stages)
|
||||
├── workshop-engine/ # Execution engine (extracted from baker)
|
||||
├── workshop-getterurl/ # URL fetching (go-getter style)
|
||||
├── docs/ # mdBook documentation (zh-CN)
|
||||
├── playground/ # Experimentation
|
||||
└── third_party/ # Empty
|
||||
├── templates/ # Pipeline templates (bake.sh, finalize.yml, prebake.yml)
|
||||
└── archived/ # Deprecated crates (7 moved here)
|
||||
```
|
||||
|
||||
## WHERE TO LOOK
|
||||
| Task | Location | Notes |
|
||||
|------|----------|-------|
|
||||
| Baker (main) | `workshop-baker/src/{bake,prebake,engine}/` | Core pipeline logic |
|
||||
| Agent server | `workshop-agent/src/` | Actix-web HTTP + TLS |
|
||||
| Pipeline config | `workshop-pipeline/src/config/` | prebake/finalize YAML |
|
||||
| LLM integration | `workshop-llm-detector/src/llm/` | Ollama/OpenAI clients |
|
||||
| Secrets | `workshop-vault/src/` | Vault client wrapper |
|
||||
| Baker (CLI) | `workshop-baker/src/{lib,main}.rs` | Clap CLI + tokio entry |
|
||||
| Pipeline stages | `workshop-baker/src/{bake,prebake,finalize}/` | Stage orchestration |
|
||||
| Notifications | `workshop-baker/src/notify/` | Plugin-based event notifications |
|
||||
|| Engine (execution) | `workshop-engine/src/` | Process mgmt, PM detection |
|
||||
| URL fetching | `workshop-getterurl/src/` | Git/HTTP/File resource getter |
|
||||
|
||||
## CODE MAP
|
||||
| Symbol | Type | Location | Role |
|
||||
|--------|------|----------|------|
|
||||
| PipelineConfig | struct | workshop-pipeline/src/lib.rs:27 | Full pipeline config |
|
||||
| CICDLLMHelper | struct | workshop-llm-detector/src/lib.rs:16 | LLM analysis entry |
|
||||
| Engine | struct | workshop-baker/src/engine.rs | Build executor |
|
||||
| VaultClient | struct | workshop-vault/src/client.rs | Secret ops |
|
||||
| Cli | struct | workshop-baker/src/lib.rs:21 | CLI argument struct |
|
||||
| ExecutionContext | struct | workshop-engine/src/types.rs:10 | Pipeline context |
|
||||
| Engine | struct | workshop-engine/src/types.rs:99 | Build executor (placeholder) |
|
||||
| PrebakeConfig | struct | workshop-baker/src/prebake/config.rs | Prebake YAML schema |
|
||||
| FinalizeConfig | struct | workshop-baker/src/finalize/config.rs:21 | Finalize YAML schema |
|
||||
| GetterUrl | struct | workshop-getterurl/src/lib.rs:84 | Parsed resource URL |
|
||||
| Getter | trait | workshop-getterurl/src/getter.rs:18 | Resource fetch trait |
|
||||
|
||||
## CONVENTIONS
|
||||
|
||||
### Rust
|
||||
- **Edition**: 2024 (non-standard, requires Rust 1.85+)
|
||||
- **Edition**: 2024 (requires Rust 1.85+), deprecates `mod.rs`
|
||||
- **Naming**: `snake_case` files/modules, `PascalCase` types, `SCREAMING_SNAKE_CASE` consts
|
||||
- **Error handling**: `thiserror` + `anyhow` combo
|
||||
- **Error handling**: `thiserror` + `anyhow` combo; `HasExitCode` trait for exit codes
|
||||
- **Async**: `#[tokio::main]` + `tokio` with "full" features
|
||||
- **Imports**: `std` → `external_crate` → `crate::module`
|
||||
- **Dual-mode**: CLI commands (prebake/bake/finalize) + daemon mode (unimplemented)
|
||||
|
||||
### Testing
|
||||
- **Rust**: `#[test]` inline, `#[tokio::test]` async, `tests/*.rs` integration
|
||||
- **Python**: `pytest` with `tests/integration/test_*.py` (Docker-based)
|
||||
- **Benchmarks**: Criterion with `harness = false`
|
||||
- **Rust**: `#[test]` inline, `#[tokio::test]` async, `tests/` integration
|
||||
- **Benchmarks**: Criterion (configured in Cargo.toml)
|
||||
|
||||
### Build Pipeline
|
||||
- **prebake.yml**: Environment setup (docker/firecracker/baremetal)
|
||||
- **bake.sh**: Build execution with `# @pipeline` decorators
|
||||
- **finalize.yml**: Packaging, deployment, notifications
|
||||
- **Annotations**: `@timeout()`, `@retry()`, `@parallel`, `@fallible`
|
||||
### CI
|
||||
- **GitHub Actions**: rust-toolchain (stable), clippy + rustfmt, `cargo test --all-features`
|
||||
- **Matrix**: workshop-baker, workshop-engine
|
||||
|
||||
## ANTI-PATTERNS (THIS PROJECT)
|
||||
|
||||
1. **Python in Rust project** — workshop-baker has Python tests (pytest.ini, test_prebake.py)
|
||||
2. **temp/ directory** — Non-standard in Rust projects
|
||||
3. **Certificates in source** — artifact workshop-cert/certs/ not excluded
|
||||
4. **Deprecated projects** — workshop-executor.old, workshop-monitor.old still present
|
||||
5. **Hardcoded mirrors** — Tsinghua University mirrors in configs
|
||||
6. **Rust 2024 edition** — May not work with stable toolchains
|
||||
1. **Hardcoded user** — "vulcan" username hardcoded throughout prebake
|
||||
2. **Unsafe libc** — workshop-baker/src/prebake/security.rs has 5+ unsafe blocks
|
||||
3. **Rust 2024 edition** — May not work with stable toolchains
|
||||
4. **Empty daemon** — `daemon/` dir is empty, Daemon command is `unimplemented!()`
|
||||
|
||||
## UNIQUE STYLES
|
||||
|
||||
- **Pipeline DSL**: Shell scripts with `# @decorator` annotations
|
||||
- **Dual-mode baker**: CLI commands + daemon mode
|
||||
- **LLM cookbook system**: YAML cookbooks for language-specific builds
|
||||
- **Notification system**: Plugin-based with batching, retry, priority groups
|
||||
- **LLM cookbook system**: YAML cookbooks for language-specific builds (archived)
|
||||
- **Chinese docs** — Documentation primarily in zh-CN
|
||||
- **User "vulcan"** — Custom build user (not root/runner)
|
||||
|
||||
## COMMANDS
|
||||
```bash
|
||||
@@ -85,23 +75,18 @@ cargo build --release -p workshop-baker
|
||||
|
||||
# Test
|
||||
cargo test -p workshop-baker
|
||||
pytest tests/integration/test_prebake.py -v
|
||||
cargo test -p workshop-engine
|
||||
|
||||
# Lint
|
||||
cargo clippy -- -D warnings
|
||||
cargo fmt --all
|
||||
|
||||
# Run baker
|
||||
cargo run --bin workshop-baker -- prebake config.yml
|
||||
cargo run --bin workshop-baker -- bake script.sh
|
||||
|
||||
# Run agent
|
||||
cargo run --bin workshop-agent
|
||||
# Run
|
||||
cargo run --bin workshop-baker -- --pipeline demo --build-id 1 --prebake prebake.yml --bake bake.sh --finalize finalize.yml
|
||||
```
|
||||
|
||||
## NOTES
|
||||
|
||||
- **Docker required** for integration tests
|
||||
- **workshop-builder-native** is a stub (3-line Hello World)
|
||||
- No root workspace Cargo.toml — each crate is independent
|
||||
- `AGENTS.md.old` at root — legacy documentation
|
||||
- **No root workspace** — each crate independent, build separately
|
||||
- **7 archived crates** — workshop-agent, workshop-cert, workshop-deviceid, workshop-helper-mac, workshop-llm-detector, workshop-pipeline, workshop-vault
|
||||
- **Python tests removed** — pytest.ini exists but test files no longer in workspace
|
||||
|
||||
@@ -8,4 +8,31 @@
|
||||
- [PIPELINE_BAREMETAL_PKGMAN](zh-CN/vol2_admin/pipeline_baremetal_pkgman.md)
|
||||
- [PIPELINE_BAREMETAL_ELEVATE](zh-CN/vol2_admin/pipeline_baremetal_elevate.md)
|
||||
- [PIPELINE_CONTAINER_PRIVILEGED](zh-CN/vol2_admin/pipeline_container_privileged.md)
|
||||
- [环境变量](zh-CN/vol2_admin/environment_variables.md)
|
||||
- [Socket 配置](zh-CN/vol2_admin/socket_config.md)
|
||||
- [工作空间](zh-CN/vol2_admin/workspace.md)
|
||||
- [部署](zh-CN/vol2_admin/deployment.md)
|
||||
- [故障排除](zh-CN/vol2_admin/troubleshooting.md)
|
||||
- [Decorator 速查表](zh-CN/vol2_admin/decorator_cheatsheet.md)
|
||||
- [Prebake 配置参考](zh-CN/vol2_admin/prebake_config_reference.md)
|
||||
- [Finalize 插件参考](zh-CN/vol2_admin/finalize_plugin_reference.md)
|
||||
- [退出码速查](zh-CN/vol2_admin/exit_code_guide.md)
|
||||
- [构建状态文件格式](zh-CN/vol2_admin/status_file_format.md)
|
||||
- [卷3 开发者手册](zh-CN/vol3_dev/index.md)
|
||||
- [架构](zh-CN/vol3_dev/architecture.md)
|
||||
- [开发环境](zh-CN/vol3_dev/development_environment.md)
|
||||
- [代码规范](zh-CN/vol3_dev/code_style.md)
|
||||
- [提交规范](zh-CN/vol3_dev/commit_convention.md)
|
||||
- [PR 流程](zh-CN/vol3_dev/pr_process.md)
|
||||
- [错误处理模式](zh-CN/vol3_dev/error_handling_patterns.md)
|
||||
- [类型系统指南](zh-CN/vol3_dev/type_system_guide.md)
|
||||
- [测试策略](zh-CN/vol3_dev/testing_strategy.md)
|
||||
- [插件系统指南](zh-CN/vol3_dev/plugin_system_guide.md)
|
||||
- [Engine 模块指南](zh-CN/vol3_dev/engine_module_guide.md)
|
||||
- [设计决策](zh-CN/vol3_dev/design_decisions.md)
|
||||
- [添加包管理器](zh-CN/vol3_dev/add_package_manager.md)
|
||||
- [自定义插件](zh-CN/vol3_dev/custom_plugin.md)
|
||||
- [自定义 Decorator](zh-CN/vol3_dev/custom_decorator.md)
|
||||
- [模板变量](zh-CN/vol3_dev/template_variables.md)
|
||||
- [邮件模板](zh-CN/vol3_dev/mail_template.md)
|
||||
- [Resource 模型](zh-CN/vol3_dev/resource_model.md)
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
# Decorator 速查表
|
||||
|
||||
bake.sh 中所有可用的 `# @decorator` 注解:
|
||||
|
||||
| Decorator | 语法 | 作用 | 示例 |
|
||||
|-----------|------|------|------|
|
||||
| `@pipeline` | `# @pipeline` | 标记为主构建步骤 | 无参数 |
|
||||
| `@after` | `# @after(func_name)` | 指定依赖的前置步骤 | `# @after(build)` |
|
||||
| `@timeout` | `# @timeout(N)` | 超时时间(秒) | `# @timeout(300)` |
|
||||
| `@retry` | `# @retry(count, delay)` | 失败重试次数与间隔(秒) | `# @retry(3, 10)` |
|
||||
| `@if` | `# @if(condition)` | 条件执行(shell 条件表达式) | `# @if([ -f Makefile ])` |
|
||||
| `@fallible` | `# @fallible` | 允许失败,不中断流水线 | 无参数 |
|
||||
| `@parallel` | `# @parallel` | 与其他 `@parallel` 步骤并发执行 | 无参数 |
|
||||
| `@loop` | `# @loop(N)` | 重复执行 N 次 | `# @loop(5)` |
|
||||
| `@export` | `# @export` | 导出环境变量供后续步骤使用 | 无参数 |
|
||||
| `@pipe` | `# @pipe(f1, f2, ...)` | 管道串联多个函数 | `# @pipe(lint, build)` |
|
||||
| `@daemon` | `# @daemon` | 作为后台守护进程运行 | 无参数 |
|
||||
| `@health` | `# @health(endpoint)` | 指定健康检查端点 | `# @health(/health)` |
|
||||
|
||||
## 语法规则
|
||||
|
||||
- `#` 和 `@` 之间必须有空格:`# @pipeline` 合法,`#@pipeline` 不合法
|
||||
- `@name` 和 `(args)` 之间不能有空格:`@timeout(60)` 合法,`@timeout (60)` 不合法
|
||||
- 每个 decorator 独占一行,位于目标函数定义的上方
|
||||
|
||||
## 常见组合
|
||||
|
||||
```bash
|
||||
# @pipeline
|
||||
# @after(setup)
|
||||
# @timeout(600)
|
||||
# @retry(2, 30)
|
||||
build() {
|
||||
cargo build --release
|
||||
}
|
||||
```
|
||||
@@ -0,0 +1,129 @@
|
||||
# 部署
|
||||
|
||||
## 安装
|
||||
|
||||
### 从源码构建
|
||||
|
||||
```bash
|
||||
cargo build --release -p workshop-baker
|
||||
```
|
||||
|
||||
编译输出位于 `target/release/workshop-baker`。
|
||||
|
||||
### 下载预编译二进制
|
||||
|
||||
从项目 Release 页面下载对应平台的预编译二进制文件,放置到 `$PATH` 可达路径即可。
|
||||
|
||||
## 运行模式
|
||||
|
||||
Baker 支持两种运行模式:
|
||||
|
||||
### CLI 模式
|
||||
|
||||
单次执行模式,每次运行完成一个构建阶段后退出:
|
||||
|
||||
```bash
|
||||
# 预烘焙阶段
|
||||
workshop-baker -u <username> -p <pipeline> -b <build-id> bare prebake config.yml
|
||||
|
||||
# 烘焙阶段
|
||||
workshop-baker -u <username> -p <pipeline> -b <build-id> bare bake script.sh \
|
||||
--bake-base ./bake_base.sh --prebake ./prebake.yml
|
||||
|
||||
# 终结阶段
|
||||
workshop-baker -u <username> -p <pipeline> -b <build-id> bare finalize config.yml
|
||||
```
|
||||
|
||||
### Daemon 模式
|
||||
|
||||
常驻后台模式通过 Socket 监听请求,支持多构建任务管理。当前处于重构中,暂未实现。Daemon 模式下不指定 `bare` 子命令即可进入:
|
||||
|
||||
```bash
|
||||
workshop-baker -u <username> -p <pipeline> -b <build-id>
|
||||
```
|
||||
|
||||
## 预烘焙环境
|
||||
|
||||
prebake 阶段负责准备构建运行环境,支持三种方式:
|
||||
|
||||
### Docker
|
||||
|
||||
在容器内执行构建,通过配置文件指定容器参数:
|
||||
|
||||
```yaml
|
||||
prebake:
|
||||
type: docker
|
||||
image: "rust:1.85"
|
||||
dockerfile: "Dockerfile.build"
|
||||
build_args:
|
||||
- "CARGO_PROFILE=release"
|
||||
```
|
||||
|
||||
- `image` — 指定预构建的 Docker 镜像
|
||||
- `dockerfile` — 指定自定义 Dockerfile 路径(与 `image` 二选一)
|
||||
- `build_args` — Docker 构建参数列表
|
||||
|
||||
### Firecracker
|
||||
|
||||
在 microVM 内执行构建,提供更强的隔离性:
|
||||
|
||||
```yaml
|
||||
prebake:
|
||||
type: firecracker
|
||||
kernel: "/path/to/vmlinux"
|
||||
rootfs: "/path/to/rootfs.ext4"
|
||||
vcpu_count: 2
|
||||
mem_size_mb: 512
|
||||
```
|
||||
|
||||
### Baremetal
|
||||
|
||||
直接在宿主机运行构建,不使用容器或虚拟机隔离:
|
||||
|
||||
```yaml
|
||||
prebake:
|
||||
type: baremetal
|
||||
pkgman: "apt"
|
||||
elevate: true
|
||||
```
|
||||
|
||||
## Systemd 服务
|
||||
|
||||
以下为 Baker Daemon 的 systemd unit 文件示例:
|
||||
|
||||
```ini
|
||||
[Unit]
|
||||
Description=HoneyBiscuitWorkshop Baker Daemon
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/usr/local/bin/workshop-baker -u vulcan -p default -b 0 --socket unix:///run/hbw.sock
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
Environment=RUST_LOG=info
|
||||
Environment=HBW_WORKSPACE=/workspace
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
```
|
||||
|
||||
部署步骤:
|
||||
|
||||
```bash
|
||||
# 安装 unit 文件
|
||||
sudo cp workshop-baker.service /etc/systemd/system/
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable --now workshop-baker
|
||||
```
|
||||
|
||||
## 权限模型
|
||||
|
||||
Baker 采用"先提权后降权"的权限模型:
|
||||
|
||||
| 阶段 | 执行用户 | 原因 |
|
||||
|------|----------|------|
|
||||
| prebake | root | 需要创建容器/虚拟机、挂载文件系统等特权操作 |
|
||||
| bake | vulcan | 降权执行,限制构建过程权限 |
|
||||
|
||||
prebake 阶段以 root 身份运行,完成环境准备工作后,Baker 会将执行权限降级为 `vulcan` 用户再进入 bake 阶段,确保构建过程不会以 root 权限运行。
|
||||
@@ -0,0 +1,44 @@
|
||||
# 环境变量
|
||||
|
||||
HoneyBiscuitWorkshop 通过以下环境变量控制构建行为:
|
||||
|
||||
| 变量 | 说明 | 默认值 |
|
||||
|------|------|--------|
|
||||
| `HBW_USERNAME` | 提交者用户名(来自 base / server) | — |
|
||||
| `HBW_PIPELINE` | 流水线标识符 | — |
|
||||
| `HBW_BUILD_ID` | 构建唯一 ID | — |
|
||||
| `HBW_DRY_RUN` | 干运行模式,不执行实际构建 | — |
|
||||
| `HBW_WORKSPACE` | 工作空间路径 | `/workspace` |
|
||||
| `RUST_LOG` | 日志级别 (error/warn/info/debug/trace) | `warn` |
|
||||
|
||||
### HBW_USERNAME
|
||||
|
||||
提交者用户名,由上游 server / base 系统传入,标识触发本次构建的用户。与构建系统内的 Unix/Windows 用户名(如 `vulcan`)无关,仅用于日志、通知与审计追踪。
|
||||
|
||||
### HBW_PIPELINE
|
||||
|
||||
标识当前运行的流水线。在 prebake 阶段自动从配置文件获取,贯穿整个构建生命周期。
|
||||
|
||||
### HBW_BUILD_ID
|
||||
|
||||
每次构建的唯一标识符,用于日志追踪和状态标识。
|
||||
|
||||
### HBW_DRY_RUN
|
||||
|
||||
设置为任意非空值时启用干运行模式。该模式下 Baker 会解析流水线配置并生成执行计划,但不会执行实际构建操作。适用于配置验证和调试。
|
||||
|
||||
### HBW_WORKSPACE
|
||||
|
||||
构建工作空间的绝对路径。所有构建输出和中间文件都将输出到此目录下。
|
||||
|
||||
### RUST_LOG
|
||||
|
||||
控制日志输出级别,遵循 `tracing` crate 的规范:
|
||||
|
||||
- `error` — 仅输出错误
|
||||
- `warn` — 输出警告和错误
|
||||
- `info` — 输出一般信息
|
||||
- `debug` — 输出调试信息
|
||||
- `trace` — 输出全部跟踪信息
|
||||
|
||||
支持模块级别过滤,例如 `RUST_LOG=workshop_baker=trace` 仅跟踪 Baker 模块。
|
||||
@@ -0,0 +1,20 @@
|
||||
# 退出码速查
|
||||
|
||||
| 退出码 | 常量名 | 含义 | 典型触发场景 |
|
||||
|--------|--------|------|-------------|
|
||||
| 0 | `EXITCODE_OK` | 成功 | 流水线正常完成 |
|
||||
| 1 | `EXITCODE_GENERAL_ERROR` | 通用错误 | 未分类的运行时错误 |
|
||||
| 2 | `EXITCODE_INVALID_ARGS` | 参数错误 | 命令行参数或配置无效 |
|
||||
| 3 | `EXITCODE_IO_ERROR` | IO 错误 | 文件读写、目录创建失败 |
|
||||
| 4 | `EXITCODE_PARSE_ERROR` | 解析错误 | YAML 配置或 Decorator 语法错误 |
|
||||
| 5 | `EXITCODE_EXECUTION_ERROR` | 执行错误 | 构建脚本执行失败 |
|
||||
| 124 | `EXITCODE_TIMEOUT` | 超时 | 构建超过 `@timeout()` 限制 |
|
||||
| 201 | `EXITCODE_PRIV_DROP_FAILED` | 权限降级失败 | 无法从 root 切换到 vulcan 用户 |
|
||||
| 233 | `PIPELINE_SKIP_ERRORCODE` | 流水线跳过 | `@if` 条件不满足,步骤跳过 |
|
||||
|
||||
## 排查命令
|
||||
|
||||
```bash
|
||||
# 结合日志定位具体错误
|
||||
RUST_LOG=debug workshop-baker -u testuser -p my-pipeline -b 1 bare bake script.sh 2>&1 | grep -i "error\|fail"
|
||||
```
|
||||
@@ -0,0 +1,91 @@
|
||||
# Finalize 插件参考
|
||||
|
||||
Finalize 阶段通过插件系统完成打包、部署与通知。
|
||||
|
||||
## 插件类型
|
||||
|
||||
| 类型 | 说明 | 配置位置 |
|
||||
|------|------|---------|
|
||||
| `internal` | 内置插件(随 baker 编译) | `finalize.yml` 的 `notification` 下直接配置 |
|
||||
| `shell` | 可执行脚本插件 | 提供 `manifest.yml` 描述文件 |
|
||||
| `dylib` | 动态链接库插件 | 通过 `dlopen` 加载 |
|
||||
| `rhai` | Rhai 脚本引擎插件 | `.rhai` 脚本文件(暂未实现) |
|
||||
|
||||
## 内置插件清单
|
||||
|
||||
### mail — SMTP 邮件通知
|
||||
|
||||
发送构建结果邮件,支持 HTML 模板与多收件人。
|
||||
|
||||
| 字段 | 必需 | 类型 | 说明 |
|
||||
|------|------|------|------|
|
||||
| `to` | 是 | String / List | 收件人地址,支持模板变量 |
|
||||
| `from` | 是 | String / Object | 发件人,单字符串为 email,Object 可含 `name` 与 `email` |
|
||||
| `cc` | 否 | List | 抄送 |
|
||||
| `bcc` | 否 | List | 密送 |
|
||||
| `reply_to` | 否 | String | 回复地址 |
|
||||
| `subject` | 否 | String | 邮件主题,支持模板语法 |
|
||||
| `body` | 否 | String | 邮件正文,支持模板语法与 HTML |
|
||||
| `schema` | 否 | String | `"default"` 或 `"custom"`,控制是否使用内置模板 |
|
||||
| `smtp` | 是 | Object | SMTP 服务器配置 |
|
||||
| `timeout` | 否 | String | 发送超时,如 `"90s"` |
|
||||
| `retry` | 否 | Integer | 失败重试次数 |
|
||||
| `fallible` | 否 | Boolean | 允许失败不中断流水线 |
|
||||
|
||||
#### smtp 子字段
|
||||
|
||||
| 字段 | 必需 | 说明 |
|
||||
|------|------|------|
|
||||
| `host` | 是 | SMTP 服务器地址 |
|
||||
| `port` | 是 | 端口号 |
|
||||
| `auth.type` | 否 | `"password"` / `"oauth2"` / `"none"` |
|
||||
| `auth.username` | 条件 | `type=password` 时必需 |
|
||||
| `auth.password` | 条件 | `type=password` 时必需 |
|
||||
| `encryption` | 否 | `"tls"` / `"starttls"` / `"none"` |
|
||||
| `connect_timeout` | 否 | 连接超时 |
|
||||
|
||||
#### 模板变量
|
||||
|
||||
| 变量 | 示例值 | 说明 |
|
||||
|------|--------|------|
|
||||
| `{{ build.status }}` | `success` | 构建状态 |
|
||||
| `{{ build.id }}` | `42` | 构建编号 |
|
||||
| `{{ build.duration }}` | `125s` | 构建耗时 |
|
||||
| `{{ build.url }}` | `https://ci.example.com/42` | 构建详情链接 |
|
||||
| `{{ pipeline.name }}` | `my-project` | 流水线名称 |
|
||||
| `{{ commit.hash }}` | `abc1234` | 提交哈希 |
|
||||
| `{{ commit.author }}` | `Alice` | 提交者 |
|
||||
| `{{ commit.message }}` | `fix: typo` | 提交信息 |
|
||||
| `{{ secret.xxx }}` | — | Vault 密钥引用 |
|
||||
|
||||
### webhook — HTTP 回调
|
||||
|
||||
向外部服务发送 HTTP POST 请求。
|
||||
|
||||
| 字段 | 必需 | 说明 |
|
||||
|------|------|------|
|
||||
| `url` | 是 | 回调地址 |
|
||||
| `method` | 否 | HTTP 方法,默认 `POST` |
|
||||
| `headers` | 否 | 请求头键值对 |
|
||||
| `body` | 否 | 请求体,支持模板变量 |
|
||||
| `timeout` | 否 | 请求超时 |
|
||||
| `retry` | 否 | 失败重试次数 |
|
||||
| `fallible` | 否 | 允许失败 |
|
||||
|
||||
## 最小配置示例
|
||||
|
||||
```yaml
|
||||
notification:
|
||||
1:
|
||||
mail:
|
||||
to: "admin@example.com"
|
||||
from: "ci@example.com"
|
||||
smtp:
|
||||
host: "smtp.gmail.com"
|
||||
port: 587
|
||||
auth:
|
||||
type: "password"
|
||||
username: "{{ secret.smtp_user }}"
|
||||
password: "{{ secret.smtp_pass }}"
|
||||
encryption: "starttls"
|
||||
```
|
||||
@@ -0,0 +1,119 @@
|
||||
# Prebake 配置参考
|
||||
|
||||
`prebake.yml` 定义构建环境的准备阶段。
|
||||
|
||||
## 顶层字段
|
||||
|
||||
| 字段 | 必需 | 类型 | 默认值 | 说明 |
|
||||
|------|------|------|--------|------|
|
||||
| `version` | 是 | String | — | 语义化版本,须满足 `^1.0` |
|
||||
| `environment` | 是 | Object | — | 构建环境类型与资源配置 |
|
||||
| `bootstrap` | 否 | Object | 见下文 | 用户、工作目录初始化 |
|
||||
| `dependencies` | 否 | Object | — | 系统包与用户包依赖 |
|
||||
| `envvars` | 否 | Object | — | 环境变量注入 |
|
||||
| `cache` | 否 | Object | — | 缓存目录与策略 |
|
||||
| `security` | 否 | Object | — | 权限降级配置 |
|
||||
| `hooks` | 否 | Object | — | 前置/后置钩子 |
|
||||
| `metadata` | 否 | Object | — | 流水线元信息 |
|
||||
|
||||
## environment
|
||||
|
||||
| 字段 | 必需 | 类型 | 默认值 | 说明 |
|
||||
|------|------|------|--------|------|
|
||||
| `builder` | 是 | String | — | `baremetal` / `docker` / `firecracker` / `custom` |
|
||||
| `resources` | 否 | Object | 见下文 | CPU、内存、磁盘、架构约束 |
|
||||
| `network` | 否 | Object | 见下文 | 网络开关与代理 |
|
||||
|
||||
### resources 默认值
|
||||
|
||||
| 字段 | 默认值 | 说明 |
|
||||
|------|--------|------|
|
||||
| `cpu` | `1` | vCPU 数量 |
|
||||
| `memory` | `1 GiB` | 内存限制 |
|
||||
| `disk` | `1 GiB` | 磁盘限制 |
|
||||
| `architecture` | `[]` | 允许的目标架构,支持字符串或数组 |
|
||||
| `tags` | `[]` | 节点标签筛选 |
|
||||
|
||||
### network
|
||||
|
||||
| 字段 | 默认值 | 说明 |
|
||||
|------|--------|------|
|
||||
| `enabled` | `true` | 是否启用网络 |
|
||||
| `outbound` | `true` | 是否允许出站连接 |
|
||||
| `dns_servers` | `[]` | 自定义 DNS 服务器列表 |
|
||||
| `proxies` | `[]` | HTTP/HTTPS 代理配置 |
|
||||
|
||||
## bootstrap
|
||||
|
||||
| 字段 | 默认值 | 说明 |
|
||||
|------|--------|------|
|
||||
| `user` | `"vulcan"` | 构建用户 |
|
||||
| `workspace.path` | `"/home/vulcan/workspace"` | 工作目录 |
|
||||
| `workspace.fallback` | `true` | 目录不存在时自动创建 |
|
||||
| `sudoers` | `null` | sudoers 文件内容 |
|
||||
| `doas` | `null` | doas.conf 文件内容 |
|
||||
| `custom` | `null` | 自定义初始化脚本 |
|
||||
|
||||
## dependencies
|
||||
|
||||
### system
|
||||
|
||||
以包管理器名为键,每个值包含:
|
||||
|
||||
| 字段 | 必需 | 说明 |
|
||||
|------|------|------|
|
||||
| `packages` | 是 | 包名列表 |
|
||||
| `repositories` | 否 | 额外软件源配置 |
|
||||
| `mirror` | 否 | 镜像地址 |
|
||||
|
||||
```yaml
|
||||
dependencies:
|
||||
system:
|
||||
pacman:
|
||||
packages: ["curl", "wget"]
|
||||
```
|
||||
|
||||
### user
|
||||
|
||||
以包管理器名为键,值为自由格式(由对应 UPM 解析)。
|
||||
|
||||
## envvars
|
||||
|
||||
| 字段 | 说明 |
|
||||
|------|------|
|
||||
| `prebake` | 在 prebake 阶段注入的环境变量 |
|
||||
| `bake` | 在 bake 阶段注入的环境变量 |
|
||||
|
||||
## cache
|
||||
|
||||
| 字段 | 必需 | 说明 |
|
||||
|------|------|------|
|
||||
| `directory` | 是 | 缓存目录列表,每项含 `path` 与可选 `strategy` |
|
||||
| `strategy` | 否 | 缓存策略配置 |
|
||||
|
||||
## 完整示例
|
||||
|
||||
```yaml
|
||||
version: "1.0"
|
||||
environment:
|
||||
builder: "baremetal"
|
||||
resources:
|
||||
cpu: 2
|
||||
memory: "4 GiB"
|
||||
architecture: ["x86_64", "aarch64"]
|
||||
bootstrap:
|
||||
user: "vulcan"
|
||||
workspace:
|
||||
path: "/home/vulcan/workspace"
|
||||
fallback: true
|
||||
dependencies:
|
||||
system:
|
||||
pacman:
|
||||
packages: ["rust", "git"]
|
||||
envvars:
|
||||
prebake:
|
||||
RUST_BACKTRACE: "1"
|
||||
cache:
|
||||
directory:
|
||||
- path: "/var/cache/pacman"
|
||||
```
|
||||
@@ -0,0 +1,18 @@
|
||||
# Socket 配置
|
||||
|
||||
Baker Daemon 通过 Socket 与 CLI 通信,支持以下连接方式:
|
||||
|
||||
- **Unix Domain Socket**: `unix://path` — 使用本地 Unix 域套接字,适用于同机通信
|
||||
```bash
|
||||
workshop-baker -u vulcan -p default -b 0 --socket unix:///run/hbw.sock
|
||||
```
|
||||
- **TCP Socket**: `tcp://host:port` — 使用 TCP 连接,适用于远程管理
|
||||
```bash
|
||||
workshop-baker -u vulcan -p default -b 0 --socket tcp://127.0.0.1:9090
|
||||
```
|
||||
- **Dryrun 模式**: `dryrun` — 测试模式,不创建实际连接
|
||||
```bash
|
||||
workshop-baker -u vulcan -p default -b 0 --socket dryrun
|
||||
```
|
||||
|
||||
默认使用 `unix:///run/hbw.sock`。
|
||||
@@ -0,0 +1,33 @@
|
||||
# 构建状态文件格式
|
||||
|
||||
Baker 将构建状态写入 `/tmp/.hbwstatus`,格式为 **JSON Lines**(每行一条独立 JSON)。
|
||||
|
||||
## 字段说明
|
||||
|
||||
| 字段 | 类型 | 说明 |
|
||||
|------|------|------|
|
||||
| `name` | String | 阶段名称 |
|
||||
| `phase` | String | 执行阶段:`prebake` / `bake` / `finalize` |
|
||||
| `substage` | String | 子阶段标识 |
|
||||
| `result` | String | 结果:`success` / `failure` / `canceled` / `skipped` |
|
||||
| `duration_ms` | u64 | 执行耗时(毫秒) |
|
||||
| `started_at` | ISO 8601 | 开始时间 |
|
||||
| `finished_at` | ISO 8601 | 结束时间 |
|
||||
| `error_message` | String/Null | 失败时的错误信息 |
|
||||
|
||||
## 示例
|
||||
|
||||
```json
|
||||
{"name":"bootstrap","phase":"Prebake","substage":"bootstrap","result":"success","duration_ms":1500,"started_at":"2026-04-25T10:00:00Z","finished_at":"2026-04-25T10:00:01.5Z","error_message":null}
|
||||
{"name":"build","phase":"Bake","substage":"build","result":"failure","duration_ms":30000,"started_at":"2026-04-25T10:01:00Z","finished_at":"2026-04-25T10:01:30Z","error_message":"cargo build failed with exit code 101"}
|
||||
```
|
||||
|
||||
## 读取状态
|
||||
|
||||
```bash
|
||||
# 查看最新阶段
|
||||
tail -1 /tmp/.hbwstatus | jq .
|
||||
|
||||
# 统计失败阶段数
|
||||
cat /tmp/.hbwstatus | jq -s 'map(select(.result == "failure")) | length'
|
||||
```
|
||||
@@ -0,0 +1,102 @@
|
||||
# 故障排除
|
||||
|
||||
## 日志调试
|
||||
|
||||
启用详细日志输出以定位问题:
|
||||
|
||||
```bash
|
||||
# 查看 Baker 详细日志
|
||||
RUST_LOG=debug workshop-baker -u testuser -p my-pipeline -b 1 bare bake script.sh
|
||||
|
||||
# 查看特定模块日志
|
||||
RUST_LOG=workshop_baker=trace workshop-baker -u testuser -p my-pipeline -b 1 bare bake script.sh
|
||||
|
||||
# 组合过滤
|
||||
RUST_LOG=workshop_baker=debug,workshop_pipeline=trace workshop-baker -u testuser -p my-pipeline -b 1 bare bake script.sh
|
||||
```
|
||||
|
||||
常用日志级别组合:
|
||||
|
||||
- `RUST_LOG=info` — 生产环境推荐
|
||||
- `RUST_LOG=debug` — 一般调试
|
||||
- `RUST_LOG=workshop_baker=trace` — 深度调试 Baker 内部逻辑
|
||||
|
||||
## 退出码对照表
|
||||
|
||||
| 退出码 | 含义 | 说明 |
|
||||
|--------|------|------|
|
||||
| 0 | OK | 构建成功完成 |
|
||||
| 1 | 通用错误 | 未分类的运行时错误 |
|
||||
| 2 | 参数错误 | 命令行参数或配置无效 |
|
||||
| 3 | IO 错误 | 文件读写失败 |
|
||||
| 4 | 解析错误 | YAML 配置或 Decorator 语法解析失败 |
|
||||
| 5 | 执行错误 | 构建脚本执行失败 |
|
||||
| 124 | 超时 | 构建超过 `@timeout()` 指定的时间限制 |
|
||||
| 201 | 权限降级失败 | 无法从 root 切换到 vulcan 用户 |
|
||||
| 233 | 流水线跳过 | 流水线条件不满足,跳过执行 |
|
||||
|
||||
## 常见问题与解决
|
||||
|
||||
### Socket 连接失败
|
||||
|
||||
**现象**: CLI 无法连接 Daemon,报 Connection refused 或 Permission denied。
|
||||
|
||||
**解决**: 检查 Unix socket 路径是否存在以及文件权限:
|
||||
|
||||
```bash
|
||||
# 检查 socket 文件
|
||||
ls -la /run/hbw.sock
|
||||
|
||||
# 确认 Daemon 正在运行
|
||||
pgrep -a workshop-baker
|
||||
|
||||
# 检查 socket 权限
|
||||
# 确保当前用户有读写权限,或使用 sudo
|
||||
```
|
||||
|
||||
### Decorator 解析错误
|
||||
|
||||
**现象**: 构建脚本的 Decorator 注解无法识别。
|
||||
|
||||
**解决**: Decorator 语法必须严格遵循 `# @name(args)` 格式。常见错误:
|
||||
|
||||
```bash
|
||||
# 错误 — @ 符号前缺少空格
|
||||
#@timeout(60)
|
||||
|
||||
# 错误 — 名称后有空格
|
||||
# @ timeout(60)
|
||||
|
||||
# 正确
|
||||
# @timeout(60)
|
||||
```
|
||||
|
||||
注意:`#` 和 `@` 之间必须有空格,`@name` 和 `(args)` 之间不能有空格。
|
||||
|
||||
### Hook 执行权限拒绝
|
||||
|
||||
**现象**: prebake 或 finalize 阶段的 Hook 脚本报 Permission denied。
|
||||
|
||||
**解决**: 检查 `vulcan` 用户的 sudoers 配置,确保其有执行必要操作的权限:
|
||||
|
||||
```bash
|
||||
# 检查 sudoers 配置
|
||||
sudo visudo -c
|
||||
|
||||
# 确认 vulcan 用户权限
|
||||
sudo -lU vulcan
|
||||
```
|
||||
|
||||
### 包管理器检测失败
|
||||
|
||||
**现象**: baremetal 模式下 Baker 无法检测系统包管理器。
|
||||
|
||||
**解决**: 检查 `/etc/os-release` 文件是否存在且内容完整:
|
||||
|
||||
```bash
|
||||
# 检查文件
|
||||
cat /etc/os-release
|
||||
|
||||
# 常见缺失情况:Docker 最小镜像或 musl 环境
|
||||
# 需手动安装 base-files 或创建该文件
|
||||
```
|
||||
@@ -0,0 +1,13 @@
|
||||
# 工作空间
|
||||
|
||||
工作空间是 Baker 执行构建的根目录,默认路径为 `/workspace`。目录结构如下:
|
||||
|
||||
```
|
||||
/workspace/
|
||||
├── src/ # 源代码检出目录
|
||||
├── build/ # 编译中间输出
|
||||
├── output/ # 构建最终输出
|
||||
└── cache/ # 构建缓存
|
||||
```
|
||||
|
||||
可通过 `HBW_WORKSPACE` 环境变量或配置文件覆盖默认路径。
|
||||
@@ -0,0 +1,41 @@
|
||||
# 添加包管理器
|
||||
|
||||
要支持新的系统包管理器,需要实现 `PackageManager` trait。
|
||||
|
||||
```rust
|
||||
trait PackageManager {
|
||||
fn name(&self) -> &str;
|
||||
fn install(&self, packages: &[&str]) -> Result<()>;
|
||||
fn update(&self) -> Result<()>;
|
||||
}
|
||||
```
|
||||
|
||||
参考 `workshop-baker/src/engine/pm/pacman.rs` 中的实现。步骤如下:
|
||||
|
||||
1. 在 `workshop-baker/src/engine/pm/` 下创建新文件(如 `yum.rs`)。
|
||||
2. 实现 `PackageManager` trait 的所有方法。
|
||||
3. 在 `pm/mod.rs` 中注册新变体到 `PackageManagerEnum`。
|
||||
4. 在配置解析中增加对应的匹配分支。
|
||||
|
||||
## 示例:pacman 实现
|
||||
|
||||
```rust
|
||||
pub struct PacmanManager;
|
||||
|
||||
impl PackageManager for PacmanManager {
|
||||
fn name(&self) -> &str { "pacman" }
|
||||
|
||||
fn install(&self, packages: &[&str]) -> Result<()> {
|
||||
let args = vec!["-S", "--noconfirm"].iter()
|
||||
.chain(packages.iter())
|
||||
.collect::<Vec<_>>();
|
||||
Command::new("pacman").args(&args).status()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn update(&self) -> Result<()> {
|
||||
Command::new("pacman").args(["-Syu", "--noconfirm"]).status()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
```
|
||||
@@ -0,0 +1,87 @@
|
||||
# 架构
|
||||
|
||||
## 整体架构
|
||||
|
||||
当前活跃维护的 crate:
|
||||
|
||||
- **workshop-baker** 是核心编排器,负责流水线的调度与执行,提供 CLI 和 Daemon 两种运行模式。
|
||||
- **workshop-vault** 封装 HashiCorp Vault 客户端,提供密钥的读写与引用解析。
|
||||
- **workshop-cert** 负责生成自签名 TLS 证书。
|
||||
- **workshop-deviceid** 提供设备唯一标识,用于多节点环境下的节点识别。
|
||||
|
||||
其余模块(如 workshop-agent、workshop-llm-detector、workshop-pipeline 等)已归档至 `archived/` 目录,不再活跃维护。
|
||||
|
||||
## 数据流
|
||||
|
||||
流水线分为三个阶段,顺序执行:
|
||||
|
||||
1. **Prebake(环境准备)**:根据 prebake.yml 配置创建构建环境,支持 Docker、Firecracker 和 Bare Metal 三种模式。包括包安装、用户创建、安全降权等。
|
||||
2. **Bake(脚本执行)**:加载 bake.sh 脚本,解析 `# @decorator` 注解,进行拓扑排序后按依赖顺序执行构建步骤。
|
||||
3. **Finalize(后处理)**:执行打包、部署、通知等收尾工作,通过插件系统完成邮件发送、Webhook 回调等操作。
|
||||
|
||||
三个阶段通过 `ExecutionContext` 串联,上下文信息在阶段间传递。
|
||||
|
||||
## 核心模块详解
|
||||
|
||||
### Engine
|
||||
|
||||
Engine 是 Bake 阶段的核心执行引擎,主要包含:
|
||||
|
||||
- **Executor**:负责脚本的执行,管理子进程的生命周期和输出捕获。
|
||||
- **PackageManager**:包管理抽象 trait,支持 pacman、apt 等多种包管理器的统一调用。
|
||||
|
||||
### Bake
|
||||
|
||||
Bake 模块处理构建脚本解析和调度:
|
||||
|
||||
- **Parser**:解析 bake.sh 中的 `# @decorator` 注解,提取超时、重试、并行等指令。
|
||||
- **Schedule**:基于依赖关系对构建步骤进行拓扑排序,支持并行执行无依赖的步骤。
|
||||
- **Builder**:使用模板引擎渲染构建脚本,支持变量替换和条件逻辑。
|
||||
|
||||
### Prebake
|
||||
|
||||
Prebake 模块负责构建环境准备:
|
||||
|
||||
- **Config**:解析 prebake.yml,提取环境类型(Docker/Firecracker/Bare Metal)和配置项。
|
||||
- **Bootstrap**:生成环境启动脚本,包括系统初始化和工具安装。
|
||||
- **Security**:处理权限降级,创建构建用户(默认 vulcan)并设置目录权限。
|
||||
|
||||
### Finalize
|
||||
|
||||
Finalize 模块完成后处理工作:
|
||||
|
||||
- **Plugin**:插件系统,支持 Shell、Dylib、Rhai 和 Internal 四种插件类型。
|
||||
- **Hook**:在流水线关键节点执行钩子函数,如构建成功后的通知发送。
|
||||
|
||||
## 类型系统
|
||||
|
||||
核心类型贯穿三个阶段:
|
||||
|
||||
- **ExecutionContext**:在 Prebake → Bake → Finalize 间传递的上下文对象,包含环境信息、变量和构建状态。
|
||||
- **ResourceLimits**:定义 CPU、内存、磁盘等资源限制,用于 CgroupManager 配置。
|
||||
- **BuildStatus**:表示构建状态(Pending/Running/Success/Failed/Cancelled),贯穿整个生命周期。
|
||||
|
||||
## 插件架构
|
||||
|
||||
插件系统基于 `Plugin` trait 和 `AsyncPluginFn` 类型别名:
|
||||
|
||||
```rust
|
||||
trait Plugin {
|
||||
async fn execute(&self, ctx: &ExecutionContext) -> Result<()>;
|
||||
}
|
||||
```
|
||||
|
||||
### 内部插件
|
||||
|
||||
内部插件随 baker 编译,直接调用 Rust 代码:
|
||||
|
||||
- **Mail**:基于 SMTP 发送构建通知邮件,支持 minijinja 模板。
|
||||
- **Webhook**:向外部服务发送 HTTP 回调。
|
||||
|
||||
### 外部插件
|
||||
|
||||
外部插件运行在 baker 进程之外:
|
||||
|
||||
- **Shell**:可执行脚本,通过子进程调用,需提供 `manifest.yml` 描述文件。
|
||||
- **Dylib**:动态链接库,通过 `dlopen` 加载并调用导出函数。
|
||||
- **Rhai**:使用 Rhai 脚本引擎执行 `.rhai` 脚本文件。
|
||||
@@ -0,0 +1,30 @@
|
||||
# 代码规范
|
||||
|
||||
## 命名约定
|
||||
|
||||
| 类型 | 风格 | 示例 |
|
||||
|------|------|------|
|
||||
| 文件/模块 | snake_case | `engine.rs`, `pm/` |
|
||||
| 类型/结构体/枚举 | PascalCase | `PipelineConfig`, `BuildStatus` |
|
||||
| 常量 | SCREAMING_SNAKE_CASE | `MAX_RETRIES`, `DEFAULT_TIMEOUT` |
|
||||
| 函数/变量 | snake_case | `parse_decorator`, `build_id` |
|
||||
|
||||
## 导入顺序
|
||||
|
||||
按以下顺序组织 `use` 语句,每组之间空一行:
|
||||
|
||||
1. `std` 标准库
|
||||
2. 外部 crate
|
||||
3. `crate::` 内部模块
|
||||
|
||||
```rust
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
|
||||
use anyhow::Result;
|
||||
use serde::Deserialize;
|
||||
use tokio::time::sleep;
|
||||
|
||||
use crate::bake::Parser;
|
||||
use crate::engine::Executor;
|
||||
```
|
||||
@@ -0,0 +1,20 @@
|
||||
# 提交规范
|
||||
|
||||
提交信息使用类型前缀:
|
||||
|
||||
| 前缀 | 说明 |
|
||||
|------|------|
|
||||
| `feat` | 新功能 |
|
||||
| `fix` | Bug 修复 |
|
||||
| `docs` | 文档变更 |
|
||||
| `test` | 测试相关 |
|
||||
| `refactor` | 重构 |
|
||||
| `style` | 代码风格(不影响功能) |
|
||||
|
||||
示例:
|
||||
|
||||
```
|
||||
feat(baker): add timeout support for build steps
|
||||
fix(agent): fix TLS certificate reload on SIGHUP
|
||||
docs: update contributing guide
|
||||
```
|
||||
@@ -0,0 +1,32 @@
|
||||
# 自定义 Decorator
|
||||
|
||||
在 bake.sh 脚本中,`# @decorator` 注解用于控制构建行为。添加自定义 Decorator 的步骤:
|
||||
|
||||
1. 在 `workshop-baker/src/bake/decorator.rs` 中的 `Decorator` enum 添加新变体:
|
||||
|
||||
```rust
|
||||
pub enum Decorator {
|
||||
Timeout { ms: u64 },
|
||||
Retry { count: u32 },
|
||||
Parallel,
|
||||
Fallible,
|
||||
MyCustom { value: String }, // 新增
|
||||
}
|
||||
```
|
||||
|
||||
2. 在 `parse_decorator` 函数的 match 中添加解析逻辑:
|
||||
|
||||
```rust
|
||||
fn parse_decorator(line: &str) -> Option<Decorator> {
|
||||
match decorator_name {
|
||||
"timeout" => Some(Decorator::Timeout { ms: ... }),
|
||||
"retry" => Some(Decorator::Retry { count: ... }),
|
||||
"parallel" => Some(Decorator::Parallel),
|
||||
"fallible" => Some(Decorator::Fallible),
|
||||
"my_custom" => Some(Decorator::MyCustom { value: ... }), // 新增
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
3. 在调度器中处理新 Decorator 的执行逻辑。
|
||||
@@ -0,0 +1,43 @@
|
||||
# 自定义插件
|
||||
|
||||
## Shell 插件
|
||||
|
||||
Shell 插件是最简单的外部插件形式,只需提供一个可执行脚本和 `manifest.yml`。
|
||||
|
||||
**可执行脚本** (`my-plugin.sh`):
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
echo "Running custom plugin"
|
||||
```
|
||||
|
||||
**manifest.yml 格式**:
|
||||
|
||||
```yaml
|
||||
name: my-plugin
|
||||
version: "1.0"
|
||||
type: shell
|
||||
command: ./my-plugin.sh
|
||||
timeout: 300
|
||||
env:
|
||||
MY_VAR: "hello"
|
||||
```
|
||||
|
||||
字段说明:
|
||||
|
||||
| 字段 | 说明 |
|
||||
|------|------|
|
||||
| name | 插件名称 |
|
||||
| version | 插件版本 |
|
||||
| type | 插件类型:`shell` / `dylib` / `rhai` |
|
||||
| command | 可执行文件路径(shell 类型) |
|
||||
| timeout | 超时时间(秒) |
|
||||
| env | 环境变量映射 |
|
||||
|
||||
## Dylib 插件
|
||||
|
||||
动态链接库,通过 `dlopen` 加载并调用约定符号的导出函数。需确保 ABI 兼容。
|
||||
|
||||
## Rhai 插件
|
||||
|
||||
使用 Rhai 脚本引擎执行 `.rhai` 文件。当前为 `todo!()` 占位,尚未实现。
|
||||
@@ -0,0 +1,412 @@
|
||||
# 设计决策
|
||||
|
||||
本文档记录了 HoneyBiscuitWorkshop 架构演进过程中的关键设计决策及其背后的思考。
|
||||
|
||||
> **注意**:本文档面向开发者,记录的是**目标架构**的设计方向,而非当前代码的实现现状。
|
||||
> 当前 bakerd 尚未完全实现(`daemon.rs` 为 `todo!()`),bare mode 仅用于开发调试。
|
||||
|
||||
---
|
||||
|
||||
## 1. Bare Mode 与 Daemon Mode
|
||||
|
||||
### 决策
|
||||
|
||||
**Bare mode 仅用于开发调试,不承载生产语义。**
|
||||
|
||||
### 背景
|
||||
|
||||
当前代码提供两种运行入口:
|
||||
|
||||
| 模式 | 入口 | 状态 | 用途 |
|
||||
|------|------|------|------|
|
||||
| **Bare mode** | `cargo run -- prebake/bake/finalize <args>` | 可用 | 开发调试 |
|
||||
| **Daemon mode** | `cargo run --` (无子命令) | `todo!()` | 生产 |
|
||||
|
||||
### 推论
|
||||
|
||||
- Bare mode 中 `ctx.privileged` 是写死的 fake 值(`Prebake: true`, `Bake: false`),不代表生产行为
|
||||
- Bare mode 中 `os_info::get()` 读取的是 HOST 的 OS 信息,不代表环境内的 OS
|
||||
- 不要从 bare mode 的行为推导生产路径的问题
|
||||
- 不要为了"修复" bare mode 的问题而给生产设计增加不必要的抽象
|
||||
|
||||
### 代码残留
|
||||
|
||||
当前 `bake.rs` 中存在以下已在设计层面淘汰、但因 bare mode 未被清理的代码:
|
||||
|
||||
- `fn privileged()` — 从未被调用
|
||||
- `type TaskResult` — 从未被使用
|
||||
- `type TaskFn` — 从未被使用
|
||||
|
||||
这些是 `bake.rs` 废弃功能的残留,不涉及生产设计。清理它们不改变任何行为。
|
||||
|
||||
---
|
||||
|
||||
## 2. 环境模型
|
||||
|
||||
### 决策
|
||||
|
||||
**bakerd 先创建环境(容器/VM/裸机),然后 baker 在环境内执行全部三个阶段(prebake → bake → finalize)。**
|
||||
|
||||
### 架构
|
||||
|
||||
```
|
||||
bakerd
|
||||
├── prepare_environment (容器/VM/裸机) ← 先建隔离环境
|
||||
├── prebake (全在环境内执行) ← 包括 depssystem 的包安装
|
||||
├── bake (全在环境内执行) ← 构建脚本
|
||||
├── finalize (全在环境内执行) ← 部署通知
|
||||
└── cleanup
|
||||
```
|
||||
|
||||
### 推论
|
||||
|
||||
- `prepare_environment` 不是 prebake 的一个 stage——它是 prebake 的前置条件
|
||||
- `depssystem.rs` 中 `os_info::get()` 读取的应当是**环境内**的 `/etc/os-release`,逻辑正确
|
||||
- 容器镜像只需拉取(`prepare_container`),不需要在代码层面管理容器生命周期——那是 bakerd 的职责
|
||||
- 当前 `prepare_container` 返回的 `_container` 被丢弃,是因为 bakerd 尚未实现,不是设计缺陷
|
||||
|
||||
### 环境类型
|
||||
|
||||
```yaml
|
||||
env:
|
||||
type: container # Docker/Podman 容器
|
||||
# type: firecracker # Firecracker 微 VM
|
||||
# type: baremetal # 裸机直接执行
|
||||
image: ubuntu:24.04
|
||||
```
|
||||
|
||||
环境声明决定了:
|
||||
|
||||
1. 隔离级别(进程级 / VM级 / 无隔离)
|
||||
2. 操作系统和工具链(通过 `image`)
|
||||
3. 包管理器(从 OS 推导)
|
||||
|
||||
---
|
||||
|
||||
## 3. PM 选择链
|
||||
|
||||
### 决策
|
||||
|
||||
**包管理器(PM)应从环境声明中推导,而不是通过运行时探测。**
|
||||
|
||||
### 选择链
|
||||
|
||||
```
|
||||
用户声明 bakerd 确定 执行时确定
|
||||
env.image: ubuntu:24.04
|
||||
│
|
||||
▼
|
||||
容器运行时 (Docker/VM)
|
||||
│ os_info::get() 在环境内
|
||||
▼
|
||||
OS: Ubuntu 24.04
|
||||
│ 从已知映射推导
|
||||
▼
|
||||
PM: apt
|
||||
│ depssystem 使用 PM trait 操作
|
||||
▼
|
||||
apt install -y {packages}
|
||||
```
|
||||
|
||||
### 为什么淘汰 detect()
|
||||
|
||||
`pm::detect()` 的当前逻辑:
|
||||
|
||||
```rust
|
||||
pub fn detect(distro: &os_info::Info) -> Option<Box<dyn PackageManager>> {
|
||||
all_managers().into_iter().find(|pm| pm.adopt(distro))
|
||||
}
|
||||
```
|
||||
|
||||
它在解决一个**已经知道答案的问题**:
|
||||
|
||||
- 用户声明了 `image: ubuntu:24.04`
|
||||
- 容器内 `/etc/os-release` 确认了 Ubuntu
|
||||
- `detect()` 遍历所有 PM 并匹配,得出 apt
|
||||
- **结果和环境声明一致,但绕了一大圈**
|
||||
|
||||
`detect()` 的唯一价值场景:
|
||||
|
||||
- **Bare mode** 用户没配环境时,省一个配置项
|
||||
- 这在生产路径中不应出现(bakerd 总是提供环境)
|
||||
|
||||
### 推论
|
||||
|
||||
- PM trait 保留(见下节),但 `adopt()` 方法在生产路径中不需要
|
||||
- `detect()` / `adopt()` / `os_info::get()` 这条路径应当是 dev-only 的便利设施
|
||||
- Daemon 模式中 PM 的选择依据是环境声明,不是运行时探测
|
||||
- 当前 `os_info::Type` 的 58 个变体覆盖不全(仅 Pacman 可用)——这是 bare mode 的问题,不影响生产设计
|
||||
|
||||
---
|
||||
|
||||
## 4. PM Trait 的职责边界
|
||||
|
||||
### 决策
|
||||
|
||||
**PM trait 保留核心操作能力,淘汰探测分类能力。**
|
||||
|
||||
### 核心方法
|
||||
|
||||
```rust
|
||||
pub trait PackageManager {
|
||||
fn name(&self) -> &'static str;
|
||||
fn update(&self) -> Vec<Command>;
|
||||
fn install(&self, package: &[String]) -> Vec<Command>;
|
||||
fn change_mirror(&self, repo: &str, osinfo: Option<&Info>) -> Vec<Command>;
|
||||
fn add_repository(&self, repo: &Value, osinfo: Option<&Info>) -> Vec<Command>;
|
||||
}
|
||||
```
|
||||
|
||||
这四个方法是 PM-specific 的操作,具有真实复杂度:
|
||||
|
||||
| 操作 | PM 间差异 | 原因 |
|
||||
|------|-----------|------|
|
||||
| `install` | 参数完全不同 | `apt install -y` vs `pacman -S --noconfirm` |
|
||||
| `update` | 命令不同 | `apt update` vs `pacman -Sy` |
|
||||
| `change_mirror` | 配置格式不同 | sources.list vs mirrorlist vs .repo |
|
||||
| `add_repository` | 仓库机制不同 | PPA vs AUR vs COPR |
|
||||
|
||||
### 为什么镜像源和仓库在容器中仍然需要
|
||||
|
||||
容器不解决源管理问题:
|
||||
|
||||
```
|
||||
docker pull ubuntu:24.04
|
||||
# 容器内 sources.list → archive.ubuntu.com
|
||||
# 但用户在北京 → 需要 mirror.tuna.tsinghua.edu.cn
|
||||
# 这不是预置在镜像中的——源是运行态配置
|
||||
```
|
||||
|
||||
`change_mirror` 和 `add_repository` 都是**容器运行时需要**的操作,不能预置在基础镜像中。PM trait 的存在价值就在于这些操作。
|
||||
|
||||
### 淘汰的方法
|
||||
|
||||
```rust
|
||||
fn adopt(&self, distro: &Info) -> bool; // 生产路径不需要
|
||||
fn binary(&self) -> &'static str; // 从未被使用
|
||||
```
|
||||
|
||||
### 实现策略
|
||||
|
||||
在 daemon 模式下,PM 的选型由环境声明确定(`image: ubuntu:24.04` → `apt`),PM 实现仅需按需注册,不需要自述适配范围:
|
||||
|
||||
```rust
|
||||
// 不再需要 detect() + adopt()
|
||||
// PM 实例由工厂方法根据 PM 名称字符串创建
|
||||
pub fn from_name(name: &str) -> Option<Box<dyn PackageManager>> {
|
||||
match name {
|
||||
"apt" => Some(Box::new(Apt)),
|
||||
"pacman" => Some(Box::new(Pacman)),
|
||||
"dnf" => Some(Box::new(Dnf)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 5. 发行版多态
|
||||
|
||||
### 决策
|
||||
|
||||
**流水线不应是发行版多态的。发行版差异由容器化解决。**
|
||||
|
||||
### 原因
|
||||
|
||||
流水线的执行体(bake 脚本)天然发行版绑定:
|
||||
|
||||
```bash
|
||||
# @pipeline
|
||||
function build() {
|
||||
cmake -B build -DCMAKE_INSTALL_PREFIX=/usr # Linux 路径
|
||||
make -j$(nproc)
|
||||
}
|
||||
```
|
||||
|
||||
- 换到 macOS:`/usr` 只读,路径不同
|
||||
- 换到 Windows:MSVC 而不是 GCC
|
||||
- 换到 Alpine:musl 而不是 glibc
|
||||
|
||||
这与 PM 的差异是同一类问题,但 PM 抽象不能解决它。**容器才是解决方案:**
|
||||
|
||||
```yaml
|
||||
# 在 Ubuntu 上构建
|
||||
env:
|
||||
type: container
|
||||
image: ubuntu:24.04
|
||||
```
|
||||
|
||||
```yaml
|
||||
# 在 Alpine 上构建
|
||||
env:
|
||||
type: container
|
||||
image: alpine:3.19
|
||||
```
|
||||
|
||||
### 各层面的发行版相关性
|
||||
|
||||
| 层面 | 发行版相关 | 解决方案 |
|
||||
|------|-----------|----------|
|
||||
| DepsSystem(系统包) | ✅ | 容器内 PM |
|
||||
| DepsUser(用户包管理器) | ❌ | 天然无关(cargo/npm/pip 跨发行版一致) |
|
||||
| Bake(构建脚本) | ✅ | 容器内执行 |
|
||||
| Finalize(后处理) | ❌ | 纯产物操作,无关发行版 |
|
||||
|
||||
### UPM 的特殊位置
|
||||
|
||||
用户包管理器(UPM,如 cargo、npm、pip、nix)天然跨发行版:
|
||||
|
||||
```bash
|
||||
cargo install --locked bat # Ubuntu 和 Arch 上都一样
|
||||
npm install -g typescript # Ubuntu 和 Arch 上都一样
|
||||
```
|
||||
|
||||
`depsuser.rs` 处理的正是这个层面。但 `collect_upm_sysdeps()` 需要知道系统 PM——这是 UPM 和系统 PM 之间的 bridge。在设计上,这个 bridge 的 PM 信息应来自环境声明,而非 `detect()`。
|
||||
|
||||
---
|
||||
|
||||
## 6. drop_after 安全模型
|
||||
|
||||
### 决策
|
||||
|
||||
**`drop_after` 指定流水线在哪个 stage 之后从 root 降权到 vulcan 用户。合理的默认值是 `DepsUser`。**
|
||||
|
||||
### Stage 顺序
|
||||
|
||||
```rust
|
||||
pub enum PrebakeStage {
|
||||
Init,
|
||||
Bootstrap,
|
||||
EarlyHook,
|
||||
DepsSystem, // ← 默认
|
||||
DepsUser,
|
||||
LateHook,
|
||||
Ready,
|
||||
Never, // ← 永远不降权
|
||||
}
|
||||
```
|
||||
|
||||
### 各 stage 的特权需求
|
||||
|
||||
| Stage | 需要 root | 原因 |
|
||||
|-------|-----------|------|
|
||||
| Bootstrap | ✅ | 创建系统用户(vulcan)、配置 sudo/doas |
|
||||
| EarlyHook | ❓ | 用户自定义 |
|
||||
| DepsSystem | ✅ | `apt install` 需要 root |
|
||||
| DepsUser | ❌ | `cargo install` 不需要 root |
|
||||
| LateHook | ❓ | 用户自定义 |
|
||||
| Ready / Bake | ❌ | 构建应以非特权用户运行 |
|
||||
|
||||
### 正确的配置
|
||||
|
||||
```yaml
|
||||
security:
|
||||
drop_after: "DepsUser"
|
||||
```
|
||||
|
||||
执行效果:
|
||||
|
||||
```
|
||||
Bootstrap root ← 创建用户
|
||||
EarlyHook root ← hook 以 root 运行
|
||||
DepsSystem root ← apt install -y {packages}
|
||||
DepsUser root ← cargo install {packages}
|
||||
LateHook vulcan ← 降权,用户脚本不能 rm -rf /
|
||||
Ready vulcan
|
||||
Bake vulcan ← 构建以 vulcan 运行
|
||||
```
|
||||
|
||||
### 禁止的配置
|
||||
|
||||
`drop_after` 设置为 `Bootstrap` 或 `EarlyHook` 在当前 stage 顺序下会损坏 DepsStage(需要 root 权限的 `apt install` 在降权后无法执行)。
|
||||
|
||||
**配置验证应拒绝 `drop_after < DepsUser`**(以 root 运行的 user 级包安装没有意义,但 DepsSystem 必须 root)。
|
||||
|
||||
### 特殊值 Never
|
||||
|
||||
```yaml
|
||||
security:
|
||||
drop_after: "Never" # 整条流水线以 root 运行
|
||||
```
|
||||
|
||||
适用场景:Docker 构建、内核模块编译等需要 root 权限的操作。Bootstrap 的 `user: root` 配置也会产生相同效果。
|
||||
|
||||
---
|
||||
|
||||
## 7. 当前代码中的真问题
|
||||
|
||||
以下问题是当前代码中确实存在的,与 bare/daemon 模式无关:
|
||||
|
||||
### 7.1 Apt::adopt 永远返回 false
|
||||
|
||||
```rust
|
||||
// workshop-engine/src/pm/apt.rs:20-23
|
||||
fn adopt(&self, distro: &Info) -> bool {
|
||||
return false; // ← 永远 false
|
||||
// todo!(); // ← 死代码
|
||||
}
|
||||
```
|
||||
|
||||
- `adopt()` 在 `return false` 后无法执行 `todo!()`
|
||||
- Apt 永远不会被 `detect()` 选中
|
||||
- 后果:Debian/Ubuntu 用户在 bare mode 下永远检测不到 PM
|
||||
- 修复方向:移除 return false,实现真正的 adopt 逻辑(Debian/Ubuntu/Mint/Pop 等发行版的匹配)
|
||||
|
||||
### 7.2 PM 覆盖严重不全
|
||||
|
||||
当前 `all_managers()` 仅注册了 Pacman:
|
||||
|
||||
```rust
|
||||
pub fn all_managers() -> Vec<Box<dyn PackageManager>> {
|
||||
vec![
|
||||
// Box::new(apt::Apt), // adopt 坏了
|
||||
Box::new(pacman::Pacman), // 唯一可用
|
||||
// Box::new(dnf::Dnf), // 未实现
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
os_info 3.14.0 定义了 58 种发行版类型,Pacman 仅覆盖其中 9 种(Arch 系)。其余 49 种无 PM 覆盖。
|
||||
|
||||
### 7.3 死代码
|
||||
|
||||
| 位置 | 内容 | 状态 |
|
||||
|------|------|------|
|
||||
| `bake.rs:98` | `fn privileged()` | 定义但未调用 |
|
||||
| `bake.rs:108` | `type TaskResult` | 定义但未使用 |
|
||||
| `bake.rs:110` | `type TaskFn` | 定义但未使用 |
|
||||
| `workshop-baker/src/prebake/stage/environment.rs` | `prepare_environment()` | 定义但未接入执行流 |
|
||||
|
||||
---
|
||||
|
||||
## 8. Daemon 设计原则(汇总)
|
||||
|
||||
以上讨论最终可以提炼为以下设计原则,供 bakerd 重构时参考:
|
||||
|
||||
### P1. 环境声明驱动
|
||||
|
||||
```
|
||||
用户声明 env.image → bakerd 创建环境 → 环境内确定 PM → depssystem 操作 PM
|
||||
```
|
||||
|
||||
PM 的选择权在环境声明,不在运行时探测。`detect()` 不出现在生产路径中。
|
||||
|
||||
### P2. 容器是发行版隔离的边界
|
||||
|
||||
发行版差异在容器边界被隔断。容器内是已知、确定的环境。PM trait 只负责"已知 PM 的操作"(install、update、mirror、repo),不负责"猜是什么 PM"。
|
||||
|
||||
### P3. 降权边界可配置但需校验
|
||||
|
||||
`drop_after` 从 root 降权到 vulcan,默认在 DepsUser 之后。配置验证需确保:降权不会损坏需要 root 的 stage。
|
||||
|
||||
### P4. 最小化 trait 抽象
|
||||
|
||||
PM trait 保留操作系统所需的核心方法。不在 trait 中嵌入分类功能(adopt)。分类是一个数据映射问题,不是多态问题。
|
||||
|
||||
### P5. Bare mode 是开发工具,不承载生产行为
|
||||
|
||||
Bare mode 的行为(host 上直接执行、write static privileged 等)不代表生产设计。不要为了优化 bare mode 而增加生产路径的抽象负担。
|
||||
|
||||
### P6. 天然跨发行版的部分不需要抽象
|
||||
|
||||
UPM(cargo/npm/pip/nix)跨发行版一致,不需要 PM 级别的抽象,不需要 detect,不需要 distro 感知。trait 只应用于确实有 PM-specific 差异的部分。
|
||||
@@ -0,0 +1,15 @@
|
||||
# 开发环境
|
||||
|
||||
本项目要求:
|
||||
|
||||
- **Rust 1.85+**(使用 edition 2024)
|
||||
- **Docker**(集成测试需要)
|
||||
- **mdBook**(文档构建,可选)
|
||||
|
||||
初始化开发环境:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/user/HoneyBiscuitWorkshop.git
|
||||
cd HoneyBiscuitWorkshop
|
||||
cargo build
|
||||
```
|
||||
@@ -0,0 +1,57 @@
|
||||
# Engine 模块指南
|
||||
|
||||
`engine/` 是 Baker 的运行时层,负责构建脚本执行、资源隔离与包管理。
|
||||
|
||||
## 模块划分
|
||||
|
||||
| 文件 | 职责 |
|
||||
|------|------|
|
||||
| `executor.rs` | 脚本执行,管理子进程生命周期与输出捕获 |
|
||||
| `cgroups.rs` | Linux cgroup v2 资源限制配置(CPU、内存、IO) |
|
||||
| `pm.rs` | 系统包管理器检测与抽象 trait |
|
||||
| `pm/pacman.rs` | Pacman 包管理器实现 |
|
||||
| `upm.rs` | 用户级包管理器(Nix、Guix、Cargo、NPM 等) |
|
||||
| `repology.rs` | Repology 包名查询与版本解析 |
|
||||
| `repology/local.rs` | 本地包数据库缓存 |
|
||||
| `socket.rs` | Unix Domain Socket 通信(daemon 模式) |
|
||||
|
||||
## PackageManager Trait
|
||||
|
||||
系统包管理器的统一接口:
|
||||
|
||||
```rust
|
||||
trait PackageManager {
|
||||
fn name(&self) -> &str;
|
||||
fn install(&self, packages: &[&str]) -> Result<()>;
|
||||
fn update(&self) -> Result<()>;
|
||||
}
|
||||
```
|
||||
|
||||
新增包管理器时:
|
||||
1. 在 `pm/` 下新建文件实现 `PackageManager`
|
||||
2. 在 `pm.rs` 的 `PackageManagerEnum` 中注册变体
|
||||
3. 在 `PrebakeConfig` 解析中增加匹配分支
|
||||
|
||||
## 资源限制
|
||||
|
||||
`cgroups.rs` 通过 Linux cgroup v2 限制构建资源:
|
||||
|
||||
| 限制项 | cgroup 文件 | 配置来源 |
|
||||
|--------|-------------|---------|
|
||||
| CPU 配额 | `cpu.max` | `resources.cpu` |
|
||||
| 内存上限 | `memory.max` | `resources.memory` |
|
||||
| 磁盘 IO | `io.max` | 预留,暂未绑定 |
|
||||
|
||||
## 执行流程
|
||||
|
||||
1. `Executor` 接收渲染后的脚本路径
|
||||
2. 根据 `PrebakeConfig` 创建 cgroup 限制(如启用)
|
||||
3. `std::process::Command` 启动子进程
|
||||
4. 实时捕获 stdout/stderr
|
||||
5. 超时或完成后,收集退出码与输出
|
||||
|
||||
## 注意事项
|
||||
|
||||
- `cgroups.rs` 仅在 Linux 上生效,非 Linux 目标编译时相关代码被条件编译排除
|
||||
- `upm.rs` 将用户包管理器调用委托给子进程,不直接管理包状态
|
||||
- `repology.rs` 通过 HTTP 查询 Repology API 将通用包名映射到发行版特定名称
|
||||
@@ -0,0 +1,60 @@
|
||||
# 错误处理模式
|
||||
|
||||
Baker 采用 `thiserror` + `anyhow` 的组合策略处理错误。
|
||||
|
||||
## 选取原则
|
||||
|
||||
| 场景 | 使用 | 原因 |
|
||||
|------|------|------|
|
||||
| 库代码(types/、bake/、engine/) | `thiserror` | 结构化错误,调用方需要 match 处理 |
|
||||
| 二进制入口(main.rs) | `anyhow` | 统一包装,快速传播 |
|
||||
| 测试代码 | `anyhow::Result` | 减少样板,失败即 panic |
|
||||
|
||||
## thiserror 示例
|
||||
|
||||
```rust
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum BakeError {
|
||||
#[error("Decorator parse error @{decorator} (line {line_num}): {reason}")]
|
||||
DecoratorParseError {
|
||||
decorator: String,
|
||||
line_num: usize,
|
||||
reason: String,
|
||||
},
|
||||
|
||||
#[error("Circular dependency: {0:?}")]
|
||||
CircularDependency(Vec<String>),
|
||||
|
||||
#[error("IO error: {0}")]
|
||||
IoError(#[from] std::io::Error),
|
||||
}
|
||||
```
|
||||
|
||||
## anyhow 示例
|
||||
|
||||
```rust
|
||||
use anyhow::{Context, Result};
|
||||
|
||||
fn read_config(path: &str) -> Result<String> {
|
||||
std::fs::read_to_string(path)
|
||||
.with_context(|| format!("Failed to read config: {}", path))
|
||||
}
|
||||
```
|
||||
|
||||
## 退出码映射
|
||||
|
||||
实现 `HasExitCode` trait 将错误映射到标准退出码:
|
||||
|
||||
```rust
|
||||
impl HasExitCode for BakeError {
|
||||
fn exit_code(&self) -> i32 {
|
||||
match self {
|
||||
BakeError::DecoratorParseError { .. } => EXITCODE_PARSE_ERROR,
|
||||
BakeError::IoError(_) => EXITCODE_IO_ERROR,
|
||||
_ => EXITCODE_GENERAL_ERROR,
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
@@ -1 +1,26 @@
|
||||
what
|
||||
# 卷3 开发者手册
|
||||
|
||||
## 项目简介
|
||||
|
||||
蜜饼工坊 (HoneyBiscuitWorkshop) 是一个 Rust 编写的 CI/CD 系统,支持 LLM 辅助的流水线自动配置。
|
||||
|
||||
## 代码结构
|
||||
|
||||
当前活跃维护的 crate:
|
||||
|
||||
| Crate | 说明 |
|
||||
|-------|------|
|
||||
| workshop-baker | 主编排器 (CLI + Daemon) |
|
||||
| workshop-vault | 密钥管理客户端 |
|
||||
| workshop-cert | TLS 证书生成器 |
|
||||
| workshop-deviceid | 设备 ID 库 |
|
||||
|
||||
其余模块(如 workshop-agent、workshop-llm-detector 等)已归档至 `archived/` 目录,不再活跃维护。
|
||||
|
||||
## 阅读指南
|
||||
|
||||
本卷面向开发者,涵盖架构、扩展和贡献指南。各章节内容如下:
|
||||
|
||||
- **架构**:整体架构、数据流、核心模块与插件系统
|
||||
- **扩展**:添加包管理器、自定义插件与 Decorator、模板语法
|
||||
- **贡献指南**:开发环境、代码规范、测试与提交流程
|
||||
@@ -0,0 +1,26 @@
|
||||
# 邮件模板
|
||||
|
||||
`MailConfig` 中的 `template` 字段支持 minijinja 语法,用于自定义邮件内容:
|
||||
|
||||
```yaml
|
||||
mail:
|
||||
smtp_host: "smtp.example.com"
|
||||
smtp_port: 587
|
||||
from: "ci@example.com"
|
||||
to: ["dev@example.com"]
|
||||
template: |
|
||||
Subject: [CI] {{PROJECT_NAME}} - {{BUILD_STATUS}}
|
||||
|
||||
项目: {{PROJECT_NAME}}
|
||||
分支: {{BRANCH}}
|
||||
提交: {{COMMIT_SHA}}
|
||||
状态: {{BUILD_STATUS}}
|
||||
|
||||
{% if BUILD_STATUS == "success" %}
|
||||
构建成功!
|
||||
{% else %}
|
||||
构建失败,请检查日志。
|
||||
{% endif %}
|
||||
```
|
||||
|
||||
邮件模板中可使用所有 ExecutionContext 提供的模板变量。
|
||||
@@ -0,0 +1,74 @@
|
||||
# 插件系统指南
|
||||
|
||||
Baker 的插件系统基于 `Plugin` trait,支持四种运行时形态。
|
||||
|
||||
## Plugin Trait
|
||||
|
||||
```rust
|
||||
trait Plugin {
|
||||
async fn execute(&self, ctx: &ExecutionContext) -> Result<()>;
|
||||
}
|
||||
```
|
||||
|
||||
所有插件接收统一的 `ExecutionContext`,包含任务 ID、构建状态、环境变量与模板变量表。
|
||||
|
||||
## 内部插件
|
||||
|
||||
随 baker 二进制编译,直接调用 Rust 代码,无额外启动开销。
|
||||
|
||||
| 插件 | 文件 | 功能 |
|
||||
|------|------|------|
|
||||
| `mail` | `finalize/plugin/internal/mail.rs` | SMTP 邮件通知,支持 minijinja 模板 |
|
||||
| `webhook` | `finalize/plugin/internal/webhook.rs` | HTTP POST 回调 |
|
||||
| `satori` | `finalize/plugin/internal/satori.rs` | Satori 集成(占位) |
|
||||
| `insitenotify` | `finalize/plugin/internal/insitenotify.rs` | 站内通知(占位) |
|
||||
|
||||
内部插件配置位于 `finalize.yml` 的 `notification` 节点下,按数字键分组:
|
||||
|
||||
```yaml
|
||||
notification:
|
||||
1:
|
||||
mail:
|
||||
to: "admin@example.com"
|
||||
2:
|
||||
webhook:
|
||||
url: "https://hooks.example.com/ci"
|
||||
```
|
||||
|
||||
## 外部插件
|
||||
|
||||
### Shell 插件
|
||||
|
||||
可执行脚本,通过子进程调用。需在同级目录提供 `manifest.yml`:
|
||||
|
||||
```yaml
|
||||
name: "my-plugin"
|
||||
type: "shell"
|
||||
entry: "run.sh"
|
||||
```
|
||||
|
||||
### Dylib 插件
|
||||
|
||||
动态链接库,通过 `dlopen` 加载并调用约定符号的导出函数。
|
||||
|
||||
### Rhai 插件
|
||||
|
||||
使用 Rhai 脚本引擎执行 `.rhai` 文件(当前为 `todo!()` 占位,尚未实现)。
|
||||
|
||||
## 插件注册流程
|
||||
|
||||
1. `finalize.yml` 中声明插件配置
|
||||
2. `FinalizeStage::Plugin` 阶段遍历配置
|
||||
3. 根据 `type` 字段实例化对应插件类型
|
||||
4. 调用 `Plugin::execute()`,传入当前 `ExecutionContext`
|
||||
|
||||
## 模板上下文
|
||||
|
||||
所有插件共享同一模板变量表,由 `ExecutionContext` 提供:
|
||||
|
||||
| 命名空间 | 可用变量 |
|
||||
|----------|---------|
|
||||
| `build` | `status`, `id`, `duration`, `url`, `status_color` |
|
||||
| `pipeline` | `name` |
|
||||
| `commit` | `hash`, `author`, `author_email`, `message` |
|
||||
| `secret` | Vault 中存储的任意密钥(通过 `{{ secret.key }}` 引用) |
|
||||
@@ -0,0 +1,11 @@
|
||||
# PR 流程
|
||||
|
||||
提交 PR 前请确保通过所有检查:
|
||||
|
||||
```bash
|
||||
cargo fmt --all -- --check
|
||||
cargo clippy -- -D warnings
|
||||
cargo test
|
||||
```
|
||||
|
||||
然后提交并创建 Pull Request。CI 会自动运行完整测试套件。
|
||||
@@ -0,0 +1,88 @@
|
||||
# Resource 模型
|
||||
|
||||
## 核心思想
|
||||
|
||||
整个系统只关心两件事:东西从哪来(fetch),东西到哪去(publish)。
|
||||
|
||||
fetch 方向:URL → 本地路径 → 使用。
|
||||
publish 方向:本地路径 → 变换(插件)→ URL。
|
||||
|
||||
两个方向共用同一套寻址方式,但 Resource 中间态只存在于 fetch 方向。
|
||||
|
||||
## URL 格式
|
||||
|
||||
```
|
||||
getter::url?param1=value1¶m2=value2
|
||||
```
|
||||
|
||||
getter 显式声明意图。不是"我猜这是个 git 仓库",而是"使用者告诉我这是什么"。
|
||||
|
||||
只有三个 getter:
|
||||
|
||||
- `git` — 克隆仓库,必要时 checkout 指定版本
|
||||
- `http` / `https` — 下载文件,必要时校验 checksum
|
||||
- `file` — 复制本地文件或目录
|
||||
|
||||
版本锁定不用 `@v1.0`(`@` 在 URL 里是认证分隔符),用 `?ref=v1.0`。
|
||||
|
||||
没有 `github.com/user/repo` 看起来应该 clone 它这种黑魔法。
|
||||
没有 `.tar.gz` 看起来应该解压它的惊喜。
|
||||
URL 是 URL,行为是行为,两者用 getter:: 显式绑定。
|
||||
|
||||
## Resource 不是网络地址
|
||||
|
||||
流水线阶段(prebake / finalize / notify)只接触 Resource:一个名字 + 一个本地路径,已就绪。它们不经由网络、不解析 URL、不碰 getter。
|
||||
|
||||
所有 fetch 行为在阶段启动前完成:
|
||||
|
||||
- worker 模式:bakerd fetch → 填入 ResourceRegistry → 启动 baker
|
||||
- standalone 模式:baker 自己 fetch → 填入 ResourceRegistry → 执行阶段
|
||||
- bare 模式:不 fetch,传入的必须是 Resource
|
||||
|
||||
ResourceRegistry 是名字到路径的映射。填进去就只读了。
|
||||
|
||||
## getterurl 是独立的 crate
|
||||
|
||||
URL 格式解析 + 基本 fetch 放在 workshop-getterurl,和 baker 本身没有耦合。任何人都可以用。
|
||||
|
||||
crate 分两层:
|
||||
|
||||
```
|
||||
结构层(parser):
|
||||
"git::https://host/repo.git?ref=v1.0" → { getter: "git", url: Url, params: Params }
|
||||
|
||||
获取层(builtin getters):
|
||||
getter.fetch(url, params, dst) → local_path
|
||||
git: git clone + optional checkout
|
||||
http: HTTP GET + optional checksum
|
||||
file: std::fs::copy
|
||||
```
|
||||
|
||||
获取层是字面意义的——不解压、不推断、不猜意图。
|
||||
|
||||
## Checksum
|
||||
|
||||
checksum 作为 URL query 参数传递。三态语义:
|
||||
|
||||
- `?checksum=sha256:abc` — 必须匹配
|
||||
- `?checksum=` — 显式禁用校验
|
||||
- 无 checksum — 如果服务端返回 Content-Digest 头,校验它
|
||||
|
||||
getter 不主动校验,它只执行 URL 里指定的操作。
|
||||
|
||||
## Artifact 不经过 Resource
|
||||
|
||||
artifact 是 bake 的产物,自产生就开始被使用,不存在中间态。它的路径就是 bake 产出的本地路径,不走 ResourceRegistry。
|
||||
|
||||
artifact 的 publish 方向由插件定义行为——可能上传、可能扫描、可能只记录。插件定义最终的行为,只要是自洽的。
|
||||
|
||||
## 三种运行模式
|
||||
|
||||
```
|
||||
fetch 执行者 fetch 在阶段内
|
||||
daemon 模式 bakerd 否
|
||||
standalone baker 自身 否(阶段启动前已完成)
|
||||
bare 模式 无 否(必须传入 Resource)
|
||||
```
|
||||
|
||||
bare 模式下传入的必须是已经就绪的 Resource。外部 URL 是 daemon 或 standalone 的事,bare 不负责。
|
||||
@@ -0,0 +1,23 @@
|
||||
# 模板变量
|
||||
|
||||
Finalize 阶段使用 `{{VARIABLE}}` 语法进行模板渲染,可用变量来自 `ExecutionContext`:
|
||||
|
||||
| 变量 | 说明 |
|
||||
|------|------|
|
||||
| `{{PROJECT_NAME}}` | 项目名称 |
|
||||
| `{{BUILD_ID}}` | 构建ID |
|
||||
| `{{BUILD_STATUS}}` | 构建状态 |
|
||||
| `{{COMMIT_SHA}}` | 提交哈希 |
|
||||
| `{{BRANCH}}` | 分支名 |
|
||||
| `{{START_TIME}}` | 构建开始时间 |
|
||||
| `{{END_TIME}}` | 构建结束时间 |
|
||||
|
||||
模板引擎基于 minijinja,支持条件、循环等高级语法:
|
||||
|
||||
```yaml
|
||||
{% if BUILD_STATUS == "success" %}
|
||||
Build {{PROJECT_NAME}}/{{BRANCH}} succeeded!
|
||||
{% else %}
|
||||
Build {{PROJECT_NAME}}/{{BRANCH}} failed.
|
||||
{% endif %}
|
||||
```
|
||||
@@ -0,0 +1,61 @@
|
||||
# 测试策略
|
||||
|
||||
## 测试层级
|
||||
|
||||
| 层级 | 位置 | 适用场景 | 工具 |
|
||||
|------|------|---------|------|
|
||||
| 单元测试 | `src/*/mod.rs` 内 `#[cfg(test)]` | 纯逻辑、无 IO、无副作用 | `#[test]` / `#[tokio::test]` |
|
||||
| 集成测试 | `tests/*.rs` | 跨模块协作、crate 公共 API | `cargo test --test name` |
|
||||
| 系统测试 | `tests/` (pytest) | Docker 环境、端到端 | `pytest` |
|
||||
|
||||
## 单元测试规范
|
||||
|
||||
```rust
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_parse_valid_duration() {
|
||||
assert_eq!(parse_duration_to_ms("30s").unwrap(), 30_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_duration_overflow_fails() {
|
||||
let result = parse_duration_to_ms("9999999999y");
|
||||
assert!(result.is_err());
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## 异步测试
|
||||
|
||||
```rust
|
||||
#[tokio::test]
|
||||
async fn test_async_execution() {
|
||||
let result = execute_command("echo hello").await.unwrap();
|
||||
assert_eq!(result.stdout, "hello\n");
|
||||
}
|
||||
```
|
||||
|
||||
## 集成测试规范
|
||||
|
||||
- 使用 `use workshop_baker::*;` 导入 crate 公共 API
|
||||
- 每个测试独立,不依赖执行顺序
|
||||
- 临时文件使用 `tempfile` crate,测试后自动清理
|
||||
|
||||
## 运行命令
|
||||
|
||||
```bash
|
||||
# 全部测试
|
||||
cargo test -p workshop-baker
|
||||
|
||||
# 仅单元测试
|
||||
cargo test --lib -p workshop-baker
|
||||
|
||||
# 仅指定集成测试
|
||||
cargo test --test types_config_integration
|
||||
|
||||
# 系统测试(需 Docker)
|
||||
pytest tests/integration/test_prebake.py -v
|
||||
```
|
||||
@@ -0,0 +1,53 @@
|
||||
# 类型系统指南
|
||||
|
||||
`types/` 模块集中定义跨模块共享的核心类型,避免循环依赖与重复定义。
|
||||
|
||||
## 设计原则
|
||||
|
||||
| 原则 | 说明 |
|
||||
|------|------|
|
||||
| 单一事实来源 | 同一概念只在 `types/` 定义一次,各模块通过 `use crate::types::` 引用 |
|
||||
| 自包含序列化 | 复杂类型自带 serde 自定义序列化/反序列化逻辑,调用方无感知 |
|
||||
| 零成本抽象 | 偏好新类型模式(如 `MemSize(u64)`)而非裸原生类型 |
|
||||
|
||||
## 核心类型速查
|
||||
|
||||
| 类型 | 文件 | 用途 |
|
||||
|------|------|------|
|
||||
| `Architecture` | `architecture.rs` | 目标架构枚举,支持 serde 单字符串或数组 |
|
||||
| `MemSize` | `memsize.rs` | 内存/磁盘大小,支持 `"1 GiB"` / `"512 MiB"` 等人类可读格式 |
|
||||
| `BuilderConfig` | `builderconfig.rs` | 构建环境配置(Docker/Firecracker/Baremetal/Custom) |
|
||||
| `CompressionMethod` | `compression.rs` | 压缩算法枚举 |
|
||||
| `CacheStrategy` / `CacheDirectory` | `cache.rs` | 缓存策略与目录配置 |
|
||||
| `CustomCommand` | `command.rs` | 自定义命令及其超时配置 |
|
||||
| `GitVersion` | `repology.rs` | Git 引用版本解析(branch/tag/commit) |
|
||||
| `RepologyEndpoint` | `repology.rs` | Repology 包查询端点配置 |
|
||||
| `BuildStatus` | `buildstatus.rs` | 流水线整体状态 |
|
||||
| `StagePhase` / `StageResult` | `buildstatus.rs` | 阶段执行阶段与结果 |
|
||||
|
||||
## Serde 定制示例
|
||||
|
||||
### 单值或数组兼容反序列化
|
||||
|
||||
`Architecture` 支持 YAML 中写 `"x86_64"` 或 `["x86_64", "aarch64"]`:
|
||||
|
||||
```rust
|
||||
fn parse_architecture<'de, D>(deserializer: D) -> Result<HashSet<Architecture>, D::Error>
|
||||
where D: serde::Deserializer<'de>
|
||||
{
|
||||
// 实现 Visitor,同时处理 visit_str 与 visit_seq
|
||||
}
|
||||
```
|
||||
|
||||
### 人类可读大小解析
|
||||
|
||||
`MemSize` 将 `"4 GiB"` 解析为底层字节数:
|
||||
|
||||
```rust
|
||||
let mem: MemSize = "4 GiB".parse()?; // MemSize(4294967296)
|
||||
```
|
||||
|
||||
## 使用约束
|
||||
|
||||
- `types/` 不依赖 `bake/`、`engine/`、`prebake/`、`finalize/` 中的任何模块
|
||||
- 新增跨模块共享类型时,应优先放入 `types/` 而非散落在业务模块中
|
||||
@@ -1,112 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# =============================================================================
|
||||
# HoneyBiscuitWorkshop System Test Runner
|
||||
# =============================================================================
|
||||
# Usage: ./runner.sh [auto|manual] [-- pytest-args...]
|
||||
#
|
||||
# Modes:
|
||||
# auto - Build, run all tests, collect reports, cleanup (default)
|
||||
# manual - Build, start container shell for interactive testing
|
||||
#
|
||||
# Examples:
|
||||
# ./runner.sh # Run all tests automatically
|
||||
# ./runner.sh auto -k test_prebake # Run only prebake tests
|
||||
# ./runner.sh manual # Drop into container shell
|
||||
# =============================================================================
|
||||
|
||||
MODE="${1:-auto}"
|
||||
shift 2>/dev/null || true
|
||||
PYTEST_ARGS="$@"
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
STAGING_DIR="/tmp/hbw-system-test"
|
||||
IMAGE_NAME="hbw-system-test:latest"
|
||||
|
||||
# ---- Cleanup handler ----
|
||||
cleanup() {
|
||||
echo "[runner] Cleaning up..."
|
||||
docker rm -f hbw-system-test-container 2>/dev/null || true
|
||||
rm -rf "$STAGING_DIR"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# ---- Step 1: Build the binary ----
|
||||
build_binary() {
|
||||
echo "[runner] Building workshop-baker (release)..."
|
||||
cd "$SCRIPT_DIR/workshop-baker"
|
||||
cargo build --release 2>&1
|
||||
echo "[runner] Build complete."
|
||||
}
|
||||
|
||||
# ---- Step 2: Prepare staging directory ----
|
||||
prepare_staging() {
|
||||
echo "[runner] Preparing staging directory..."
|
||||
rm -rf "$STAGING_DIR"
|
||||
mkdir -p "$STAGING_DIR"
|
||||
|
||||
# Copy binary
|
||||
cp "$SCRIPT_DIR/workshop-baker/target/release/workshop-baker" "$STAGING_DIR/"
|
||||
chmod +x "$STAGING_DIR/workshop-baker"
|
||||
|
||||
# Copy examples if they exist
|
||||
if [ -d "$SCRIPT_DIR/workshop-baker/examples" ]; then
|
||||
cp -r "$SCRIPT_DIR/workshop-baker/examples" "$STAGING_DIR/examples"
|
||||
else
|
||||
mkdir -p "$STAGING_DIR/examples"
|
||||
fi
|
||||
|
||||
# Copy llm.env if it exists
|
||||
if [ -f "$SCRIPT_DIR/llm.env" ]; then
|
||||
cp "$SCRIPT_DIR/llm.env" "$STAGING_DIR/llm.env"
|
||||
fi
|
||||
|
||||
echo "[runner] Staging ready at $STAGING_DIR"
|
||||
}
|
||||
|
||||
# ---- Step 3: Build Docker image ----
|
||||
build_image() {
|
||||
echo "[runner] Building Docker image: $IMAGE_NAME ..."
|
||||
docker build -t "$IMAGE_NAME" -f "$SCRIPT_DIR/tests/Dockerfile" "$SCRIPT_DIR"
|
||||
echo "[runner] Image built: $IMAGE_NAME"
|
||||
}
|
||||
|
||||
# ---- Step 4: Run container ----
|
||||
run_container() {
|
||||
local mode="$1"
|
||||
echo "[runner] Starting container in $mode mode..."
|
||||
|
||||
mkdir -p "$SCRIPT_DIR/tests/results"
|
||||
|
||||
if [ "$mode" = "manual" ]; then
|
||||
docker run -it --rm \
|
||||
--name hbw-system-test-container \
|
||||
--privileged \
|
||||
-v "$STAGING_DIR:/workshop:ro" \
|
||||
-v "$SCRIPT_DIR/tests/results:/results" \
|
||||
-e TEST_MODE=manual \
|
||||
"$IMAGE_NAME"
|
||||
else
|
||||
docker run --rm \
|
||||
--name hbw-system-test-container \
|
||||
--privileged \
|
||||
-v "$STAGING_DIR:/workshop:ro" \
|
||||
-v "$SCRIPT_DIR/tests/results:/results" \
|
||||
-e TEST_MODE=auto \
|
||||
"$IMAGE_NAME" $PYTEST_ARGS
|
||||
fi
|
||||
}
|
||||
|
||||
# ---- Main flow ----
|
||||
echo "============================================"
|
||||
echo " HoneyBiscuitWorkshop System Test Runner"
|
||||
echo " Mode: $MODE"
|
||||
echo "============================================"
|
||||
|
||||
build_binary
|
||||
prepare_staging
|
||||
build_image
|
||||
run_container "$MODE"
|
||||
|
||||
echo "[runner] Done."
|
||||
@@ -1,32 +0,0 @@
|
||||
FROM archlinux:latest
|
||||
|
||||
RUN sed -i 's|https://geo.mirror.pkgbuild.com|https://mirrors.tuna.tsinghua.edu.cn/archlinux|g' /etc/pacman.d/mirrorlist && \
|
||||
sed -i 's|https://mirror.rackspace.com/archlinux|https://mirrors.tuna.tsinghua.edu.cn/archlinux|g' /etc/pacman.d/mirrorlist && \
|
||||
echo 'Server = https://mirrors.tuna.tsinghua.edu.cn/archlinux/$repo/os/$arch' >> /etc/pacman.d/mirrorlist
|
||||
|
||||
RUN pacman -Syu --noconfirm && \
|
||||
pacman -S --noconfirm \
|
||||
python python-pip base-devel git curl wget sudo \
|
||||
which procps-ng shadow && \
|
||||
pacman -Scc --noconfirm
|
||||
|
||||
# Create test user
|
||||
RUN useradd -m -s /bin/bash testuser
|
||||
|
||||
# Python environment
|
||||
WORKDIR /app
|
||||
COPY tests/requirements.txt ./requirements.txt
|
||||
RUN python -m venv /opt/test-venv && \
|
||||
/opt/test-venv/bin/pip install -r requirements.txt
|
||||
|
||||
# Copy test suite
|
||||
COPY tests/ ./tests/
|
||||
COPY tests/entrypoint.sh ./entrypoint.sh
|
||||
RUN chmod +x entrypoint.sh
|
||||
|
||||
# Environment
|
||||
ENV PATH="/opt/test-venv/bin:$PATH"
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
ENV PYTHONDONTWRITEBYTECODE=1
|
||||
|
||||
ENTRYPOINT ["./entrypoint.sh"]
|
||||
@@ -1,175 +0,0 @@
|
||||
"""
|
||||
Root conftest for HoneyBiscuitWorkshop system tests.
|
||||
|
||||
Provides core fixtures for:
|
||||
- Locating the baker binary and examples
|
||||
- Running subprocess commands with output capture
|
||||
- Managing temporary working directories
|
||||
"""
|
||||
import os
|
||||
import subprocess
|
||||
import shutil
|
||||
import tempfile
|
||||
import pytest
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
"""Register custom markers."""
|
||||
config.addinivalue_line("markers", "prebake: prebake stage tests")
|
||||
config.addinivalue_line("markers", "bake: bake stage tests")
|
||||
config.addinivalue_line("markers", "finalize: finalize stage tests")
|
||||
config.addinivalue_line("markers", "llm: tests requiring LLM API")
|
||||
config.addinivalue_line("markers", "slow: slow running tests")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def baker_bin():
|
||||
"""Path to the workshop-baker binary."""
|
||||
path = os.environ.get("BAKER_BIN", "/workshop/workshop-baker")
|
||||
if not os.path.isfile(path):
|
||||
pytest.skip(f"Baker binary not found at {path}")
|
||||
if not os.access(path, os.X_OK):
|
||||
pytest.skip(f"Baker binary not executable at {path}")
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def examples_dir():
|
||||
"""Path to the examples directory (may be None)."""
|
||||
path = os.environ.get("EXAMPLES_DIR", "/workshop/examples")
|
||||
if not os.path.isdir(path):
|
||||
return None
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def llm_env():
|
||||
"""LLM environment variables dict (from llm.env)."""
|
||||
env = {}
|
||||
for var in [
|
||||
"OPENAI_BASE_URL",
|
||||
"OPENAI_API_KEY",
|
||||
"ANTHROPIC_BASE_URL",
|
||||
"ANTHROPIC_API_KEY",
|
||||
"MODEL",
|
||||
"REASONING_EFFORT",
|
||||
]:
|
||||
val = os.environ.get(var)
|
||||
if val:
|
||||
env[var] = val
|
||||
return env
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def work_dir():
|
||||
"""Create a temporary working directory for a test, cleaned up after."""
|
||||
d = tempfile.mkdtemp(prefix="hbw-test-")
|
||||
yield d
|
||||
shutil.rmtree(d, ignore_errors=True)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def run_cmd():
|
||||
"""Fixture that provides a command runner function.
|
||||
|
||||
Returns a function that runs a command and returns
|
||||
(stdout, stderr, returncode, success).
|
||||
"""
|
||||
|
||||
def _run(args, cwd=None, env=None, timeout=120):
|
||||
"""Run a command and capture output.
|
||||
|
||||
Args:
|
||||
args: Command as list of strings.
|
||||
cwd: Working directory.
|
||||
env: Environment variables (merged with os.environ).
|
||||
timeout: Timeout in seconds.
|
||||
|
||||
Returns:
|
||||
dict with keys: stdout, stderr, returncode, success
|
||||
"""
|
||||
full_env = {**os.environ}
|
||||
if env:
|
||||
full_env.update(env)
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
args,
|
||||
cwd=cwd,
|
||||
env=full_env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
return {
|
||||
"stdout": result.stdout,
|
||||
"stderr": result.stderr,
|
||||
"returncode": result.returncode,
|
||||
"success": result.returncode == 0,
|
||||
}
|
||||
except subprocess.TimeoutExpired:
|
||||
return {
|
||||
"stdout": "",
|
||||
"stderr": f"Command timed out after {timeout}s",
|
||||
"returncode": -1,
|
||||
"success": False,
|
||||
}
|
||||
except Exception as e:
|
||||
return {
|
||||
"stdout": "",
|
||||
"stderr": str(e),
|
||||
"returncode": -1,
|
||||
"success": False,
|
||||
}
|
||||
|
||||
return _run
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def run_baker(baker_bin, run_cmd):
|
||||
"""Fixture that provides a baker command runner.
|
||||
|
||||
Returns a function that runs workshop-baker with common flags.
|
||||
"""
|
||||
|
||||
def _run(subcommand, *args, cwd=None, env=None, timeout=120):
|
||||
"""Run workshop-baker with a subcommand.
|
||||
|
||||
Args:
|
||||
subcommand: Baker subcommand (prebake, bake, finalize).
|
||||
*args: Additional arguments to the subcommand.
|
||||
cwd: Working directory.
|
||||
env: Extra environment variables.
|
||||
timeout: Timeout in seconds.
|
||||
|
||||
Returns:
|
||||
dict with keys: stdout, stderr, returncode, success
|
||||
"""
|
||||
cmd = [
|
||||
baker_bin,
|
||||
"-u", "testuser",
|
||||
"-p", "test-pipeline",
|
||||
"-b", "test-build-001",
|
||||
subcommand,
|
||||
]
|
||||
cmd.extend(args)
|
||||
return run_cmd(cmd, cwd=cwd, env=env, timeout=timeout)
|
||||
|
||||
return _run
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fixtures_dir():
|
||||
"""Path to the test fixtures directory."""
|
||||
return os.path.join(os.path.dirname(__file__), "fixtures")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def bake_workspace():
|
||||
"""Create /workspace for bake tests (baker writes temp scripts there)."""
|
||||
existed = os.path.exists("/workspace")
|
||||
if not existed:
|
||||
os.makedirs("/workspace", exist_ok=True)
|
||||
yield
|
||||
if not existed:
|
||||
shutil.rmtree("/workspace", ignore_errors=True)
|
||||
@@ -1,36 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Load LLM environment if available
|
||||
if [ -f /workshop/llm.env ]; then
|
||||
set -a
|
||||
source /workshop/llm.env
|
||||
set +a
|
||||
fi
|
||||
|
||||
# Export test configuration
|
||||
export BAKER_BIN=/workshop/workshop-baker
|
||||
export EXAMPLES_DIR=/workshop/examples
|
||||
export TEST_MODE=${TEST_MODE:-auto}
|
||||
|
||||
if [ "$TEST_MODE" = "manual" ]; then
|
||||
echo "========================================"
|
||||
echo " HoneyBiscuitWorkshop System Tests"
|
||||
echo " Mode: MANUAL (interactive shell)"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
echo "Binary: $BAKER_BIN"
|
||||
echo "Examples: $EXAMPLES_DIR"
|
||||
echo ""
|
||||
echo "Run tests manually:"
|
||||
echo " cd /app && pytest tests/ -v"
|
||||
echo ""
|
||||
exec /bin/bash
|
||||
else
|
||||
echo "========================================"
|
||||
echo " HoneyBiscuitWorkshop System Tests"
|
||||
echo " Mode: AUTO (running pytest)"
|
||||
echo "========================================"
|
||||
cd /app
|
||||
exec python -m pytest tests/ -v --tb=short --junitxml=/results/report.xml "$@"
|
||||
fi
|
||||
Vendored
-6
@@ -1,6 +0,0 @@
|
||||
#!/bin/bash
|
||||
[ -f /dev/shm/bakeexport.{{ name }} ] && . /dev/shm/bakeexport.{{ name }} || true
|
||||
{{ condition }}
|
||||
{{ main }}
|
||||
{{ name }}
|
||||
{{ export }}
|
||||
Vendored
-25
@@ -1,25 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# @pipeline
|
||||
step_one() {
|
||||
echo "Step 1: Setup"
|
||||
mkdir -p /tmp/bake-test-output
|
||||
echo "step1-done" > /tmp/bake-test-output/step1.txt
|
||||
}
|
||||
|
||||
# @pipeline
|
||||
step_two() {
|
||||
echo "Step 2: Build"
|
||||
echo "step2-done" > /tmp/bake-test-output/step2.txt
|
||||
}
|
||||
|
||||
# @pipeline
|
||||
step_three() {
|
||||
echo "Step 3: Verify"
|
||||
if [ -f /tmp/bake-test-output/step1.txt ] && [ -f /tmp/bake-test-output/step2.txt ]; then
|
||||
echo "All steps completed successfully"
|
||||
else
|
||||
echo "ERROR: Missing step outputs"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
Vendored
-5
@@ -1,5 +0,0 @@
|
||||
#!/bin/bash
|
||||
echo "Hello from simple bake script"
|
||||
echo "Task ID: $HBW_TASKID"
|
||||
echo "Pipeline: $HBW_PIPELINE"
|
||||
exit 0
|
||||
Vendored
-22
@@ -1,22 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# @pipeline
|
||||
init_workspace() {
|
||||
echo "Initializing workspace"
|
||||
mkdir -p /tmp/bake-after-test
|
||||
echo "init" > /tmp/bake-after-test/status.txt
|
||||
}
|
||||
|
||||
# @pipeline
|
||||
# @after(init_workspace)
|
||||
fetch_source() {
|
||||
echo "Fetching source (depends on init_workspace)"
|
||||
echo "fetched" > /tmp/bake-after-test/source.txt
|
||||
}
|
||||
|
||||
# @pipeline
|
||||
# @after(fetch_source)
|
||||
build_project() {
|
||||
echo "Building project (depends on fetch_source)"
|
||||
echo "built" > /tmp/bake-after-test/build.txt
|
||||
}
|
||||
-4
@@ -1,4 +0,0 @@
|
||||
#!/bin/bash
|
||||
echo "EARLY_HOOK_EXECUTED"
|
||||
echo "Working directory: $(pwd)"
|
||||
echo "User: $(whoami)"
|
||||
Vendored
-3
@@ -1,3 +0,0 @@
|
||||
#!/bin/bash
|
||||
echo "LATE_HOOK_EXECUTED"
|
||||
echo "Working directory: $(pwd)"
|
||||
Vendored
-6
@@ -1,6 +0,0 @@
|
||||
version: "0.0.1"
|
||||
plugin: {}
|
||||
notification: {}
|
||||
artifact: []
|
||||
cleanup:
|
||||
policy: "auto"
|
||||
-13
@@ -1,13 +0,0 @@
|
||||
version: "0.0.1"
|
||||
plugin: {}
|
||||
notification: {}
|
||||
artifact: []
|
||||
cleanup:
|
||||
policy: "auto"
|
||||
hooks:
|
||||
early:
|
||||
- name: "finalize-early"
|
||||
command: "echo 'finalize early hook'"
|
||||
late:
|
||||
- name: "finalize-late"
|
||||
command: "echo 'finalize late hook'"
|
||||
-8
@@ -1,8 +0,0 @@
|
||||
version: "1.0"
|
||||
environment:
|
||||
builder: "baremetal"
|
||||
bootstrap:
|
||||
user: "vulcan"
|
||||
workspace:
|
||||
path: "/home/vulcan/workspace"
|
||||
fallback: true
|
||||
Vendored
-8
@@ -1,8 +0,0 @@
|
||||
version: "1.0"
|
||||
environment:
|
||||
builder: "baremetal"
|
||||
bootstrap:
|
||||
user: "vulcan"
|
||||
workspace:
|
||||
path: "/home/vulcan/workspace"
|
||||
fallback: true
|
||||
-15
@@ -1,15 +0,0 @@
|
||||
version: "1.0"
|
||||
environment:
|
||||
builder: "baremetal"
|
||||
bootstrap:
|
||||
user: "vulcan"
|
||||
workspace:
|
||||
path: "/home/vulcan/workspace"
|
||||
fallback: true
|
||||
hooks:
|
||||
early:
|
||||
- name: "test-early-hook"
|
||||
command: "echo 'early hook executed'"
|
||||
late:
|
||||
- name: "test-late-hook"
|
||||
command: "echo 'late hook executed'"
|
||||
-14
@@ -1,14 +0,0 @@
|
||||
version: "1.0"
|
||||
environment:
|
||||
builder: "baremetal"
|
||||
bootstrap:
|
||||
user: "vulcan"
|
||||
workspace:
|
||||
path: "/home/vulcan/workspace"
|
||||
fallback: true
|
||||
dependencies:
|
||||
system:
|
||||
pacman:
|
||||
packages:
|
||||
- "curl"
|
||||
- "wget"
|
||||
@@ -1,12 +0,0 @@
|
||||
"""
|
||||
LLM-based log judgment system for system tests.
|
||||
|
||||
Uses LLM APIs to evaluate non-deterministic test outputs (logs, stderr, etc.)
|
||||
where simple string matching is insufficient.
|
||||
"""
|
||||
|
||||
from .client import LLMClient
|
||||
from .cache import LLMCache
|
||||
from .judge import judge_log, JudgmentResult
|
||||
|
||||
__all__ = ["LLMClient", "LLMCache", "judge_log", "JudgmentResult"]
|
||||
@@ -1,56 +0,0 @@
|
||||
"""
|
||||
Disk-based cache for LLM API responses.
|
||||
|
||||
Uses SHA256 of (prompt + model + system_prompt) as cache key.
|
||||
Cache files stored as JSON in ``.cache/`` directory.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class LLMCache:
|
||||
"""SHA256-based disk cache for LLM API responses."""
|
||||
|
||||
def __init__(self, cache_dir: str | None = None):
|
||||
if cache_dir is None:
|
||||
import os
|
||||
|
||||
cache_dir = os.path.join(os.path.dirname(__file__), ".cache")
|
||||
self.cache_dir = Path(cache_dir)
|
||||
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def _make_key(self, prompt: str, model: str, system_prompt: str = "") -> str:
|
||||
"""Create deterministic cache key from request parameters."""
|
||||
data = json.dumps(
|
||||
{"prompt": prompt, "model": model, "system": system_prompt},
|
||||
sort_keys=True,
|
||||
)
|
||||
return hashlib.sha256(data.encode()).hexdigest()
|
||||
|
||||
def get(self, prompt: str, model: str, system_prompt: str = "") -> dict | None:
|
||||
"""Retrieve cached response if exists."""
|
||||
key = self._make_key(prompt, model, system_prompt)
|
||||
path = self.cache_dir / f"{key}.json"
|
||||
if path.exists():
|
||||
try:
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
except (json.JSONDecodeError, IOError):
|
||||
return None
|
||||
return None
|
||||
|
||||
def set(
|
||||
self, prompt: str, model: str, system_prompt: str, response: dict
|
||||
) -> None:
|
||||
"""Store response in cache."""
|
||||
key = self._make_key(prompt, model, system_prompt)
|
||||
path = self.cache_dir / f"{key}.json"
|
||||
with open(path, "w") as f:
|
||||
json.dump(response, f, ensure_ascii=False)
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Clear all cached responses."""
|
||||
for f in self.cache_dir.glob("*.json"):
|
||||
f.unlink()
|
||||
@@ -1,187 +0,0 @@
|
||||
"""
|
||||
LLM client supporting both Anthropic and OpenAI APIs.
|
||||
|
||||
Prefers Anthropic format (per project convention).
|
||||
Falls back to OpenAI format if Anthropic is unavailable.
|
||||
"""
|
||||
|
||||
import os
|
||||
import logging
|
||||
from .cache import LLMCache
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LLMClient:
|
||||
"""LLM client with caching support. Prefers Anthropic API."""
|
||||
|
||||
def __init__(self, cache: LLMCache | None = None):
|
||||
self.cache = cache or LLMCache()
|
||||
self._anthropic_client = None
|
||||
self._openai_client = None
|
||||
self._init_clients()
|
||||
|
||||
def _init_clients(self):
|
||||
"""Initialize API clients based on available environment variables."""
|
||||
# Anthropic (preferred)
|
||||
anthropic_key = os.environ.get("ANTHROPIC_API_KEY")
|
||||
anthropic_base = os.environ.get("ANTHROPIC_BASE_URL")
|
||||
if anthropic_key:
|
||||
try:
|
||||
import anthropic
|
||||
|
||||
kwargs: dict = {"api_key": anthropic_key}
|
||||
if anthropic_base:
|
||||
kwargs["base_url"] = anthropic_base
|
||||
self._anthropic_client = anthropic.Anthropic(**kwargs)
|
||||
logger.info("Anthropic client initialized")
|
||||
except ImportError:
|
||||
logger.warning("anthropic package not installed")
|
||||
|
||||
# OpenAI (fallback)
|
||||
openai_key = os.environ.get("OPENAI_API_KEY")
|
||||
openai_base = os.environ.get("OPENAI_BASE_URL")
|
||||
if openai_key:
|
||||
try:
|
||||
import openai
|
||||
|
||||
kwargs = {"api_key": openai_key}
|
||||
if openai_base:
|
||||
kwargs["base_url"] = openai_base
|
||||
self._openai_client = openai.OpenAI(**kwargs)
|
||||
logger.info("OpenAI client initialized")
|
||||
except ImportError:
|
||||
logger.warning("openai package not installed")
|
||||
|
||||
@property
|
||||
def available(self) -> bool:
|
||||
"""Check if any LLM client is available."""
|
||||
return (
|
||||
self._anthropic_client is not None or self._openai_client is not None
|
||||
)
|
||||
|
||||
def complete(
|
||||
self, prompt: str, system_prompt: str = "", max_tokens: int = 2000
|
||||
) -> str:
|
||||
"""Send a completion request, with caching.
|
||||
|
||||
Args:
|
||||
prompt: User message.
|
||||
system_prompt: System message (sets role / persona).
|
||||
max_tokens: Max response tokens.
|
||||
|
||||
Returns:
|
||||
LLM response text.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If no client is available.
|
||||
"""
|
||||
model = os.environ.get("MODEL", "deepseek-v4-flash")
|
||||
|
||||
# Check cache
|
||||
cached = self.cache.get(prompt, model, system_prompt)
|
||||
if cached:
|
||||
logger.info("Cache hit for LLM request")
|
||||
return cached["content"]
|
||||
|
||||
# Try Anthropic first
|
||||
if self._anthropic_client:
|
||||
content = self._call_anthropic(
|
||||
prompt, system_prompt, model, max_tokens
|
||||
)
|
||||
elif self._openai_client:
|
||||
content = self._call_openai(prompt, system_prompt, model, max_tokens)
|
||||
else:
|
||||
raise RuntimeError(
|
||||
"No LLM client available. "
|
||||
"Set ANTHROPIC_API_KEY or OPENAI_API_KEY."
|
||||
)
|
||||
|
||||
# Cache the response
|
||||
self.cache.set(prompt, model, system_prompt, {"content": content})
|
||||
return content
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Private helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _call_anthropic(
|
||||
self, prompt: str, system_prompt: str, model: str, max_tokens: int
|
||||
) -> str:
|
||||
"""Call Anthropic API."""
|
||||
kwargs: dict = {
|
||||
"model": model,
|
||||
"max_tokens": max_tokens,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
}
|
||||
if system_prompt:
|
||||
kwargs["system"] = system_prompt
|
||||
|
||||
# Only add reasoning_effort if the model supports it
|
||||
reasoning_effort = os.environ.get("REASONING_EFFORT")
|
||||
if reasoning_effort:
|
||||
try:
|
||||
kwargs["reasoning_effort"] = reasoning_effort
|
||||
response = self._anthropic_client.messages.create(**kwargs)
|
||||
except Exception as e:
|
||||
if (
|
||||
"reasoning_effort" in str(e).lower()
|
||||
or "unexpected" in str(e).lower()
|
||||
):
|
||||
del kwargs["reasoning_effort"]
|
||||
response = self._anthropic_client.messages.create(**kwargs)
|
||||
else:
|
||||
raise
|
||||
else:
|
||||
response = self._anthropic_client.messages.create(**kwargs)
|
||||
|
||||
# Extract text from response content blocks
|
||||
if hasattr(response, "content") and response.content:
|
||||
block = response.content[0]
|
||||
if hasattr(block, "text"):
|
||||
return block.text
|
||||
if hasattr(block, "type") and block.type == "thinking":
|
||||
for b in response.content:
|
||||
if hasattr(b, "text"):
|
||||
return b.text
|
||||
return str(block)
|
||||
return str(response)
|
||||
|
||||
def _call_openai(
|
||||
self, prompt: str, system_prompt: str, model: str, max_tokens: int
|
||||
) -> str:
|
||||
"""Call OpenAI-compatible API."""
|
||||
messages: list[dict] = []
|
||||
if system_prompt:
|
||||
messages.append({"role": "system", "content": system_prompt})
|
||||
messages.append({"role": "user", "content": prompt})
|
||||
|
||||
kwargs: dict = {
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"max_tokens": max_tokens,
|
||||
}
|
||||
|
||||
# Only add reasoning_effort if supported
|
||||
reasoning_effort = os.environ.get("REASONING_EFFORT")
|
||||
if reasoning_effort:
|
||||
try:
|
||||
kwargs["reasoning_effort"] = reasoning_effort
|
||||
response = self._openai_client.chat.completions.create(
|
||||
**kwargs
|
||||
)
|
||||
except Exception as e:
|
||||
if (
|
||||
"reasoning_effort" in str(e).lower()
|
||||
or "unexpected" in str(e).lower()
|
||||
):
|
||||
del kwargs["reasoning_effort"]
|
||||
response = self._openai_client.chat.completions.create(
|
||||
**kwargs
|
||||
)
|
||||
else:
|
||||
raise
|
||||
else:
|
||||
response = self._openai_client.chat.completions.create(**kwargs)
|
||||
|
||||
return response.choices[0].message.content
|
||||
@@ -1,149 +0,0 @@
|
||||
"""
|
||||
LLM-based log judgment for system tests.
|
||||
|
||||
Evaluates logs against expected behavior using LLM semantic analysis.
|
||||
Returns structured pass/fail results with reasoning.
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from .client import LLMClient
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
JUDGE_SYSTEM_PROMPT = """\
|
||||
You are a test result judge for a CI/CD system called HoneyBiscuitWorkshop.
|
||||
|
||||
Your job: Evaluate whether actual command output/logs match expected behavior.
|
||||
|
||||
RULES:
|
||||
1. You MUST respond with valid JSON in this exact format:
|
||||
{"verdict": "pass" | "fail", "reason": "brief explanation"}
|
||||
2. "pass" = the output confirms the expected behavior occurred
|
||||
3. "fail" = the output shows the expected behavior did NOT occur, or shows an error
|
||||
4. Ignore irrelevant noise (timestamps, debug lines, unrelated warnings)
|
||||
5. Focus ONLY on whether the specific expected behavior is confirmed or contradicted
|
||||
6. Be strict: if the evidence is ambiguous, verdict is "fail"
|
||||
7. Do NOT include any text outside the JSON object"""
|
||||
|
||||
|
||||
@dataclass
|
||||
class JudgmentResult:
|
||||
"""Result of an LLM log judgment."""
|
||||
|
||||
verdict: str # "pass" or "fail"
|
||||
reason: str
|
||||
raw_response: str
|
||||
|
||||
@property
|
||||
def passed(self) -> bool:
|
||||
return self.verdict == "pass"
|
||||
|
||||
|
||||
def _extract_relevant_logs(full_log: str, max_chars: int = 4000) -> str:
|
||||
"""Extract relevant portions from logs, removing noise.
|
||||
|
||||
Keeps the last *max_chars* characters (most recent / relevant).
|
||||
Removes common noise patterns.
|
||||
"""
|
||||
lines = full_log.strip().split("\n")
|
||||
|
||||
noise_patterns = [
|
||||
r"^\d{4}-\d{2}-\d{2}T",
|
||||
r"^DEBUG ",
|
||||
r"^TRACE ",
|
||||
]
|
||||
filtered = [
|
||||
line
|
||||
for line in lines
|
||||
if not any(re.match(p, line) for p in noise_patterns)
|
||||
]
|
||||
|
||||
text = "\n".join(filtered)
|
||||
if len(text) > max_chars:
|
||||
text = "...\n" + text[-max_chars:]
|
||||
return text
|
||||
|
||||
|
||||
def judge_log(
|
||||
operation: str,
|
||||
expected_behavior: str,
|
||||
actual_log: str,
|
||||
client: LLMClient | None = None,
|
||||
) -> JudgmentResult:
|
||||
"""Judge whether actual logs match expected behavior.
|
||||
|
||||
Args:
|
||||
operation: Description of what was attempted
|
||||
(e.g. ``"prebake bootstrap stage"``).
|
||||
expected_behavior: What should have happened
|
||||
(e.g. ``"user vulcan was created"``).
|
||||
actual_log: The actual stdout / stderr output.
|
||||
client: ``LLMClient`` instance (creates default if ``None``).
|
||||
|
||||
Returns:
|
||||
``JudgmentResult`` with verdict and reason.
|
||||
"""
|
||||
if client is None:
|
||||
client = LLMClient()
|
||||
|
||||
if not client.available:
|
||||
logger.warning("No LLM client available, returning fail verdict")
|
||||
return JudgmentResult(
|
||||
verdict="fail",
|
||||
reason="No LLM client available for judgment",
|
||||
raw_response="",
|
||||
)
|
||||
|
||||
relevant_log = _extract_relevant_logs(actual_log)
|
||||
|
||||
prompt = f"""\
|
||||
## Operation
|
||||
{operation}
|
||||
|
||||
## Expected Behavior
|
||||
{expected_behavior}
|
||||
|
||||
## Actual Output (relevant excerpts)
|
||||
```
|
||||
{relevant_log}
|
||||
```
|
||||
|
||||
Evaluate: Does the actual output confirm the expected behavior occurred?
|
||||
Respond with JSON only."""
|
||||
|
||||
try:
|
||||
response = client.complete(
|
||||
prompt=prompt,
|
||||
system_prompt=JUDGE_SYSTEM_PROMPT,
|
||||
max_tokens=500,
|
||||
)
|
||||
|
||||
# Parse JSON response — handle potential markdown code blocks
|
||||
text = response.strip()
|
||||
if text.startswith("```"):
|
||||
lines = text.split("\n")
|
||||
text = "\n".join(lines[1:-1]) if len(lines) > 2 else text
|
||||
|
||||
result = json.loads(text)
|
||||
return JudgmentResult(
|
||||
verdict=result.get("verdict", "fail"),
|
||||
reason=result.get("reason", "No reason provided"),
|
||||
raw_response=response,
|
||||
)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"Failed to parse LLM response as JSON: {e}")
|
||||
return JudgmentResult(
|
||||
verdict="fail",
|
||||
reason=f"LLM response was not valid JSON: {response[:200]}",
|
||||
raw_response=response,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"LLM judgment failed: {e}")
|
||||
return JudgmentResult(
|
||||
verdict="fail",
|
||||
reason=f"LLM call failed: {str(e)}",
|
||||
raw_response="",
|
||||
)
|
||||
@@ -1,34 +0,0 @@
|
||||
"""
|
||||
Environment probes for verifying system state in workshop-baker tests.
|
||||
|
||||
Each probe is a standalone function that can be called from any test.
|
||||
Probes return bool or raise AssertionError with descriptive message.
|
||||
"""
|
||||
|
||||
from .user import user_exists, user_uid, user_gid, user_in_group, user_shell
|
||||
from .file import (
|
||||
file_exists,
|
||||
dir_exists,
|
||||
file_permissions,
|
||||
file_contains,
|
||||
file_owned_by,
|
||||
)
|
||||
from .package import package_installed, package_version
|
||||
from .process import process_running, process_count
|
||||
|
||||
__all__ = [
|
||||
"user_exists",
|
||||
"user_uid",
|
||||
"user_gid",
|
||||
"user_in_group",
|
||||
"user_shell",
|
||||
"file_exists",
|
||||
"dir_exists",
|
||||
"file_permissions",
|
||||
"file_contains",
|
||||
"file_owned_by",
|
||||
"package_installed",
|
||||
"package_version",
|
||||
"process_running",
|
||||
"process_count",
|
||||
]
|
||||
@@ -1,43 +0,0 @@
|
||||
"""File and directory existence / permission probes."""
|
||||
|
||||
import os
|
||||
import stat
|
||||
import grp
|
||||
import pwd
|
||||
|
||||
|
||||
def file_exists(path: str) -> bool:
|
||||
"""Check if *path* exists and is a regular file."""
|
||||
return os.path.isfile(path)
|
||||
|
||||
|
||||
def dir_exists(path: str) -> bool:
|
||||
"""Check if *path* exists and is a directory."""
|
||||
return os.path.isdir(path)
|
||||
|
||||
|
||||
def file_permissions(path: str) -> int:
|
||||
"""Return the permission mode of *path* as an octal int (e.g. ``0o755``).
|
||||
|
||||
Raises ``FileNotFoundError`` if *path* doesn't exist.
|
||||
"""
|
||||
return stat.S_IMODE(os.stat(path).st_mode)
|
||||
|
||||
|
||||
def file_contains(path: str, text: str) -> bool:
|
||||
"""Check if a file contains the specified text."""
|
||||
with open(path, "r", errors="replace") as f:
|
||||
return text in f.read()
|
||||
|
||||
|
||||
def file_owned_by(path: str, username: str) -> bool:
|
||||
"""Check if a file is owned by the specified user."""
|
||||
st = os.stat(path)
|
||||
owner = pwd.getpwuid(st.st_uid).pw_name
|
||||
return owner == username
|
||||
|
||||
|
||||
def file_group(path: str) -> str:
|
||||
"""Get the group owner of a file."""
|
||||
st = os.stat(path)
|
||||
return grp.getgrgid(st.st_gid).gr_name
|
||||
@@ -1,34 +0,0 @@
|
||||
"""Package installation probes for Arch Linux (pacman)."""
|
||||
|
||||
import subprocess
|
||||
|
||||
|
||||
def package_installed(package_name: str) -> bool:
|
||||
"""Check if a pacman package is installed."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["pacman", "-Q", package_name],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return result.returncode == 0
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
|
||||
def package_version(package_name: str) -> str | None:
|
||||
"""Get installed version of a package, or ``None`` if not installed."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["pacman", "-Q", package_name],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
# Output format: "package_name version"
|
||||
parts = result.stdout.strip().split()
|
||||
if len(parts) >= 2:
|
||||
return parts[1]
|
||||
return None
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
@@ -1,44 +0,0 @@
|
||||
"""Process existence probes."""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
|
||||
def process_running(process_name: str) -> bool:
|
||||
"""Check if a process with the given name is running."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["pgrep", "-x", process_name],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return result.returncode == 0
|
||||
except FileNotFoundError:
|
||||
# Fall back to /proc scan if pgrep not available
|
||||
try:
|
||||
for pid_dir in os.listdir("/proc"):
|
||||
if pid_dir.isdigit():
|
||||
try:
|
||||
with open(f"/proc/{pid_dir}/comm") as f:
|
||||
if f.read().strip() == process_name:
|
||||
return True
|
||||
except (IOError, OSError):
|
||||
continue
|
||||
except OSError:
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def process_count(process_name: str) -> int:
|
||||
"""Count processes with the given name."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["pgrep", "-c", "-x", process_name],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
return int(result.stdout.strip())
|
||||
return 0
|
||||
except (FileNotFoundError, ValueError):
|
||||
return 0
|
||||
@@ -1,37 +0,0 @@
|
||||
"""User existence and attribute probes."""
|
||||
|
||||
import pwd
|
||||
import grp
|
||||
|
||||
|
||||
def user_exists(username: str) -> bool:
|
||||
"""Check if a system user exists."""
|
||||
try:
|
||||
pwd.getpwnam(username)
|
||||
return True
|
||||
except KeyError:
|
||||
return False
|
||||
|
||||
|
||||
def user_uid(username: str) -> int:
|
||||
"""Get the UID of a user. Raises KeyError if user doesn't exist."""
|
||||
return pwd.getpwnam(username).pw_uid
|
||||
|
||||
|
||||
def user_gid(username: str) -> int:
|
||||
"""Get the primary GID of a user. Raises KeyError if user doesn't exist."""
|
||||
return pwd.getpwnam(username).pw_gid
|
||||
|
||||
|
||||
def user_in_group(username: str, groupname: str) -> bool:
|
||||
"""Check if a user is a member of a group."""
|
||||
try:
|
||||
group = grp.getgrnam(groupname)
|
||||
return username in group.gr_mem
|
||||
except KeyError:
|
||||
return False
|
||||
|
||||
|
||||
def user_shell(username: str) -> str:
|
||||
"""Get the login shell of a user. Raises KeyError if user doesn't exist."""
|
||||
return pwd.getpwnam(username).pw_shell
|
||||
@@ -1,6 +0,0 @@
|
||||
pytest>=8.0.0
|
||||
pytest-timeout>=2.2.0
|
||||
pytest-xdist>=3.5.0
|
||||
anthropic>=0.40.0,<0.90.0
|
||||
openai>=1.50.0,<2.0.0
|
||||
pyyaml>=6.0
|
||||
@@ -1,129 +0,0 @@
|
||||
import os
|
||||
import pytest
|
||||
|
||||
|
||||
FIXTURES = os.path.join(os.path.dirname(__file__), "fixtures", "bake")
|
||||
|
||||
|
||||
@pytest.mark.bake
|
||||
class TestBakeSimpleScript:
|
||||
|
||||
def test_simple_script_runs(self, run_baker, bake_workspace, work_dir):
|
||||
prebake_cfg = os.path.join(
|
||||
os.path.dirname(__file__), "fixtures", "prebake", "minimal.yml"
|
||||
)
|
||||
script = os.path.join(FIXTURES, "simple.sh")
|
||||
bake_base = os.path.join(FIXTURES, "bake_base.sh")
|
||||
|
||||
result = run_baker(
|
||||
"bake", script,
|
||||
"--bake-base", bake_base,
|
||||
"--prebake", prebake_cfg,
|
||||
cwd=work_dir,
|
||||
timeout=60,
|
||||
)
|
||||
assert result["success"], f"Bake failed: {result['stderr']}"
|
||||
assert "Hello from simple bake script" in result["stdout"]
|
||||
|
||||
def test_simple_script_sets_env_vars(self, run_baker, bake_workspace, work_dir):
|
||||
prebake_cfg = os.path.join(
|
||||
os.path.dirname(__file__), "fixtures", "prebake", "minimal.yml"
|
||||
)
|
||||
script = os.path.join(FIXTURES, "simple.sh")
|
||||
bake_base = os.path.join(FIXTURES, "bake_base.sh")
|
||||
|
||||
result = run_baker(
|
||||
"bake", script,
|
||||
"--bake-base", bake_base,
|
||||
"--prebake", prebake_cfg,
|
||||
cwd=work_dir,
|
||||
timeout=60,
|
||||
)
|
||||
assert result["success"]
|
||||
assert "test-pipeline" in result["stdout"]
|
||||
|
||||
|
||||
@pytest.mark.bake
|
||||
class TestBakePipeline:
|
||||
|
||||
def test_pipeline_functions_execute(self, run_baker, bake_workspace, work_dir):
|
||||
prebake_cfg = os.path.join(
|
||||
os.path.dirname(__file__), "fixtures", "prebake", "minimal.yml"
|
||||
)
|
||||
script = os.path.join(FIXTURES, "pipeline.sh")
|
||||
bake_base = os.path.join(FIXTURES, "bake_base.sh")
|
||||
|
||||
result = run_baker(
|
||||
"bake", script,
|
||||
"--bake-base", bake_base,
|
||||
"--prebake", prebake_cfg,
|
||||
cwd=work_dir,
|
||||
timeout=60,
|
||||
)
|
||||
assert result["success"], f"Pipeline bake failed: {result['stderr']}"
|
||||
assert os.path.exists("/tmp/bake-test-output/step1.txt")
|
||||
assert os.path.exists("/tmp/bake-test-output/step2.txt")
|
||||
|
||||
def test_pipeline_step_outputs_correct(self, run_baker, bake_workspace, work_dir):
|
||||
prebake_cfg = os.path.join(
|
||||
os.path.dirname(__file__), "fixtures", "prebake", "minimal.yml"
|
||||
)
|
||||
script = os.path.join(FIXTURES, "pipeline.sh")
|
||||
bake_base = os.path.join(FIXTURES, "bake_base.sh")
|
||||
|
||||
result = run_baker(
|
||||
"bake", script,
|
||||
"--bake-base", bake_base,
|
||||
"--prebake", prebake_cfg,
|
||||
cwd=work_dir,
|
||||
timeout=60,
|
||||
)
|
||||
assert result["success"]
|
||||
with open("/tmp/bake-test-output/step1.txt") as f:
|
||||
assert f.read().strip() == "step1-done"
|
||||
with open("/tmp/bake-test-output/step2.txt") as f:
|
||||
assert f.read().strip() == "step2-done"
|
||||
|
||||
|
||||
@pytest.mark.bake
|
||||
class TestBakeAfterDependencies:
|
||||
|
||||
def test_after_ordering(self, run_baker, bake_workspace, work_dir):
|
||||
prebake_cfg = os.path.join(
|
||||
os.path.dirname(__file__), "fixtures", "prebake", "minimal.yml"
|
||||
)
|
||||
script = os.path.join(FIXTURES, "with_after.sh")
|
||||
bake_base = os.path.join(FIXTURES, "bake_base.sh")
|
||||
|
||||
result = run_baker(
|
||||
"bake", script,
|
||||
"--bake-base", bake_base,
|
||||
"--prebake", prebake_cfg,
|
||||
cwd=work_dir,
|
||||
timeout=60,
|
||||
)
|
||||
assert result["success"], f"@after bake failed: {result['stderr']}"
|
||||
assert os.path.exists("/tmp/bake-after-test/status.txt")
|
||||
assert os.path.exists("/tmp/bake-after-test/source.txt")
|
||||
assert os.path.exists("/tmp/bake-after-test/build.txt")
|
||||
|
||||
|
||||
@pytest.mark.bake
|
||||
class TestBakeStatusFiles:
|
||||
|
||||
def test_status_written_after_bake(self, run_baker, bake_workspace, work_dir):
|
||||
prebake_cfg = os.path.join(
|
||||
os.path.dirname(__file__), "fixtures", "prebake", "minimal.yml"
|
||||
)
|
||||
script = os.path.join(FIXTURES, "simple.sh")
|
||||
bake_base = os.path.join(FIXTURES, "bake_base.sh")
|
||||
|
||||
result = run_baker(
|
||||
"bake", script,
|
||||
"--bake-base", bake_base,
|
||||
"--prebake", prebake_cfg,
|
||||
cwd=work_dir,
|
||||
timeout=60,
|
||||
)
|
||||
assert result["success"]
|
||||
assert os.path.exists("/tmp/.hbwstatus")
|
||||
@@ -1,77 +0,0 @@
|
||||
import os
|
||||
import pytest
|
||||
|
||||
from probes import file
|
||||
from llm import judge_log, LLMClient
|
||||
|
||||
|
||||
FIXTURES = os.path.join(os.path.dirname(__file__), "fixtures", "finalize")
|
||||
|
||||
|
||||
@pytest.mark.finalize
|
||||
class TestFinalizeMinimal:
|
||||
|
||||
def test_minimal_finalize_succeeds(self, run_baker, work_dir):
|
||||
config = os.path.join(FIXTURES, "minimal.yml")
|
||||
result = run_baker("finalize", config, cwd=work_dir, timeout=60)
|
||||
assert result["success"], f"Minimal finalize failed: {result['stderr']}"
|
||||
|
||||
|
||||
@pytest.mark.finalize
|
||||
class TestFinalizeHooks:
|
||||
|
||||
def test_early_hook_executes(self, run_baker, work_dir):
|
||||
config = os.path.join(FIXTURES, "with_hooks.yml")
|
||||
result = run_baker("finalize", config, cwd=work_dir, timeout=60)
|
||||
assert result["success"]
|
||||
assert "finalize early hook" in result["stdout"] or "finalize early hook" in result["stderr"]
|
||||
|
||||
def test_late_hook_executes(self, run_baker, work_dir):
|
||||
config = os.path.join(FIXTURES, "with_hooks.yml")
|
||||
result = run_baker("finalize", config, cwd=work_dir, timeout=60)
|
||||
assert result["success"]
|
||||
assert "finalize late hook" in result["stdout"] or "finalize late hook" in result["stderr"]
|
||||
|
||||
|
||||
@pytest.mark.finalize
|
||||
class TestFinalizeStatus:
|
||||
|
||||
def test_finalize_writes_status(self, run_baker, work_dir):
|
||||
config = os.path.join(FIXTURES, "minimal.yml")
|
||||
result = run_baker("finalize", config, cwd=work_dir, timeout=60)
|
||||
assert result["success"]
|
||||
assert os.path.exists("/tmp/.hbwstatus")
|
||||
|
||||
|
||||
@pytest.mark.finalize
|
||||
@pytest.mark.llm
|
||||
class TestFinalizeWithLLM:
|
||||
|
||||
def test_finalize_hooks_via_llm(self, run_baker, llm_env, work_dir):
|
||||
if not llm_env:
|
||||
pytest.skip("LLM environment not configured")
|
||||
|
||||
config = os.path.join(FIXTURES, "with_hooks.yml")
|
||||
result = run_baker("finalize", config, cwd=work_dir, timeout=60)
|
||||
|
||||
combined_log = result["stdout"] + "\n" + result["stderr"]
|
||||
|
||||
judgment = judge_log(
|
||||
operation="finalize with early and late hooks",
|
||||
expected_behavior="Both early and late hooks executed successfully, "
|
||||
"printing 'finalize early hook' and 'finalize late hook'",
|
||||
actual_log=combined_log,
|
||||
)
|
||||
assert judgment.passed, f"LLM judgment failed: {judgment.reason}"
|
||||
|
||||
|
||||
@pytest.mark.finalize
|
||||
class TestFinalizePlugins:
|
||||
|
||||
@pytest.mark.xfail(reason="Plugin download requires network/git access", strict=False)
|
||||
def test_plugin_download(self, run_baker, work_dir):
|
||||
pass
|
||||
|
||||
@pytest.mark.xfail(reason="Artifact stage not yet implemented", strict=False)
|
||||
def test_artifact_collection(self, run_baker, work_dir):
|
||||
pass
|
||||
@@ -1,94 +0,0 @@
|
||||
import os
|
||||
import pytest
|
||||
|
||||
from probes import user, file, package
|
||||
|
||||
|
||||
FIXTURES = os.path.join(os.path.dirname(__file__), "fixtures", "prebake")
|
||||
|
||||
|
||||
@pytest.mark.prebake
|
||||
class TestPrebakeBootstrap:
|
||||
|
||||
def test_bootstrap_creates_vulcan_user(self, run_baker, work_dir):
|
||||
config = os.path.join(FIXTURES, "bootstrap_only.yml")
|
||||
result = run_baker("prebake", config, cwd=work_dir, timeout=60)
|
||||
assert result["success"], f"prebake failed: {result['stderr']}"
|
||||
assert user.user_exists("vulcan")
|
||||
|
||||
def test_bootstrap_creates_workspace(self, run_baker, work_dir):
|
||||
config = os.path.join(FIXTURES, "bootstrap_only.yml")
|
||||
result = run_baker("prebake", config, cwd=work_dir, timeout=60)
|
||||
assert result["success"]
|
||||
assert file.dir_exists("/home/vulcan/workspace") or file.dir_exists("/workspace")
|
||||
|
||||
def test_bootstrap_creates_symlink(self, run_baker, work_dir):
|
||||
config = os.path.join(FIXTURES, "bootstrap_only.yml")
|
||||
result = run_baker("prebake", config, cwd=work_dir, timeout=60)
|
||||
assert result["success"]
|
||||
assert os.path.islink("/workspace")
|
||||
|
||||
@pytest.mark.xfail(reason="Sudoers not created without package manager config", strict=False)
|
||||
def test_bootstrap_creates_sudoers(self, run_baker, work_dir):
|
||||
config = os.path.join(FIXTURES, "bootstrap_only.yml")
|
||||
result = run_baker("prebake", config, cwd=work_dir, timeout=60)
|
||||
assert result["success"]
|
||||
assert file.file_exists("/etc/sudoers.d/workshop")
|
||||
|
||||
def test_bootstrap_generates_bootstrap_script(self, run_baker, work_dir):
|
||||
config = os.path.join(FIXTURES, "bootstrap_only.yml")
|
||||
result = run_baker("prebake", config, cwd=work_dir, timeout=60)
|
||||
assert result["success"]
|
||||
assert file.file_exists("/bootstrap.sh")
|
||||
|
||||
def test_bootstrap_script_permissions(self, run_baker, work_dir):
|
||||
config = os.path.join(FIXTURES, "bootstrap_only.yml")
|
||||
result = run_baker("prebake", config, cwd=work_dir, timeout=60)
|
||||
assert result["success"]
|
||||
mode = file.file_permissions("/bootstrap.sh")
|
||||
assert mode & 0o111, f"/bootstrap.sh not executable, mode: {oct(mode)}"
|
||||
|
||||
|
||||
@pytest.mark.prebake
|
||||
class TestPrebakeDeps:
|
||||
|
||||
@pytest.mark.xfail(reason="Package installation may fail in test environment")
|
||||
def test_pacman_packages_installed(self, run_baker, work_dir):
|
||||
config = os.path.join(FIXTURES, "with_pacman.yml")
|
||||
result = run_baker("prebake", config, cwd=work_dir, timeout=120)
|
||||
assert result["success"], f"prebake failed: {result['stderr']}"
|
||||
assert package.package_installed("curl")
|
||||
assert package.package_installed("wget")
|
||||
|
||||
|
||||
@pytest.mark.prebake
|
||||
class TestPrebakeHooks:
|
||||
|
||||
@pytest.mark.xfail(reason="Late hook fails after privilege drop to vulcan (Permission denied)", strict=False)
|
||||
def test_early_hook_executes(self, run_baker, work_dir):
|
||||
config = os.path.join(FIXTURES, "with_hooks.yml")
|
||||
result = run_baker("prebake", config, cwd=work_dir, timeout=60)
|
||||
assert result["success"], f"prebake with hooks failed: stdout={result['stdout']!r} stderr={result['stderr']!r}"
|
||||
assert "early hook executed" in result["stdout"] or "early hook executed" in result["stderr"]
|
||||
|
||||
@pytest.mark.xfail(reason="Late hook fails after privilege drop to vulcan (Permission denied)", strict=False)
|
||||
def test_late_hook_executes(self, run_baker, work_dir):
|
||||
config = os.path.join(FIXTURES, "with_hooks.yml")
|
||||
result = run_baker("prebake", config, cwd=work_dir, timeout=60)
|
||||
assert result["success"], f"prebake with hooks failed: stdout={result['stdout']!r} stderr={result['stderr']!r}"
|
||||
assert "late hook executed" in result["stdout"] or "late hook executed" in result["stderr"]
|
||||
|
||||
|
||||
@pytest.mark.prebake
|
||||
class TestPrebakeStatus:
|
||||
|
||||
def test_status_files_written(self, run_baker, work_dir):
|
||||
config = os.path.join(FIXTURES, "bootstrap_only.yml")
|
||||
result = run_baker("prebake", config, cwd=work_dir, timeout=60)
|
||||
assert result["success"]
|
||||
assert os.path.exists("/tmp/.hbwstatus")
|
||||
|
||||
def test_minimal_config_succeeds(self, run_baker, work_dir):
|
||||
config = os.path.join(FIXTURES, "minimal.yml")
|
||||
result = run_baker("prebake", config, cwd=work_dir, timeout=60)
|
||||
assert result["success"], f"Minimal prebake failed: {result['stderr']}"
|
||||
Generated
+107
@@ -0,0 +1,107 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.106"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.45"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_core"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.149"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"memchr",
|
||||
"serde",
|
||||
"serde_core",
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.117"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
||||
|
||||
[[package]]
|
||||
name = "workshop-baker-params"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
|
||||
@@ -0,0 +1,8 @@
|
||||
[package]
|
||||
name = "workshop-baker-params"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = "1.0.149"
|
||||
@@ -0,0 +1,24 @@
|
||||
use crate::apply_field;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use crate::{ParamVal, Writer, Overridden, traits::MergeParams};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BakeParams {
|
||||
#[serde(default)] pub workspace: ParamVal<String>,
|
||||
}
|
||||
|
||||
impl MergeParams for BakeParams {
|
||||
fn defaults() -> Self { Self { workspace: ParamVal::new("/workspace".into()) } }
|
||||
fn apply(&mut self, i: Self, w: Writer, ov: &mut Overridden, p: &str) {
|
||||
apply_field!(self.workspace, i.workspace, w, ov, p, "workspace");
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for BakeParams { fn default() -> Self { Self::defaults() } }
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BakeSettings { pub workspace: String }
|
||||
impl Default for BakeSettings { fn default() -> Self { BakeParams::defaults().into() } }
|
||||
impl From<&BakeParams> for BakeSettings { fn from(p: &BakeParams) -> Self { Self { workspace: p.workspace.value.clone() } } }
|
||||
impl From<BakeParams> for BakeSettings { fn from(p: BakeParams) -> Self { (&p).into() } }
|
||||
@@ -0,0 +1,23 @@
|
||||
use crate::apply_field;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use crate::{ParamVal, Writer, Overridden, traits::MergeParams};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct FinalizeParams {
|
||||
#[serde(default)] pub artifact_retention_days: ParamVal<u32>,
|
||||
}
|
||||
|
||||
impl MergeParams for FinalizeParams {
|
||||
fn defaults() -> Self { Self { artifact_retention_days: ParamVal::new(7) } }
|
||||
fn apply(&mut self, i: Self, w: Writer, ov: &mut Overridden, p: &str) {
|
||||
apply_field!(self.artifact_retention_days, i.artifact_retention_days, w, ov, p, "artifact_retention_days");
|
||||
}
|
||||
}
|
||||
impl Default for FinalizeParams { fn default() -> Self { Self::defaults() } }
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FinalizeSettings { pub artifact_retention_days: u32 }
|
||||
impl Default for FinalizeSettings { fn default() -> Self { FinalizeParams::defaults().into() } }
|
||||
impl From<&FinalizeParams> for FinalizeSettings { fn from(p: &FinalizeParams) -> Self { Self { artifact_retention_days: p.artifact_retention_days.value } } }
|
||||
impl From<FinalizeParams> for FinalizeSettings { fn from(p: FinalizeParams) -> Self { (&p).into() } }
|
||||
@@ -0,0 +1,168 @@
|
||||
pub mod notification;
|
||||
pub mod prebake;
|
||||
pub mod bake;
|
||||
pub mod finalize;
|
||||
pub mod writer;
|
||||
pub mod param;
|
||||
pub mod traits;
|
||||
|
||||
pub use notification::{NotificationParams, NotificationSettings};
|
||||
pub use prebake::{PrebakeParams, PrebakeSettings};
|
||||
pub use bake::{BakeParams, BakeSettings};
|
||||
pub use finalize::{FinalizeParams, FinalizeSettings};
|
||||
pub use writer::Writer;
|
||||
pub use param::ParamVal;
|
||||
pub use traits::MergeParams;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// Root parameter tree. Each component gets its own subtree.
|
||||
/// Used for both IPC (Serialized to JSON) and in-process merging.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct Params {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub notification: Option<NotificationParams>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub prebake: Option<PrebakeParams>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub bake: Option<BakeParams>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub finalize: Option<FinalizeParams>,
|
||||
}
|
||||
|
||||
/// Apply and merge helpers.
|
||||
impl Params {
|
||||
/// Merge a layer from a specific writer into the current params.
|
||||
/// Returns entries that were overridden (for debug).
|
||||
pub fn apply(&mut self, incoming: Params, writer: Writer) -> Overridden {
|
||||
let mut ov = Overridden::new();
|
||||
if let Some(l) = incoming.notification {
|
||||
if let Some(ref mut b) = self.notification {
|
||||
b.apply(l, writer, &mut ov, "notification.");
|
||||
} else {
|
||||
self.notification = Some(l);
|
||||
}
|
||||
}
|
||||
if let Some(l) = incoming.prebake {
|
||||
if let Some(ref mut b) = self.prebake {
|
||||
b.apply(l, writer, &mut ov, "prebake.");
|
||||
} else {
|
||||
self.prebake = Some(l);
|
||||
}
|
||||
}
|
||||
if let Some(l) = incoming.bake {
|
||||
if let Some(ref mut b) = self.bake {
|
||||
b.apply(l, writer, &mut ov, "bake.");
|
||||
} else {
|
||||
self.bake = Some(l);
|
||||
}
|
||||
}
|
||||
if let Some(l) = incoming.finalize {
|
||||
if let Some(ref mut b) = self.finalize {
|
||||
b.apply(l, writer, &mut ov, "finalize.");
|
||||
} else {
|
||||
self.finalize = Some(l);
|
||||
}
|
||||
}
|
||||
ov
|
||||
}
|
||||
|
||||
/// Convert to baker-consumable settings. All params guaranteed to have values.
|
||||
pub fn to_settings(&self) -> Result<AllSettings, String> {
|
||||
Ok(AllSettings {
|
||||
notification: self.notification.as_ref().map(NotificationSettings::from).unwrap_or_default(),
|
||||
prebake: self.prebake.as_ref().map(PrebakeSettings::from).unwrap_or_default(),
|
||||
bake: self.bake.as_ref().map(BakeSettings::from).unwrap_or_default(),
|
||||
finalize: self.finalize.as_ref().map(FinalizeSettings::from).unwrap_or_default(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Plain-value settings for baker consumption. No metadata, no Option.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AllSettings {
|
||||
pub notification: NotificationSettings,
|
||||
pub prebake: PrebakeSettings,
|
||||
pub bake: BakeSettings,
|
||||
pub finalize: FinalizeSettings,
|
||||
}
|
||||
|
||||
/// Debug — values that were overridden during merge, keyed by dot-path.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct Overridden {
|
||||
pub entries: std::collections::HashMap<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
impl Overridden {
|
||||
pub fn new() -> Self { Self { entries: HashMap::new() } }
|
||||
pub fn is_empty(&self) -> bool { self.entries.is_empty() }
|
||||
}
|
||||
|
||||
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_empty_params_serializes_empty() {
|
||||
let p = Params::default();
|
||||
let json = serde_json::to_string(&p).unwrap();
|
||||
assert_eq!(json, "{}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_notification_params_serialize() {
|
||||
let mut p = Params::default();
|
||||
p.notification = Some(NotificationParams::defaults());
|
||||
let json = serde_json::to_string_pretty(&p).unwrap();
|
||||
assert!(json.contains("max_lives"));
|
||||
assert!(json.contains("3"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_apply_courier_overrides_default() {
|
||||
let mut base = Params::default();
|
||||
base.notification = Some(NotificationParams::defaults());
|
||||
let incoming = Params {
|
||||
notification: Some(NotificationParams {
|
||||
max_lives: ParamVal::with(5, Writer::Courier),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
let ov = base.apply(incoming, Writer::Courier);
|
||||
let s = base.to_settings().unwrap();
|
||||
assert_eq!(s.notification.max_lives, 5);
|
||||
assert!(ov.is_empty()); // default had no writer, not an "override"
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_apply_upstream_wins() {
|
||||
let mut base = Params::default();
|
||||
base.notification = Some(NotificationParams {
|
||||
max_lives: ParamVal::with(3, Writer::Base),
|
||||
..Default::default()
|
||||
});
|
||||
let incoming = Params {
|
||||
notification: Some(NotificationParams {
|
||||
max_lives: ParamVal::with(5, Writer::Courier),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
let ov = base.apply(incoming, Writer::Courier);
|
||||
let s = base.to_settings().unwrap();
|
||||
// base(3) > courier(2) — rejected, base stays
|
||||
assert_eq!(s.notification.max_lives, 3);
|
||||
assert!(ov.is_empty()); // nothing overridden
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_deny_unknown_fields() {
|
||||
let bad = r#"{"max_lives":{"value":5},"ghost_param":{"value":1}}"#;
|
||||
let result: Result<NotificationParams, _> = serde_json::from_str(bad);
|
||||
assert!(result.is_err()); // unknown field should be denied
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
use crate::apply_field;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use crate::{ParamVal, Writer, Overridden, traits::MergeParams};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct NotificationParams {
|
||||
#[serde(default)] pub max_lives: ParamVal<u32>,
|
||||
#[serde(default)] pub timeout_ms: ParamVal<u64>,
|
||||
#[serde(default)] pub retry_backoff: ParamVal<f64>,
|
||||
#[serde(default)] pub retry_delay_seconds: ParamVal<f64>,
|
||||
#[serde(default)] pub max_retry_delay_seconds: ParamVal<f64>,
|
||||
#[serde(default)] pub life_impact_factor: ParamVal<f64>,
|
||||
#[serde(default)] pub batch_period: ParamVal<u32>,
|
||||
#[serde(default)] pub batch_watermark: ParamVal<u32>,
|
||||
#[serde(default)] pub template_ref_depth: ParamVal<u32>,
|
||||
}
|
||||
|
||||
impl MergeParams for NotificationParams {
|
||||
fn defaults() -> Self {
|
||||
Self {
|
||||
max_lives: ParamVal::new(3), timeout_ms: ParamVal::new(30000),
|
||||
retry_backoff: ParamVal::new(2.0), retry_delay_seconds: ParamVal::new(5.0),
|
||||
max_retry_delay_seconds: ParamVal::new(300.0),
|
||||
life_impact_factor: ParamVal::new(0.0),
|
||||
batch_period: ParamVal::new(30), batch_watermark: ParamVal::new(5),
|
||||
template_ref_depth: ParamVal::new(10),
|
||||
}
|
||||
}
|
||||
fn apply(&mut self, incoming: Self, writer: Writer, ov: &mut Overridden, prefix: &str) {
|
||||
apply_field!(self.max_lives, incoming.max_lives, writer, ov, prefix, "max_lives");
|
||||
apply_field!(self.timeout_ms, incoming.timeout_ms, writer, ov, prefix, "timeout_ms");
|
||||
apply_field!(self.retry_backoff, incoming.retry_backoff, writer, ov, prefix, "retry_backoff");
|
||||
apply_field!(self.retry_delay_seconds, incoming.retry_delay_seconds, writer, ov, prefix, "retry_delay_seconds");
|
||||
apply_field!(self.max_retry_delay_seconds, incoming.max_retry_delay_seconds, writer, ov, prefix, "max_retry_delay_seconds");
|
||||
apply_field!(self.life_impact_factor, incoming.life_impact_factor, writer, ov, prefix, "life_impact_factor");
|
||||
apply_field!(self.batch_period, incoming.batch_period, writer, ov, prefix, "batch_period");
|
||||
apply_field!(self.batch_watermark, incoming.batch_watermark, writer, ov, prefix, "batch_watermark");
|
||||
apply_field!(self.template_ref_depth, incoming.template_ref_depth, writer, ov, prefix, "template_ref_depth");
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for NotificationParams { fn default() -> Self { Self::defaults() } }
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct NotificationSettings {
|
||||
pub max_lives: u32, pub timeout_ms: u64, pub retry_backoff: f64,
|
||||
pub retry_delay_seconds: f64, pub max_retry_delay_seconds: f64,
|
||||
pub life_impact_factor: f64,
|
||||
pub batch_period: u32, pub batch_watermark: u32, pub template_ref_depth: u32,
|
||||
}
|
||||
|
||||
impl Default for NotificationSettings { fn default() -> Self { NotificationParams::defaults().into() } }
|
||||
|
||||
impl From<&NotificationParams> for NotificationSettings {
|
||||
fn from(p: &NotificationParams) -> Self { Self {
|
||||
max_lives: p.max_lives.value, timeout_ms: p.timeout_ms.value,
|
||||
retry_backoff: p.retry_backoff.value, retry_delay_seconds: p.retry_delay_seconds.value,
|
||||
max_retry_delay_seconds: p.max_retry_delay_seconds.value,
|
||||
life_impact_factor: p.life_impact_factor.value,
|
||||
batch_period: p.batch_period.value, batch_watermark: p.batch_watermark.value,
|
||||
template_ref_depth: p.template_ref_depth.value,
|
||||
}}
|
||||
}
|
||||
impl From<NotificationParams> for NotificationSettings { fn from(p: NotificationParams) -> Self { (&p).into() } }
|
||||
@@ -0,0 +1,32 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use crate::Writer;
|
||||
|
||||
/// A strongly-typed parameter value with writer tracking.
|
||||
///
|
||||
/// Serializes as `{ "value": 3, "writer": "base" }`.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ParamVal<T> {
|
||||
pub value: T,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub writer: Option<Writer>,
|
||||
}
|
||||
|
||||
impl<T: Default> Default for ParamVal<T> {
|
||||
fn default() -> Self {
|
||||
ParamVal { value: T::default(), writer: None }
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> ParamVal<T> {
|
||||
pub fn new(value: T) -> Self {
|
||||
ParamVal { value, writer: None }
|
||||
}
|
||||
pub fn with(value: T, writer: Writer) -> Self {
|
||||
ParamVal { value, writer: Some(writer) }
|
||||
}
|
||||
/// Whether an incoming value from `incoming_writer` should override this one.
|
||||
pub fn should_override(&self, incoming: Writer) -> bool {
|
||||
let current = self.writer.map_or(0, |w| w.priority());
|
||||
incoming.priority() > current
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
pub mod network;
|
||||
pub mod cache;
|
||||
pub mod engine;
|
||||
|
||||
use crate::apply_field;
|
||||
use crate::{ParamVal, Writer, Overridden, traits::MergeParams};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct PrebakeParams {
|
||||
#[serde(default)] pub bootstrap_user: ParamVal<String>,
|
||||
#[serde(default)] pub workspace: ParamVal<String>,
|
||||
#[serde(default)] pub network: network::NetworkParams,
|
||||
#[serde(default)] pub cache: cache::CacheParams,
|
||||
#[serde(default)] pub engine: engine::EngineParams,
|
||||
#[serde(default)] pub repology_endpoint: ParamVal<String>,
|
||||
}
|
||||
|
||||
impl MergeParams for PrebakeParams {
|
||||
fn defaults() -> Self {
|
||||
Self {
|
||||
bootstrap_user: ParamVal::new("vulcan".into()), workspace: ParamVal::new("/home/vulcan/workspace".into()),
|
||||
network: network::NetworkParams::defaults(), cache: cache::CacheParams::defaults(),
|
||||
engine: engine::EngineParams::defaults(),
|
||||
repology_endpoint: ParamVal::new("default".into()),
|
||||
}
|
||||
}
|
||||
fn apply(&mut self, i: Self, w: Writer, ov: &mut Overridden, p: &str) {
|
||||
apply_field!(self.bootstrap_user, i.bootstrap_user, w, ov, p, "bootstrap_user");
|
||||
apply_field!(self.workspace, i.workspace, w, ov, p, "workspace");
|
||||
apply_field!(self.repology_endpoint, i.repology_endpoint, w, ov, p, "repology_endpoint");
|
||||
self.network.apply(i.network, w, ov, &format!("{}network.", p));
|
||||
self.cache.apply(i.cache, w, ov, &format!("{}cache.", p));
|
||||
self.engine.apply(i.engine, w, ov, &format!("{}engine.", p));
|
||||
}
|
||||
}
|
||||
impl Default for PrebakeParams { fn default() -> Self { Self::defaults() } }
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PrebakeSettings {
|
||||
pub bootstrap_user: String, pub workspace: String,
|
||||
pub network: network::NetworkSettings, pub cache: cache::CacheSettings,
|
||||
pub engine: engine::EngineSettings, pub repology_endpoint: String,
|
||||
}
|
||||
impl Default for PrebakeSettings { fn default() -> Self { PrebakeParams::defaults().into() } }
|
||||
impl From<&PrebakeParams> for PrebakeSettings {
|
||||
fn from(p: &PrebakeParams) -> Self { Self {
|
||||
bootstrap_user: p.bootstrap_user.value.clone(), workspace: p.workspace.value.clone(),
|
||||
network: (&p.network).into(), cache: (&p.cache).into(), engine: (&p.engine).into(),
|
||||
repology_endpoint: p.repology_endpoint.value.clone(),
|
||||
}}
|
||||
}
|
||||
impl From<PrebakeParams> for PrebakeSettings { fn from(p: PrebakeParams) -> Self { (&p).into() } }
|
||||
@@ -0,0 +1,37 @@
|
||||
use crate::apply_field;
|
||||
use crate::{ParamVal, Writer, Overridden, traits::MergeParams};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct CacheParams {
|
||||
#[serde(default)] pub strategy: ParamVal<String>,
|
||||
#[serde(default)] pub mode: ParamVal<String>,
|
||||
#[serde(default)] pub ttl_days: ParamVal<u32>,
|
||||
#[serde(default)] pub max_size_gb: ParamVal<u32>,
|
||||
#[serde(default)] pub cleanup_policy: ParamVal<String>,
|
||||
}
|
||||
impl MergeParams for CacheParams {
|
||||
fn defaults() -> Self {
|
||||
Self {
|
||||
strategy: ParamVal::new("always".into()), mode: ParamVal::new("zstd".into()),
|
||||
ttl_days: ParamVal::new(30), max_size_gb: ParamVal::new(20),
|
||||
cleanup_policy: ParamVal::new("lru".into()),
|
||||
}
|
||||
}
|
||||
fn apply(&mut self, i: Self, w: Writer, ov: &mut Overridden, p: &str) {
|
||||
apply_field!(self.strategy, i.strategy, w, ov, p, "strategy");
|
||||
apply_field!(self.mode, i.mode, w, ov, p, "mode");
|
||||
apply_field!(self.ttl_days, i.ttl_days, w, ov, p, "ttl_days");
|
||||
apply_field!(self.max_size_gb, i.max_size_gb, w, ov, p, "max_size_gb");
|
||||
apply_field!(self.cleanup_policy, i.cleanup_policy, w, ov, p, "cleanup_policy");
|
||||
}
|
||||
}
|
||||
impl Default for CacheParams { fn default() -> Self { Self::defaults() } }
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct CacheSettings { pub strategy: String, pub mode: String, pub ttl_days: u32, pub max_size_gb: u32, pub cleanup_policy: String }
|
||||
impl Default for CacheSettings { fn default() -> Self { CacheParams::defaults().into() } }
|
||||
impl From<&CacheParams> for CacheSettings { fn from(p: &CacheParams) -> Self {
|
||||
Self { strategy: p.strategy.value.clone(), mode: p.mode.value.clone(), ttl_days: p.ttl_days.value, max_size_gb: p.max_size_gb.value, cleanup_policy: p.cleanup_policy.value.clone() }
|
||||
} }
|
||||
impl From<CacheParams> for CacheSettings { fn from(p: CacheParams) -> Self { (&p).into() } }
|
||||
@@ -0,0 +1,21 @@
|
||||
use crate::apply_field;
|
||||
use crate::{ParamVal, Writer, Overridden, traits::MergeParams};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct EngineParams {
|
||||
#[serde(default)] pub timeout_ms: ParamVal<u64>,
|
||||
}
|
||||
impl MergeParams for EngineParams {
|
||||
fn defaults() -> Self { Self { timeout_ms: ParamVal::new(300000) } }
|
||||
fn apply(&mut self, i: Self, w: Writer, ov: &mut Overridden, p: &str) {
|
||||
apply_field!(self.timeout_ms, i.timeout_ms, w, ov, p, "timeout_ms");
|
||||
}
|
||||
}
|
||||
impl Default for EngineParams { fn default() -> Self { Self::defaults() } }
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct EngineSettings { pub timeout_ms: u64 }
|
||||
impl Default for EngineSettings { fn default() -> Self { EngineParams::defaults().into() } }
|
||||
impl From<&EngineParams> for EngineSettings { fn from(p: &EngineParams) -> Self { Self { timeout_ms: p.timeout_ms.value } } }
|
||||
impl From<EngineParams> for EngineSettings { fn from(p: EngineParams) -> Self { (&p).into() } }
|
||||
@@ -0,0 +1,23 @@
|
||||
use crate::apply_field;
|
||||
use crate::{ParamVal, Writer, Overridden, traits::MergeParams};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct NetworkParams {
|
||||
#[serde(default)] pub enabled: ParamVal<bool>,
|
||||
#[serde(default)] pub outbound: ParamVal<bool>,
|
||||
}
|
||||
impl MergeParams for NetworkParams {
|
||||
fn defaults() -> Self { Self { enabled: ParamVal::new(true), outbound: ParamVal::new(true) } }
|
||||
fn apply(&mut self, i: Self, w: Writer, ov: &mut Overridden, p: &str) {
|
||||
apply_field!(self.enabled, i.enabled, w, ov, p, "enabled");
|
||||
apply_field!(self.outbound, i.outbound, w, ov, p, "outbound");
|
||||
}
|
||||
}
|
||||
impl Default for NetworkParams { fn default() -> Self { Self::defaults() } }
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct NetworkSettings { pub enabled: bool, pub outbound: bool }
|
||||
impl Default for NetworkSettings { fn default() -> Self { NetworkParams::defaults().into() } }
|
||||
impl From<&NetworkParams> for NetworkSettings { fn from(p: &NetworkParams) -> Self { Self { enabled: p.enabled.value, outbound: p.outbound.value } } }
|
||||
impl From<NetworkParams> for NetworkSettings { fn from(p: NetworkParams) -> Self { (&p).into() } }
|
||||
@@ -0,0 +1,28 @@
|
||||
use crate::{Writer, Overridden};
|
||||
|
||||
/// A parameter tree node that can merge itself with an incoming layer.
|
||||
pub trait MergeParams: Sized {
|
||||
fn apply(&mut self, incoming: Self, writer: Writer, ov: &mut Overridden, prefix: &str);
|
||||
fn defaults() -> Self;
|
||||
}
|
||||
|
||||
/// Apply one field's merge logic.
|
||||
/// ```ignore
|
||||
/// apply_field!(self.foo, incoming.foo, writer, ov, prefix, "foo");
|
||||
/// ```
|
||||
#[macro_export]
|
||||
macro_rules! apply_field {
|
||||
($self:ident.$field:ident, $incoming:ident.$field2:ident, $writer:expr, $ov:ident, $prefix:expr, $name:expr) => {
|
||||
let old_w = $self.$field.writer;
|
||||
if $self.$field.should_override($writer) {
|
||||
let old_v = ::std::mem::replace(&mut $self.$field.value, $incoming.$field2.value);
|
||||
$self.$field.writer = ::std::option::Option::Some($writer);
|
||||
if let ::std::option::Option::Some(w) = old_w {
|
||||
$ov.entries.insert(
|
||||
format!("{}{}", $prefix, $name),
|
||||
::serde_json::json!({"value": old_v, "writer": w.as_str()}),
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum Writer {
|
||||
Base,
|
||||
Courier,
|
||||
Bakerd,
|
||||
}
|
||||
|
||||
impl Writer {
|
||||
/// Priority: Base=3 > Courier=2 > Bakerd=1
|
||||
pub fn priority(self) -> u8 {
|
||||
match self {
|
||||
Writer::Base => 3,
|
||||
Writer::Courier => 2,
|
||||
Writer::Bakerd => 1,
|
||||
}
|
||||
}
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Writer::Base => "base", Writer::Courier => "courier", Writer::Bakerd => "bakerd",
|
||||
}
|
||||
}
|
||||
}
|
||||
impl Default for Writer { fn default() -> Self { Writer::Base } }
|
||||
@@ -1,3 +1,4 @@
|
||||
temp
|
||||
examples
|
||||
quicktest.sh
|
||||
_env.sh
|
||||
|
||||
Generated
+125
-1561
File diff suppressed because it is too large
Load Diff
+14
-14
@@ -15,23 +15,20 @@ path = "src/main.rs"
|
||||
default = []
|
||||
integration-tests = []
|
||||
|
||||
[[test]]
|
||||
name = "engine_integration"
|
||||
path = "tests/engine_integration.rs"
|
||||
|
||||
[dependencies]
|
||||
workshop-engine = { path = "../workshop-engine" }
|
||||
anyhow = "1.0.100"
|
||||
bollard = { version = "0.20.0", features = ["buildkit", "chrono", "ssh"] }
|
||||
cgroups-rs = "0.5.0"
|
||||
chrono = { version = "0.4.42", features = ["serde"] }
|
||||
clap = { version = "4.5.53", features = ["derive"] }
|
||||
clap = { version = "4.5.53", features = ["derive", "env"] }
|
||||
config = "0.15.19"
|
||||
duration-str = "0.21.0"
|
||||
env_logger = "0.11.8"
|
||||
glob = "0.3.2"
|
||||
futures-util = "0.3.31"
|
||||
lazy_static = "1.5.0"
|
||||
lettre = { version = "0.11", features = ["tokio1-native-tls"] }
|
||||
lettre = { version = "0.11", default-features = false, features = ["tokio1-rustls", "rustls-tls", "builder", "smtp-transport"] }
|
||||
log = "0.4.28"
|
||||
minijinja = "2.14.0"
|
||||
regex = "1.12.2"
|
||||
@@ -50,16 +47,19 @@ uuid = { version = "1.19.0", features = ["v4"] }
|
||||
libc = "0.2"
|
||||
os_info = "3.14.0"
|
||||
privdrop = "0.5.6"
|
||||
reqwest = { version = "0.12", features = ["json"] }
|
||||
which = "8.0.2"
|
||||
gix = "0.81.0"
|
||||
sha2 = "0.10"
|
||||
flate2 = "1.0"
|
||||
zstd = "0.13"
|
||||
globset = "0.4.18"
|
||||
axum = "0.7"
|
||||
workshop-baker-params = { version = "0.1.0", path = "../workshop-baker-params" }
|
||||
|
||||
[dev-dependencies]
|
||||
criterion = "0.5"
|
||||
|
||||
[[bench]]
|
||||
name = "parser_bench"
|
||||
harness = false
|
||||
[lints.rust]
|
||||
dead_code = "allow"
|
||||
unreachable_code = "allow"
|
||||
|
||||
[lints.clippy]
|
||||
inherent_to_string = "allow"
|
||||
non_canonical_partial_ord_impl = "allow"
|
||||
|
||||
|
||||
@@ -1,93 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import random
|
||||
import string
|
||||
|
||||
OUTPUT_DIR = "/tmp"
|
||||
DECORATORS = [
|
||||
"# @pipeline",
|
||||
"# @fallible",
|
||||
"# @parallel",
|
||||
"# @export",
|
||||
"# @timeout(60)",
|
||||
"# @retry(3, 5)",
|
||||
"# @if(condition)",
|
||||
"# @after(dep)",
|
||||
"# @pipe(step1, step2)",
|
||||
"# @health(/health)",
|
||||
"# @loop(10)",
|
||||
]
|
||||
FUNCTION_BODIES = [
|
||||
"echo 'Processing...';",
|
||||
"local var=$(date +%s);",
|
||||
"if [ -f /tmp/test ]; then echo 'exists'; fi;",
|
||||
"for i in $(seq 1 10); do echo $i; done;",
|
||||
"case $1 in start) echo 'starting';; stop) echo 'stopping';; esac;",
|
||||
"read -r line < /dev/stdin;",
|
||||
"export PATH=$PATH:/usr/local/bin;",
|
||||
"set -e; set -u;",
|
||||
"trap 'echo error' ERR;",
|
||||
"cd /tmp || exit 1;",
|
||||
"local result=$(grep -r 'pattern' /var/log/ 2>/dev/null);",
|
||||
'for f in /tmp/*.tmp; do rm -f "$f"; done',
|
||||
"if [[ $var -gt 100 ]]; then echo 'large'; fi;",
|
||||
'while read -r line; do echo "$line"; done < /etc/passwd;',
|
||||
'select opt in a b c; do echo "Selected $opt"; break; done;',
|
||||
]
|
||||
|
||||
|
||||
def random_string(length=8):
|
||||
return "".join(random.choices(string.ascii_letters + string.digits, k=length))
|
||||
|
||||
|
||||
def generate_function(body_lines):
|
||||
lines = []
|
||||
func_name = f"func_{random_string(6)}"
|
||||
for _ in range(random.randint(0, 3)):
|
||||
lines.append(random.choice(DECORATORS))
|
||||
lines.append(f"{func_name}() {{")
|
||||
for _ in range(body_lines):
|
||||
lines.append(" " + random.choice(FUNCTION_BODIES))
|
||||
lines.append("}")
|
||||
lines.append("")
|
||||
return lines
|
||||
|
||||
|
||||
def generate_script(target_lines, num_functions, target_bytes):
|
||||
lines = []
|
||||
total_lines = 0
|
||||
body_per_func = max(1, (target_lines - num_functions * 3) // num_functions)
|
||||
|
||||
for _ in range(num_functions):
|
||||
func_lines = generate_function(body_per_func)
|
||||
lines.extend(func_lines)
|
||||
total_lines += len(func_lines)
|
||||
|
||||
while (
|
||||
total_lines < target_lines
|
||||
or len("\n".join(lines).encode("utf-8")) < target_bytes
|
||||
):
|
||||
lines.append(random.choice(FUNCTION_BODIES))
|
||||
total_lines += 1
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main():
|
||||
sizes = [
|
||||
("small", 1000, 10, 50_000),
|
||||
("medium", 5000, 50, 500_000),
|
||||
("large", 10000, 100, 950_000),
|
||||
]
|
||||
for name, lines, funcs, target_bytes in sizes:
|
||||
script = generate_script(lines, funcs, target_bytes)
|
||||
filepath = os.path.join(OUTPUT_DIR, f"bench_{name}.sh")
|
||||
with open(filepath, "w") as f:
|
||||
f.write(script)
|
||||
actual_bytes = len(script.encode("utf-8"))
|
||||
actual_lines = len(script.splitlines())
|
||||
print(f"Generated {filepath}: lines={actual_lines}, bytes={actual_bytes}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,61 +0,0 @@
|
||||
use criterion::{Criterion, black_box, criterion_group, criterion_main};
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
fn parse_large_script(c: &mut Criterion) {
|
||||
let script_path = Path::new("/tmp/bench_large.sh");
|
||||
|
||||
let script_content = fs::read_to_string(script_path)
|
||||
.expect("Failed to read benchmark script. Run `python3 benches/generate.py` first.");
|
||||
|
||||
let byte_size = script_content.len();
|
||||
let line_count = script_content.lines().count();
|
||||
|
||||
println!("Benchmark file: {} lines, {} bytes", line_count, byte_size);
|
||||
|
||||
c.bench_function("parse_script_large", |b| {
|
||||
b.iter(|| {
|
||||
let result =
|
||||
workshop_baker::bake::parser::parse_script(black_box(&script_content.clone()));
|
||||
black_box(result)
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
fn parse_medium_script(c: &mut Criterion) {
|
||||
let script_path = Path::new("/tmp/bench_medium.sh");
|
||||
|
||||
let script_content = fs::read_to_string(script_path)
|
||||
.expect("Failed to read benchmark script. Run `python3 benches/generate.py` first.");
|
||||
|
||||
c.bench_function("parse_script_medium", |b| {
|
||||
b.iter(|| {
|
||||
let result =
|
||||
workshop_baker::bake::parser::parse_script(black_box(&script_content.clone()));
|
||||
black_box(result)
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
fn parse_small_script(c: &mut Criterion) {
|
||||
let script_path = Path::new("/tmp/bench_small.sh");
|
||||
|
||||
let script_content = fs::read_to_string(script_path)
|
||||
.expect("Failed to read benchmark script. Run `python3 benches/generate.py` first.");
|
||||
|
||||
c.bench_function("parse_script_small", |b| {
|
||||
b.iter(|| {
|
||||
let result =
|
||||
workshop_baker::bake::parser::parse_script(black_box(&script_content.clone()));
|
||||
black_box(result)
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
criterion_group!(
|
||||
benches,
|
||||
parse_small_script,
|
||||
parse_medium_script,
|
||||
parse_large_script
|
||||
);
|
||||
criterion_main!(benches);
|
||||
@@ -1,62 +1,57 @@
|
||||
# workshop-baker/src
|
||||
|
||||
**Generated:** 2026-04-22
|
||||
**Commit:** 7b3b71a
|
||||
**Generated:** 2026-05-19
|
||||
**Commit:** 28abc5d
|
||||
|
||||
Baker CLI + execution engine for HoneyBiscuitWorkshop pipelines.
|
||||
|
||||
## OVERVIEW
|
||||
|
||||
CLI tool and runtime for executing CI/CD pipelines with resource isolation, privilege dropping, and package management integration.
|
||||
CLI tool and runtime for executing CI/CD pipelines. Orchestrates prebake → bake → finalize stages with notification dispatch.
|
||||
|
||||
## STRUCTURE
|
||||
|
||||
```
|
||||
src/
|
||||
├── bake.rs # Top-level bake orchestration
|
||||
├── bake/ # Script parsing and building
|
||||
│ ├── parser.rs # @decorator-aware shell parser
|
||||
│ ├── decorator.rs # @pipeline, @retry, @parallel, etc.
|
||||
│ ├── builder.rs # Script template builder
|
||||
│ └── schedule.rs # Function ordering via @after deps
|
||||
├── engine/ # Execution runtime
|
||||
│ ├── executor.rs # Script execution with cgroups
|
||||
│ ├── cgroups.rs # Linux cgroup resource limits
|
||||
│ ├── repology.rs # Package name resolution
|
||||
│ ├── pm.rs # Package manager detection
|
||||
│ └── upm.rs # User package manager (Nix/Guix)
|
||||
├── prebake/ # Build environment setup
|
||||
│ ├── config.rs # PrebakeConfig YAML schema
|
||||
│ ├── stage/ # Bootstrap, DepsSystem, DepsUser, Hooks
|
||||
│ └── security.rs # Privilege dropping logic
|
||||
├── types/ # Shared type definitions
|
||||
│ ├── repology.rs # RepologyEndpoint enum
|
||||
│ ├── builderconfig.rs
|
||||
│ ├── cache.rs
|
||||
│ └── memsize.rs
|
||||
├── monitor/ # Resource usage monitoring
|
||||
│ ├── cgroups.rs # cgroup-based usage collection
|
||||
│ └── jobobject.rs # Windows job objects (stub)
|
||||
├── daemon.rs # Unix socket daemon (TODO: refactor)
|
||||
└── lib.rs # CLI struct + module exports
|
||||
├── lib.rs # Module declarations + Cli struct + Commands enum
|
||||
├── main.rs # tokio::main entry point
|
||||
├── bare.rs # Bare mode (run individual stages)
|
||||
├── error.rs # CliError (wraps stage errors)
|
||||
├── bake.rs # Bake orchestration (script → execution)
|
||||
├── bake/ # Script parsing, decorator DSL, scheduling
|
||||
├── prebake.rs # Prebake orchestration (env setup)
|
||||
├── prebake/ # Config, stages, security, env providers
|
||||
├── finalize.rs # Finalize orchestration (post-build)
|
||||
├── finalize/ # Config, plugins, hooks, templates
|
||||
├── notify.rs # Notification system (batching, retry)
|
||||
├── notify/ # Types, templates, interactive, queue
|
||||
├── daemon/ # Empty (Daemon command = unimplemented!)
|
||||
├── monitor/ # Resource usage monitoring (cgroups)
|
||||
├── types.rs # Module re-exports
|
||||
├── types/ # buildstatus, resource types
|
||||
├── utils.rs # Module root (fetch submodule)
|
||||
└── utils/ # fetch.rs
|
||||
```
|
||||
|
||||
## WHERE TO LOOK
|
||||
|
||||
| Task | Location |
|
||||
|------|----------|
|
||||
| Script parsing with @decorators | `bake/parser.rs:29` - `parse_script()` |
|
||||
| Decorator types | `bake/decorator.rs` - `Decorator` enum |
|
||||
| Build execution | `engine/executor.rs` - `Executor::execute_script()` |
|
||||
| Resource limits | `engine/types.rs:56` - `ResourceLimits` struct |
|
||||
| Prebake stages | `prebake/stage/` subdirectories |
|
||||
| Package detection | `engine/pm.rs:detection()` |
|
||||
| CLI entry | `lib.rs:26` - `cli::Cli` struct |
|
||||
| CLI args | `lib.rs:21` - `Cli` struct |
|
||||
| Entry point | `main.rs:91` - `#[tokio::main]` |
|
||||
| Pipeline orchestration | `main.rs:68-74` - prebake→bake→finalize chain |
|
||||
| Bare mode (single stage) | `bare.rs:14` - `run_bare()` |
|
||||
| Script parsing | `bake/parser.rs:56` - `parse_script()` |
|
||||
| Decorator DSL | `bake/decorator.rs` - `Decorator` enum |
|
||||
| Prebake stages | `prebake/stage/` - Bootstrap→DepsSystem→DepsUser→Hooks→Ready |
|
||||
| Plugin system | `finalize/plugin.rs` + `finalize/plugin/` |
|
||||
| Notification dispatch | `notify.rs:66` - `notify()` async loop |
|
||||
| Notification types | `notify/types.rs` - `NotificationRule`, `ParsedTrigger` |
|
||||
|
||||
## CONVENTIONS
|
||||
|
||||
- **ExecutionContext**: Passed through all stages, carries task_id, username, env_vars, timeout
|
||||
- **PrebakeStage**: Ordered enum controlling stage execution sequence (Bootstrap -> Ready)
|
||||
- **Decorator**: Attached to shell functions via `# @name(args)` comments above function declarations
|
||||
- **Engine**: Lightweight wrapper around cgroup manager for resource isolation
|
||||
- **Dual-mode**: CLI commands (prebake, bake, finalize) vs daemon mode via Unix socket
|
||||
- **PrebakeStage**: Ordered enum Bootstrap→EarlyHook→DepsSystem→DepsUser→LateHook→Ready
|
||||
- **Decorator**: `# @name(args)` comments above shell function declarations
|
||||
- **Notification**: Plugin-based dispatch with batching, exponential backoff, priority groups
|
||||
- **Engine**: Delegated to `workshop-engine` crate (formerly in-tree)
|
||||
- **Dual-mode**: CLI commands (standalone) vs Daemon mode (unimplemented)
|
||||
|
||||
## ANTI-PATTERNS
|
||||
1. **Empty daemon/** directory — Daemon command is `unimplemented!()`
|
||||
2. **Dead code allowed** — `lints.rust.dead_code = "allow"` in Cargo.toml
|
||||
3. **Engine was extracted** — workshop-engine is its own crate now, src/engine/ removed
|
||||
4. **Python test infra exists but files removed** — pytest.ini remains, test dir is empty
|
||||
|
||||
+15
-58
@@ -1,19 +1,18 @@
|
||||
use crate::ExecutionContext;
|
||||
use crate::bake::constant::{DEFAULT_WORKSPACE, TRIVIAL_SCRIPT_NAME};
|
||||
use crate::bake::error::BakeError;
|
||||
use crate::bake::parser::Function;
|
||||
use crate::cli::Cli;
|
||||
use crate::constant::{DEFAULT_WORKSPACE, STANDALONE_STATUS_DIR, TRIVIAL_SCRIPT_NAME};
|
||||
use crate::engine::EventSender;
|
||||
use crate::error::BakeError;
|
||||
use crate::prebake;
|
||||
use crate::prebake::PrebakeConfig;
|
||||
use crate::prebake::security::get_drop_after;
|
||||
use crate::prebake::stage::PrebakeStage;
|
||||
use crate::types::buildstatus::{StageInfo, StagePhase, StageResult, write_stage_status};
|
||||
use crate::{Engine, prebake};
|
||||
use chrono::Utc;
|
||||
use std::path::{Path, PathBuf};
|
||||
use workshop_engine::{Engine, EventSender, ExecutionContext};
|
||||
|
||||
mod builder;
|
||||
pub mod constant;
|
||||
mod decorator;
|
||||
pub mod error;
|
||||
pub mod parser;
|
||||
mod schedule;
|
||||
|
||||
@@ -21,7 +20,7 @@ pub async fn bake(
|
||||
script_path: &Path,
|
||||
bake_base_path: &Path,
|
||||
prebake_path: &Path,
|
||||
cli: &Cli,
|
||||
_cli: &Cli,
|
||||
ctx: &mut ExecutionContext,
|
||||
event_tx: EventSender,
|
||||
) -> Result<(), BakeError> {
|
||||
@@ -44,14 +43,13 @@ pub async fn bake(
|
||||
let use_template = true;
|
||||
|
||||
let engine = Engine::new();
|
||||
if let Some(env) = &prebake.envvars {
|
||||
if let Some(prebake_env) = &env.bake {
|
||||
if let Some(env) = &prebake.envvars
|
||||
&& let Some(prebake_env) = &env.bake
|
||||
{
|
||||
ctx.env_vars.extend(prebake_env.clone());
|
||||
}
|
||||
}
|
||||
|
||||
if !has_pipeline || trivial {
|
||||
let stage_start = Utc::now();
|
||||
let function = Function {
|
||||
name: TRIVIAL_SCRIPT_NAME.to_string(),
|
||||
decorators: vec![],
|
||||
@@ -59,33 +57,12 @@ pub async fn bake(
|
||||
};
|
||||
let script =
|
||||
builder::build_script(&function, bake_base_path, &workspace, None, use_template)?;
|
||||
let result = engine.execute_script(&script, &ctx, &event_tx).await;
|
||||
let stage_result = if result.is_ok() {
|
||||
StageResult::Success
|
||||
} else {
|
||||
StageResult::Failure
|
||||
};
|
||||
let _ = write_stage_status(
|
||||
STANDALONE_STATUS_DIR,
|
||||
StageInfo {
|
||||
name: "trivial".to_string(),
|
||||
phase: StagePhase::Bake,
|
||||
substage: "trivial".to_string(),
|
||||
result: stage_result,
|
||||
duration_ms: Utc::now()
|
||||
.signed_duration_since(stage_start)
|
||||
.num_milliseconds() as u64,
|
||||
started_at: stage_start,
|
||||
finished_at: Utc::now(),
|
||||
error_message: result.as_ref().err().map(|e| e.to_string()),
|
||||
},
|
||||
);
|
||||
let result = engine.execute_script(&script, ctx, &event_tx).await;
|
||||
result?;
|
||||
} else {
|
||||
let sorted_functions = schedule::sort_function(functions.pipeline_functions)?;
|
||||
let remaining = functions.remaining_code;
|
||||
for func in sorted_functions {
|
||||
let stage_start = Utc::now();
|
||||
let mut modified_func = func.clone();
|
||||
modified_func.body = remaining.clone() + &func.body;
|
||||
let script = builder::build_script(
|
||||
@@ -95,27 +72,7 @@ pub async fn bake(
|
||||
None,
|
||||
use_template,
|
||||
)?;
|
||||
let result = engine.execute_script(&script, &ctx, &event_tx).await;
|
||||
let stage_result = if result.is_ok() {
|
||||
StageResult::Success
|
||||
} else {
|
||||
StageResult::Failure
|
||||
};
|
||||
let _ = write_stage_status(
|
||||
STANDALONE_STATUS_DIR,
|
||||
StageInfo {
|
||||
name: func.name.clone(),
|
||||
phase: StagePhase::Bake,
|
||||
substage: func.name.clone(),
|
||||
result: stage_result,
|
||||
duration_ms: Utc::now()
|
||||
.signed_duration_since(stage_start)
|
||||
.num_milliseconds() as u64,
|
||||
started_at: stage_start,
|
||||
finished_at: Utc::now(),
|
||||
error_message: result.as_ref().err().map(|e| e.to_string()),
|
||||
},
|
||||
);
|
||||
let result = engine.execute_script(&script, ctx, &event_tx).await;
|
||||
result?;
|
||||
}
|
||||
}
|
||||
@@ -139,11 +96,11 @@ fn get_workspace(prebake_config: &PrebakeConfig) -> PathBuf {
|
||||
}
|
||||
|
||||
fn privileged(prebake_config: &PrebakeConfig) -> bool {
|
||||
if let Some(bootstrap) = &prebake_config.bootstrap {
|
||||
if bootstrap.user == "root" {
|
||||
if let Some(bootstrap) = &prebake_config.bootstrap
|
||||
&& bootstrap.user == "root"
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
let drop_after = get_drop_after(&prebake_config.security).unwrap_or_default();
|
||||
PrebakeStage::Never == drop_after
|
||||
}
|
||||
|
||||
@@ -12,14 +12,16 @@ bake/
|
||||
├── parser.rs # @decorator-aware shell script parser
|
||||
├── decorator.rs # Decorator enum + 100+ unit tests
|
||||
├── builder.rs # Script template assembly
|
||||
└── schedule.rs # Function ordering via @after dependencies
|
||||
├── schedule.rs # Function ordering via @after dependencies
|
||||
├── error.rs # BakeError enum
|
||||
└── constant.rs # DEFAULT_WORKSPACE, TRIVIAL_SCRIPT_NAME
|
||||
```
|
||||
|
||||
## WHERE TO LOOK
|
||||
|
||||
| Task | Location |
|
||||
|------|----------|
|
||||
| Parse script | `parser.rs:29` - `parse_script()` |
|
||||
Parse script | `parser.rs:56` - `parse_script()` |
|
||||
| Decorator types | `decorator.rs` - `Decorator` enum variants |
|
||||
| Build script | `builder.rs` - template builder |
|
||||
| Schedule deps | `schedule.rs` - @after resolution |
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use crate::bake::constant::{PIPELINE_SKIP_ERRORCODE, TRIVIAL_SCRIPT_NAME};
|
||||
use crate::bake::decorator::Decorator;
|
||||
use crate::bake::error::BakeError;
|
||||
use crate::bake::parser::Function;
|
||||
use crate::constant::TRIVIAL_SCRIPT_NAME;
|
||||
use minijinja::{Environment, context};
|
||||
use std::fs::Permissions;
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
@@ -8,12 +9,11 @@ use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
use tempfile::NamedTempFile;
|
||||
|
||||
use crate::constant::PIPELINE_SKIP_ERRORCODE;
|
||||
use crate::error::BakeError;
|
||||
|
||||
/// Template for use_template=false, which just executes the main body without any wrapping.
|
||||
const TRIVIAL_TEMPLATE: &str = "{{ main }}";
|
||||
|
||||
/// Renders a `Function` into an executable shell script using a minijinja template.
|
||||
/// If `use_template` is true, wraps the body in `bake_base`; otherwise uses a trivial template.
|
||||
pub fn build_script(
|
||||
function: &Function,
|
||||
bake_base: &Path,
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
pub const DEFAULT_WORKSPACE: &str = "/workspace";
|
||||
pub const BAKE_SCRIPT_PATH: &str = "bake.sh";
|
||||
pub const TRIVIAL_SCRIPT_NAME: &str = "BAKE_TRIVIAL";
|
||||
pub const PIPELINE_SKIP_ERRORCODE: u8 = 233;
|
||||
@@ -3,7 +3,7 @@ use regex::Regex;
|
||||
use std::fmt::Display;
|
||||
use std::fmt::Formatter;
|
||||
|
||||
use crate::error::BakeError;
|
||||
use crate::bake::error::BakeError;
|
||||
|
||||
lazy_static! {
|
||||
// "# @decorator" or "# @decorator(parameters)"
|
||||
@@ -15,20 +15,34 @@ lazy_static! {
|
||||
static ref INVALID_ARGUMENT: &'static str = "invalid argument";
|
||||
}
|
||||
|
||||
/// Pipeline decorator representing shell script annotations in `# @name(args)` syntax.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum Decorator {
|
||||
Unknown, // for general error reporting when the decorator name is not recognized
|
||||
/// Unrecognized decorator name (used for error reporting).
|
||||
Unknown,
|
||||
/// Marks a function as a main pipeline step.
|
||||
Pipeline,
|
||||
/// Conditional execution with shell condition.
|
||||
If(String),
|
||||
/// Allows the step to fail without halting the pipeline.
|
||||
Fallible,
|
||||
/// Repeat the function body N times.
|
||||
Loop(usize),
|
||||
/// Run after the specified function completes.
|
||||
After(String),
|
||||
/// Execution timeout in seconds.
|
||||
Timeout(u32),
|
||||
/// Retry N times with M second delay between attempts.
|
||||
Retry(usize, u32),
|
||||
/// Export environment variables to subsequent steps.
|
||||
Export,
|
||||
/// Chain functions via pipe with comma-separated names.
|
||||
Pipe(Vec<String>),
|
||||
/// Run concurrently with other parallel functions.
|
||||
Parallel,
|
||||
/// Run as a background daemon process.
|
||||
Daemon,
|
||||
/// Health check endpoint for daemon verification.
|
||||
Health(String),
|
||||
}
|
||||
|
||||
@@ -52,6 +66,8 @@ impl Display for Decorator {
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses a line for `# @name(args)` decorator syntax.
|
||||
/// Returns `Ok(None)` if the line is not a decorator line, `Ok(Some(Decorator))` on match.
|
||||
pub fn parse_decorator(line: &str, line_num: usize) -> Result<Option<Decorator>, BakeError> {
|
||||
let caps = match DECORATOR_REGEX.captures(line) {
|
||||
Some(c) => c,
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
use thiserror::Error;
|
||||
|
||||
use workshop_engine::{
|
||||
ExecutionError,
|
||||
error::{EXITCODE_IO_ERROR, EXITCODE_PARSE_ERROR, HasExitCode},
|
||||
};
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum BakeError {
|
||||
#[error("Failed to generate staged script {script}: {reason}")]
|
||||
ScriptGenerationFailed { script: String, reason: String },
|
||||
|
||||
#[error("Template error: {0}")]
|
||||
TemplateError(#[from] minijinja::Error),
|
||||
|
||||
#[error("Circular Dependency detected in bake.sh")]
|
||||
CircularDependency(Vec<String>),
|
||||
|
||||
#[error("Unknown @after of function {0}: {1}")]
|
||||
UnknownDependency(String, String),
|
||||
|
||||
#[error("IO Error: {0}")]
|
||||
IoError(#[from] std::io::Error),
|
||||
|
||||
#[error("YAML parse error: {0}")]
|
||||
YamlParseError(#[from] serde_yaml::Error),
|
||||
|
||||
#[error("Failed to parse decorator @{decorator} (line #{line_num}): {reason}")]
|
||||
DecoratorParseError {
|
||||
decorator: String,
|
||||
line_num: usize,
|
||||
reason: String,
|
||||
},
|
||||
|
||||
#[error("Script execution error: {0}")]
|
||||
ScriptExecutionError(#[from] ExecutionError),
|
||||
}
|
||||
|
||||
impl HasExitCode for BakeError {
|
||||
fn exit_code(&self) -> i32 {
|
||||
match self {
|
||||
BakeError::ScriptGenerationFailed { .. } => EXITCODE_IO_ERROR,
|
||||
BakeError::TemplateError(_) => EXITCODE_PARSE_ERROR,
|
||||
BakeError::CircularDependency(_) => EXITCODE_PARSE_ERROR,
|
||||
BakeError::UnknownDependency(..) => EXITCODE_PARSE_ERROR,
|
||||
BakeError::IoError(_) => EXITCODE_IO_ERROR,
|
||||
BakeError::YamlParseError(_) => EXITCODE_PARSE_ERROR,
|
||||
BakeError::DecoratorParseError { .. } => EXITCODE_PARSE_ERROR,
|
||||
BakeError::ScriptExecutionError(e) => e.exit_code(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
use super::decorator::{Decorator, parse_decorator};
|
||||
use crate::bake::BakeError;
|
||||
use crate::bake::error::BakeError;
|
||||
use lazy_static::lazy_static;
|
||||
use regex::Regex;
|
||||
use shlex::Shlex;
|
||||
@@ -12,14 +12,19 @@ lazy_static! {
|
||||
static ref DECORATOR_HEADER_REGEX: Regex = Regex::new(r"\s*#\s+@").unwrap();
|
||||
}
|
||||
|
||||
/// A parsed shell function with decorator annotations and body content.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Function {
|
||||
/// Function name (alphanumeric with underscore/hyphen).
|
||||
pub name: String,
|
||||
/// Decorators applied to this function.
|
||||
pub decorators: Vec<Decorator>,
|
||||
/// Raw function body including `function name() { ... }` syntax.
|
||||
pub body: String,
|
||||
}
|
||||
|
||||
impl Function {
|
||||
/// Finds the first decorator matching the given predicate.
|
||||
pub fn find_decorator<T>(&self, matcher: impl Fn(&Decorator) -> Option<T>) -> Option<T> {
|
||||
self.decorators.iter().find_map(matcher)
|
||||
}
|
||||
@@ -37,12 +42,17 @@ enum ParserState {
|
||||
},
|
||||
}
|
||||
|
||||
/// Result of parsing a shell script with decorator annotations.
|
||||
#[derive(Debug)]
|
||||
pub struct ParsedScript {
|
||||
/// Functions marked with `@pipeline` decorator.
|
||||
pub pipeline_functions: Vec<Function>,
|
||||
/// Code not belonging to any pipeline function (comments, helpers, etc.).
|
||||
pub remaining_code: String,
|
||||
}
|
||||
|
||||
/// Parses shell scripts with `# @decorator(args)` annotations into a `Function` list.
|
||||
/// Lines prefixed with `# @` before a function definition become decorators for that function.
|
||||
pub fn parse_script(text: &str) -> Result<ParsedScript, BakeError> {
|
||||
let mut pipeline_functions = Vec::new();
|
||||
let mut remaining_code = String::new();
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
use super::{decorator::Decorator, parser::Function};
|
||||
use crate::error::BakeError;
|
||||
use crate::bake::error::BakeError;
|
||||
use std::collections::HashMap;
|
||||
use topological_sort::TopologicalSort;
|
||||
|
||||
/// Topologically sorts functions by explicit `@after` dependencies and implicit ordering
|
||||
/// for consecutive `@pipeline` functions (each pipeline function implicitly depends on
|
||||
/// the previous one unless it has an explicit `@after`).
|
||||
pub fn sort_function(functions: Vec<Function>) -> Result<Vec<Function>, BakeError> {
|
||||
let function_mapped = functions
|
||||
.iter()
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
use crate::cli::{BareCommands, Cli};
|
||||
use crate::error::CliError;
|
||||
use crate::types::resource::ResourceRegistry;
|
||||
use crate::{bake::bake, finalize::finalize, prebake::prebake};
|
||||
use workshop_engine::{EventSender, ExecutionContext};
|
||||
fn or_workshop(path: &Option<PathBuf>, name: &str) -> PathBuf {
|
||||
path
|
||||
.clone()
|
||||
.unwrap_or_else(|| PathBuf::from(format!(".workshop/{}", name)))
|
||||
}
|
||||
|
||||
pub async fn run_bare(
|
||||
cli: &Cli,
|
||||
command: &BareCommands,
|
||||
ctx: &mut ExecutionContext,
|
||||
event_tx: EventSender,
|
||||
pipeline: &str,
|
||||
build_id: &str,
|
||||
registry: &ResourceRegistry,
|
||||
) -> Result<(), CliError> {
|
||||
let (stage, config) = match command {
|
||||
BareCommands::Prebake { .. } => ("prebake", or_workshop(&cli.prebake, "prebake.yml")),
|
||||
BareCommands::Bake { .. } => ("bake", or_workshop(&cli.bake, "bake.sh")),
|
||||
BareCommands::Finalize { .. } => {
|
||||
("finalize", or_workshop(&cli.finalize, "finalize.yml"))
|
||||
}
|
||||
};
|
||||
|
||||
let prebake_path = or_workshop(&cli.prebake, "prebake.yml");
|
||||
let bake_base = or_workshop(&cli.bake_base, "bake_base.sh");
|
||||
|
||||
ctx.task_id = format!("{}-{}-{}", pipeline, build_id, stage);
|
||||
ctx.pipeline_name = pipeline.to_string();
|
||||
ctx.standalone = true;
|
||||
|
||||
match command {
|
||||
BareCommands::Prebake { .. } => {
|
||||
prebake(&config, cli, None, ctx, event_tx, None).await?;
|
||||
}
|
||||
BareCommands::Bake { .. } => {
|
||||
bake(&config, &bake_base, &prebake_path, cli, ctx, event_tx).await?;
|
||||
}
|
||||
BareCommands::Finalize { .. } => {
|
||||
finalize(&config, cli, ctx, event_tx, registry).await?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
pub const DEFAULT_WORKSPACE: &str = "/workspace";
|
||||
pub const BOOTSTRAP_SCRIPT_PATH: &str = "/bootstrap.sh";
|
||||
pub const BAKE_SCRIPT_PATH: &str = "bake.sh";
|
||||
pub const PIPELINE_SKIP_ERRORCODE: u8 = 233;
|
||||
pub const TRIVIAL_SCRIPT_NAME: &str = "BAKE_TRIVIAL";
|
||||
pub const FINALIZE_PLUGIN_MANIFEST: &str = "manifest";
|
||||
pub const FINALIZE_PLUGIN_MANIFEST_EXTENSION: &[&str] = &[".yml", ".yaml"];
|
||||
pub const FINALIZE_SHELL_ENVPREFIX: &str = "HBW_ARG";
|
||||
pub const FINALIZE_STANDALONE_PLUGIN_PATH: &str = "/tmp/baker/plugin";
|
||||
|
||||
/// Standalone mode build status file directory
|
||||
pub const STANDALONE_STATUS_DIR: &str = "/tmp";
|
||||
/// Standalone mode build status file name
|
||||
pub const STANDALONE_STATUS_FILE: &str = ".hbwstatus";
|
||||
@@ -1,193 +0,0 @@
|
||||
use crate::cli::Cli;
|
||||
// use serde::Serialize;
|
||||
use serde_json::Value;
|
||||
// use std::path::PathBuf;
|
||||
// use std::sync::Arc;
|
||||
use tokio::io::{AsyncBufReadExt, BufReader};
|
||||
use tokio::net::UnixStream;
|
||||
// use tokio::process::{Child, Command};
|
||||
// use tokio::signal;
|
||||
// use tokio::sync::RwLock;
|
||||
// use tokio::time::{Duration, interval};
|
||||
|
||||
// #[derive(Debug, Clone, Serialize)]
|
||||
// struct Heartbeat {
|
||||
// pipeline_id: String,
|
||||
// build_id: String,
|
||||
// stage: String,
|
||||
// timestamp: u64,
|
||||
// status: String,
|
||||
// subtasks: Vec<String>,
|
||||
// }
|
||||
|
||||
pub async fn daemon(_cli: &Cli) {
|
||||
todo!("Daemon should be refactored");
|
||||
// let socket = PathBuf::from(settings.get_string("socket").unwrap());
|
||||
// let agentsocket = settings.get_string("agentsocket").unwrap();
|
||||
// let _ = std::fs::remove_file(&socket);
|
||||
|
||||
// let heartbeat = Arc::new(RwLock::new(Heartbeat {
|
||||
// pipeline_id: cli.pipeline.clone(),
|
||||
// build_id: cli.build_id.clone(),
|
||||
// stage: String::new(),
|
||||
// timestamp: 0,
|
||||
// status: String::new(),
|
||||
// subtasks: Vec::new(),
|
||||
// }));
|
||||
|
||||
// // let heartbeat_clone = heartbeat.clone();
|
||||
// let heartbeat_task = {
|
||||
// let hb = heartbeat.clone();
|
||||
// tokio::spawn(async move {
|
||||
// let mut tick = interval(Duration::from_secs(1));
|
||||
// loop {
|
||||
// tick.tick().await;
|
||||
// let mut hb = hb.write().await;
|
||||
// hb.timestamp = std::time::SystemTime::now()
|
||||
// .duration_since(std::time::UNIX_EPOCH)
|
||||
// .unwrap()
|
||||
// .as_millis() as u64;
|
||||
|
||||
// // 只输出到stdout(模拟未来发送给Agent)
|
||||
// println!("[HEARTBEAT] {}", serde_json::to_string(&*hb).unwrap());
|
||||
// }
|
||||
// })
|
||||
// };
|
||||
|
||||
// let socket_task = {
|
||||
// let socket = socket.clone();
|
||||
// tokio::spawn(async move {
|
||||
// log::info!("Creating Unix socket at {}", socket.display());
|
||||
// let listener =
|
||||
// UnixListener::bind(&socket).expect("Failed to bind Unix socket");
|
||||
|
||||
// loop {
|
||||
// match listener.accept().await {
|
||||
// Ok((stream, _)) => {
|
||||
// tokio::spawn(handle_connection(stream));
|
||||
// }
|
||||
// Err(e) => {
|
||||
// log::error!("Accept error: {}", e);
|
||||
// break;
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// })
|
||||
// };
|
||||
|
||||
// let mut tasks: Vec<TaskSpec> = Vec::new();
|
||||
|
||||
// let execution_task = {
|
||||
// let hb = heartbeat.clone();
|
||||
// tokio::spawn(async move {
|
||||
// let mut hb = hb.write().await;
|
||||
// let mut children: Vec<(String, Child)> = vec![];
|
||||
|
||||
// let executable = std::env::current_exe().unwrap();
|
||||
|
||||
// for (index, task) in tasks.iter().enumerate() {
|
||||
// let mut command = task.command(&executable);
|
||||
// command.stdout(std::process::Stdio::piped());
|
||||
// command.stderr(std::process::Stdio::piped());
|
||||
// let child = command
|
||||
// .spawn()
|
||||
// .expect(format!("Failed to spawn child {}", index).as_str());
|
||||
|
||||
// let pid = child.id().unwrap_or(0).to_string();
|
||||
// children.push((pid.clone(), child));
|
||||
|
||||
// hb.subtasks.push(pid.clone());
|
||||
// }
|
||||
|
||||
// for (pid, mut child) in children {
|
||||
// let status = child.wait().await.expect("Failed to wait child");
|
||||
// log::info!("Subtask {} exited with: {}", pid, status);
|
||||
|
||||
// hb.subtasks.retain(|p| p != &pid);
|
||||
// }
|
||||
|
||||
// hb.stage = "Success".to_string();
|
||||
// })
|
||||
// };
|
||||
|
||||
// tokio::select! {
|
||||
// _ = heartbeat_task => {
|
||||
// log::error!("Heartbeat task died");
|
||||
// }
|
||||
// _ = execution_task => {
|
||||
// log::info!("All subtasks completed");
|
||||
// }
|
||||
// _ = socket_task => {
|
||||
// log::error!("Socket task died");
|
||||
// }
|
||||
// _ = signal::ctrl_c() => {
|
||||
// log::info!("Shutting down...");
|
||||
// }
|
||||
// }
|
||||
// for pid in heartbeat.read().await.subtasks.clone() {
|
||||
// let _ = Command::new("kill").arg("-TERM").arg(&pid).status().await;
|
||||
// }
|
||||
|
||||
// let _ = std::fs::remove_file(&socket);
|
||||
}
|
||||
|
||||
async fn _handle_connection(stream: UnixStream) {
|
||||
let peer_addr = match stream.peer_addr() {
|
||||
Ok(addr) => format!("{:?}", addr),
|
||||
Err(_) => "unknown".to_string(),
|
||||
};
|
||||
|
||||
log::debug!("New connection from: {}", peer_addr);
|
||||
|
||||
let mut reader = BufReader::new(stream);
|
||||
let mut line = String::new();
|
||||
|
||||
loop {
|
||||
line.clear();
|
||||
match reader.read_line(&mut line).await {
|
||||
Ok(0) => {
|
||||
log::debug!("Connection closed by peer: {}", peer_addr);
|
||||
break;
|
||||
}
|
||||
Ok(_) => {
|
||||
// 简单打印原始JSON(预留字段解析空间)
|
||||
match serde_json::from_str::<Value>(&line) {
|
||||
Ok(json) => {
|
||||
// 只提取关键字段打印(字段可能变化)
|
||||
let msgtype = json
|
||||
.get("msgtype")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("unknown");
|
||||
let name = json
|
||||
.get("name")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("unnamed");
|
||||
|
||||
log::info!("[{}] Message from '{}': {}", msgtype, name, line.trim());
|
||||
|
||||
// TODO: 字段稳定后,可扩展为结构化打印
|
||||
// match msgtype {
|
||||
// "ResourceUsage" => print_resource(&json),
|
||||
// "Log" => print_log(&json),
|
||||
// _ => log::debug!("Unknown type: {}", msgtype),
|
||||
// }
|
||||
}
|
||||
Err(e) => {
|
||||
log::warn!(
|
||||
"Invalid JSON from {}: {} - Error: {}",
|
||||
peer_addr,
|
||||
line.trim(),
|
||||
e
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
log::error!("Failed to read from {}: {}", peer_addr, e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log::debug!("Connection handler for {} exiting", peer_addr);
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
//! Engine module - execution engine for workshop-baker
|
||||
//!
|
||||
//! This module provides the core execution functionality for running commands
|
||||
//! in isolated environments with resource management.
|
||||
|
||||
pub mod cgroups;
|
||||
pub mod executor;
|
||||
pub mod pm;
|
||||
pub mod repology;
|
||||
pub mod types;
|
||||
pub mod upm;
|
||||
|
||||
// Re-export commonly used types for convenience
|
||||
pub use types::{
|
||||
Engine, EventSender, ExecutionContext, ExecutionError, ExecutionEvent, ExecutionResult,
|
||||
ResourceLimits, ResourceUsage, StreamType,
|
||||
};
|
||||
@@ -1,46 +0,0 @@
|
||||
# workshop-baker/src/engine
|
||||
|
||||
**Generated:** 2026-04-22
|
||||
**Commit:** 7b3b71a
|
||||
|
||||
Execution runtime with resource isolation and package management.
|
||||
|
||||
## STRUCTURE
|
||||
|
||||
```
|
||||
engine/
|
||||
├── executor.rs # Script execution with cgroups, pipes, exit codes
|
||||
├── cgroups.rs # Linux cgroup resource limits (memory, CPU, IO)
|
||||
├── types.rs # ResourceLimits, ExecutionResult, ProcessConfig
|
||||
├── pm.rs # Package manager detection (apt, pacman, etc.)
|
||||
├── pm/ # PM implementations
|
||||
│ ├── apt.rs # Debian/Ubuntu (WIP - todo!() stubs)
|
||||
│ └── pacman.rs # Arch Linux
|
||||
├── upm.rs # User package managers (Nix/Guix)
|
||||
├── upm/custom.rs # Custom PM support (unimplemented)
|
||||
└── repology.rs # Repology API for package name resolution
|
||||
└── local.rs # Python-based local repology (non-standard)
|
||||
```
|
||||
|
||||
## WHERE TO LOOK
|
||||
|
||||
| Task | Location |
|
||||
|------|----------|
|
||||
| Execute script with limits | `executor.rs:Executor::execute_script()` |
|
||||
| Resource limits | `types.rs:56` - `ResourceLimits` struct |
|
||||
| Cgroup management | `cgroups.rs` - CgroupManager v1/v2 |
|
||||
| Package detection | `pm.rs:detection()` |
|
||||
| Repology lookup | `repology.rs` - RepologyEndpoint |
|
||||
|
||||
## CONVENTIONS
|
||||
|
||||
- **Executor**: Lightweight wrapper around CgroupManager for isolation
|
||||
- **ResourceLimits**: Memory, CPU, IO bounds passed to cgroups
|
||||
- **Package Manager Detection**: Auto-detects from `/etc/os-release`
|
||||
- **WIP**: apt.rs has `todo!()` stubs (lines 22, 45, 50)
|
||||
|
||||
## ANTI-PATTERNS
|
||||
|
||||
1. **Python in Rust** — `repology/local.rs` embeds Python repology client
|
||||
2. **Unsafe libc** — Multiple `unsafe { libc::geteuid() }` in security.rs (parent)
|
||||
3. **Unwrap heavy** — cgroups.rs has many `.unwrap()` calls
|
||||
@@ -1,104 +0,0 @@
|
||||
use std::fs;
|
||||
use std::io::{self, Write};
|
||||
use std::path::PathBuf;
|
||||
|
||||
pub struct CgroupManager {
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
impl CgroupManager {
|
||||
pub fn new(build_id: &str) -> io::Result<Self> {
|
||||
let path = PathBuf::from(format!("/sys/fs/cgroup/ws-executor/{}", build_id));
|
||||
|
||||
fs::create_dir_all(&path)?;
|
||||
|
||||
Ok(Self { path })
|
||||
}
|
||||
|
||||
pub fn set_memory_limit(&self, limit_bytes: u64) -> io::Result<()> {
|
||||
let mem_max_path = self.path.join("memory.max");
|
||||
let content = if limit_bytes == 0 {
|
||||
"max".to_string()
|
||||
} else {
|
||||
limit_bytes.to_string()
|
||||
};
|
||||
fs::write(&mem_max_path, content)?;
|
||||
|
||||
let mem_swap_path = self.path.join("memory.swap.max");
|
||||
fs::write(&mem_swap_path, "max")?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn set_cpu_weight(&self, weight: u64) -> io::Result<()> {
|
||||
let cpu_weight_path = self.path.join("cpu.weight");
|
||||
let weight = weight.clamp(1, 10000);
|
||||
fs::write(&cpu_weight_path, weight.to_string())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn set_cpu_max(&self, max_us: u64, period_us: u64) -> io::Result<()> {
|
||||
let cpu_max_path = self.path.join("cpu.max");
|
||||
let content = if max_us == 0 {
|
||||
"max".to_string()
|
||||
} else {
|
||||
format!("{} {}", max_us, period_us)
|
||||
};
|
||||
fs::write(&cpu_max_path, content)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn add_process(&self, pid: u32) -> io::Result<()> {
|
||||
let cgroup_procs_path = self.path.join("cgroup.procs");
|
||||
let mut file = fs::OpenOptions::new()
|
||||
.append(true)
|
||||
.open(&cgroup_procs_path)?;
|
||||
|
||||
writeln!(file, "{}", pid)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn destroy(&self) -> io::Result<()> {
|
||||
if self.path.exists() {
|
||||
fs::remove_dir(&self.path)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &PathBuf {
|
||||
&self.path
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
#[ignore]
|
||||
fn test_cgroup_manager_new() {
|
||||
let build_id = "test-build-123";
|
||||
let manager = CgroupManager::new(build_id).unwrap();
|
||||
assert!(manager.path().exists());
|
||||
manager.destroy().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore]
|
||||
fn test_set_memory_limit() {
|
||||
let build_id = "test-mem-123";
|
||||
let manager = CgroupManager::new(build_id).unwrap();
|
||||
manager.set_memory_limit(1024 * 1024 * 1024).unwrap();
|
||||
manager.destroy().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore]
|
||||
fn test_set_cpu_weight() {
|
||||
let build_id = "test-cpu-123";
|
||||
let manager = CgroupManager::new(build_id).unwrap();
|
||||
manager.set_cpu_weight(1024).unwrap();
|
||||
manager.destroy().unwrap();
|
||||
}
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
use os_info::Info;
|
||||
use serde_yaml::Value;
|
||||
use tokio::process::Command;
|
||||
// TODO: apt
|
||||
// mod apt;
|
||||
mod pacman;
|
||||
|
||||
pub trait PackageManager {
|
||||
/// Name of package manager
|
||||
fn name(&self) -> &'static str;
|
||||
|
||||
/// Adopt for a distro
|
||||
fn adopt(&self, distro: &Info) -> bool;
|
||||
|
||||
/// Binary name
|
||||
fn binary(&self) -> &'static str;
|
||||
|
||||
/// The command of updating
|
||||
fn update(&self) -> Vec<Command>;
|
||||
|
||||
/// The command of installing packages
|
||||
fn install(&self, package: &[String]) -> Vec<Command>;
|
||||
|
||||
// Check whether a package is installed
|
||||
// Currently not planned
|
||||
// Package manager should handle installed packages themselves
|
||||
// just as --needed in pacman
|
||||
// Also the PM module should generate command, not execute them
|
||||
// fn is_installed(&self, _package: &str) -> Option<bool> {unimplemented!()}
|
||||
|
||||
/// Change major mirror of package manager
|
||||
fn change_mirror(&self, repo: &str, osinfo: Option<&Info>) -> Vec<Command>;
|
||||
|
||||
/// add a custom repository
|
||||
fn add_repository(&self, repo: &Value, osinfo: Option<&Info>) -> Vec<Command>;
|
||||
}
|
||||
|
||||
pub fn all_managers() -> Vec<Box<dyn PackageManager>> {
|
||||
vec![
|
||||
// Box::new(apt::Apt),
|
||||
Box::new(pacman::Pacman),
|
||||
// Box::new(dnf::Dnf),
|
||||
]
|
||||
}
|
||||
|
||||
pub fn detect(distro: &os_info::Info) -> Option<Box<dyn PackageManager>> {
|
||||
all_managers().into_iter().find(|pm| pm.adopt(distro))
|
||||
}
|
||||
|
||||
pub fn select(manager: &str) -> Option<Box<dyn PackageManager>> {
|
||||
// TODO: allow specify PM
|
||||
// log::warn!("Manually select package manager is not recommended.");
|
||||
// log::warn!("Use at your own risk.");
|
||||
all_managers().into_iter().find(|pm| pm.name() == manager)
|
||||
}
|
||||
@@ -1,115 +0,0 @@
|
||||
use crate::engine::pm::PackageManager;
|
||||
use os_info::{Info, Version};
|
||||
use serde_yaml::Value;
|
||||
use tokio::process::Command;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Apt;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct AptRepositories {
|
||||
url: String,
|
||||
key: Option<String>,
|
||||
}
|
||||
|
||||
impl PackageManager for Apt {
|
||||
fn name(&self) -> &'static str {
|
||||
"apt"
|
||||
}
|
||||
|
||||
fn adopt(&self, distro: &Info) -> bool {
|
||||
return false;
|
||||
// todo!();
|
||||
}
|
||||
|
||||
fn binary(&self) -> &'static str {
|
||||
"apt"
|
||||
}
|
||||
|
||||
fn update(&self) -> Command {
|
||||
let mut cmd = Command::new("apt");
|
||||
cmd.arg("update");
|
||||
cmd
|
||||
}
|
||||
|
||||
fn install(&self, package: &[String]) -> Command {
|
||||
let mut cmd = Command::new("apt");
|
||||
cmd.arg("install");
|
||||
cmd.arg("-y"); // 自动确认
|
||||
cmd.args(package);
|
||||
cmd
|
||||
}
|
||||
|
||||
fn change_mirror(&self, repo: &str, osinfo: Option<&Info>) -> Command {
|
||||
|
||||
todo!();
|
||||
}
|
||||
|
||||
fn add_repository(&self, repo: &Value, osinfo: Option<&Info>) -> Vec<Command> {
|
||||
|
||||
todo!();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_update_command_contents() {
|
||||
let cmd = Apt.update();
|
||||
let cmd = cmd.as_std();
|
||||
assert_eq!(cmd.get_program().to_string_lossy(), "apt");
|
||||
let args: Vec<String> = cmd
|
||||
.get_args()
|
||||
.map(|a| a.to_string_lossy().into_owned())
|
||||
.collect();
|
||||
assert_eq!(args, vec!["update"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_install_command_contents() {
|
||||
let cmd = Apt.install(&["curl".to_string()]);
|
||||
let cmd = cmd.as_std();
|
||||
assert_eq!(cmd.get_program().to_string_lossy(), "apt");
|
||||
let args: Vec<String> = cmd
|
||||
.get_args()
|
||||
.map(|a| a.to_string_lossy().into_owned())
|
||||
.collect();
|
||||
assert_eq!(args, vec!["install", "-y", "curl"]);
|
||||
}
|
||||
}
|
||||
|
||||
/// 辅助函数:从 os_info 中提取 Codename
|
||||
fn extract_codename(info: Option<&Info>) -> Option<String> {
|
||||
let info = info?;
|
||||
|
||||
// 1. 直接获取 codename (os_info 较新版本支持)
|
||||
if let Some(codename) = info.codename() {
|
||||
if !codename.is_empty() {
|
||||
return Some(codename.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
// 2. 尝试从 version 中推断 (针对某些把 codename 放在 Custom 里的情况)
|
||||
match info.version() {
|
||||
Version::Custom(v) => {
|
||||
// 有些系统可能把 codename 放在这里,虽然不标准
|
||||
if !v.chars().next().map(|c| c.is_numeric()).unwrap_or(true) {
|
||||
return Some(v.clone());
|
||||
}
|
||||
}
|
||||
Version::Semantic(major, minor, _) => {
|
||||
// 对于 Ubuntu/Debian,如果没有 codename,有时可以用版本号兜底 (如 22.04)
|
||||
// 但 apt 源通常更喜欢 codename。如果实在没有,返回版本号字符串作为最后手段
|
||||
return Some(format!("{}.{}", major, minor));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
// 3. 如果 os_info 没提供,且是常见发行版,可以尝试根据 Type 猜测?
|
||||
// 不推荐硬编码猜测,因为版本太多。
|
||||
// 最好返回 None,让调用者决定是报错还是让用户手动指定。
|
||||
|
||||
None
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
pub mod local;
|
||||
|
||||
use crate::types::repology::RepologyEndpoint;
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum RepologyError {
|
||||
Unimplemented,
|
||||
RemoteNotSupported,
|
||||
}
|
||||
|
||||
pub async fn resolve_package_names(
|
||||
packages: &[String],
|
||||
mapped: bool,
|
||||
endpoint: &RepologyEndpoint,
|
||||
target_pm: &str,
|
||||
) -> Result<HashMap<String, Vec<String>>, RepologyError> {
|
||||
let mut result = HashMap::new();
|
||||
|
||||
if packages.is_empty() {
|
||||
return Ok(result);
|
||||
}
|
||||
|
||||
if mapped {
|
||||
result.insert(target_pm.to_string(), packages.to_vec());
|
||||
return Ok(result);
|
||||
}
|
||||
|
||||
match endpoint {
|
||||
RepologyEndpoint::Local | RepologyEndpoint::Default => {
|
||||
let resolved = local::resolve(packages, target_pm).await;
|
||||
if !resolved.is_empty() {
|
||||
result.insert(target_pm.to_string(), resolved);
|
||||
}
|
||||
}
|
||||
RepologyEndpoint::None => {
|
||||
result.insert(target_pm.to_string(), packages.to_vec());
|
||||
}
|
||||
RepologyEndpoint::Disabled => {}
|
||||
RepologyEndpoint::Remote | RepologyEndpoint::Server => {
|
||||
return Err(RepologyError::Unimplemented);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
use super::types::EventSender;
|
||||
use async_trait::async_trait;
|
||||
use serde_yaml::Value;
|
||||
mod custom;
|
||||
|
||||
use crate::{ExecutionContext, error::DependencyError};
|
||||
|
||||
pub struct UPMSysDeps {
|
||||
pub packages: Vec<String>,
|
||||
pub mapped: bool,
|
||||
}
|
||||
|
||||
impl UPMSysDeps {
|
||||
pub fn new(packages: Vec<String>, mapped: bool) -> Self {
|
||||
Self { packages, mapped }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub trait UserPackageManager {
|
||||
/// Name of package manager
|
||||
fn name(&self) -> &'static str;
|
||||
|
||||
/// Indicates that DepsSystem needs to install these packages
|
||||
fn system_dependency(&self, config: &Value) -> UPMSysDeps;
|
||||
|
||||
/// Main function of User Package Manager
|
||||
async fn main(
|
||||
&self,
|
||||
config: &Value,
|
||||
ctx: &ExecutionContext,
|
||||
event_tx: &EventSender,
|
||||
) -> Result<(), DependencyError>;
|
||||
}
|
||||
|
||||
pub fn all_managers() -> Vec<Box<dyn UserPackageManager>> {
|
||||
vec![Box::new(custom::Custom)]
|
||||
}
|
||||
|
||||
pub fn select(manager: &str) -> Option<Box<dyn UserPackageManager>> {
|
||||
all_managers().into_iter().find(|pm| pm.name() == manager)
|
||||
}
|
||||
+19
-153
@@ -1,164 +1,30 @@
|
||||
use os_info::Info;
|
||||
use std::path::PathBuf;
|
||||
use thiserror::Error;
|
||||
|
||||
pub const EXITCODE_OK: i32 = 0;
|
||||
pub const EXITCODE_GENERAL_ERROR: i32 = 1;
|
||||
pub const EXITCODE_INVALID_ARGS: i32 = 2;
|
||||
pub const EXITCODE_IO_ERROR: i32 = 3;
|
||||
pub const EXITCODE_PARSE_ERROR: i32 = 4;
|
||||
pub const EXITCODE_EXECUTION_ERROR: i32 = 5;
|
||||
pub const EXITCODE_TIMEOUT: i32 = 124;
|
||||
pub const EXITCODE_PRIV_DROP_FAILED: i32 = 201;
|
||||
use workshop_engine::HasExitCode;
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum BakeError {
|
||||
#[error("Failed to generate staged script {script}: {reason}")]
|
||||
ScriptGenerationFailed { script: String, reason: String },
|
||||
|
||||
#[error("Template error: {0}")]
|
||||
TemplateError(#[from] minijinja::Error),
|
||||
|
||||
#[error("Circular Dependency detected in bake.sh")]
|
||||
CircularDependency(Vec<String>),
|
||||
|
||||
#[error("Unknown @after of function {0}: {1}")]
|
||||
UnknownDependency(String, String),
|
||||
|
||||
#[error("IO Error: {0}")]
|
||||
IoError(#[from] std::io::Error),
|
||||
|
||||
#[error("YAML parse error: {0}")]
|
||||
YamlParseError(#[from] serde_yaml::Error),
|
||||
|
||||
#[error("Failed to parse decorator @{decorator} (line #{line_num}): {reason}")]
|
||||
DecoratorParseError {
|
||||
decorator: String,
|
||||
line_num: usize,
|
||||
reason: String,
|
||||
},
|
||||
|
||||
#[error("Script execution error: {0}")]
|
||||
ScriptExecutionError(#[from] ExecutionError),
|
||||
pub enum CliError {
|
||||
#[error("{0}")]
|
||||
Prebake(#[from] crate::prebake::error::PrebakeError),
|
||||
#[error("{0}")]
|
||||
Bake(#[from] crate::bake::error::BakeError),
|
||||
#[error("{0}")]
|
||||
Finalize(#[from] crate::finalize::FinalizeError),
|
||||
#[error("{0}")]
|
||||
General(anyhow::Error),
|
||||
}
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum PrebakeError {
|
||||
#[error("Privilege drop failed: {0}")]
|
||||
PrivDropFailed(String),
|
||||
|
||||
#[error("User not found: {0}")]
|
||||
UserNotFound(String),
|
||||
|
||||
#[error("Invalid privilege state: current_uid={current_uid}, target_uid={target_uid}")]
|
||||
InvalidPrivilegeState { current_uid: u32, target_uid: u32 },
|
||||
|
||||
#[error("Invalid stage: current={current}, expected={expected}")]
|
||||
InvalidStage { current: String, expected: String },
|
||||
|
||||
#[error("Failed to validate prebake.yml.")]
|
||||
ValidateError(),
|
||||
|
||||
#[error("IO error: {0}")]
|
||||
IoError(#[from] std::io::Error),
|
||||
|
||||
#[error("YAML parse error: {0}")]
|
||||
YamlParseError(#[from] serde_yaml::Error),
|
||||
|
||||
#[error("Bad prebake.yml: {0}")]
|
||||
ConfigError(String),
|
||||
impl From<anyhow::Error> for CliError {
|
||||
fn from(e: anyhow::Error) -> Self {
|
||||
CliError::General(e)
|
||||
}
|
||||
}
|
||||
|
||||
impl PrebakeError {
|
||||
impl CliError {
|
||||
pub fn exit_code(&self) -> i32 {
|
||||
match self {
|
||||
PrebakeError::PrivDropFailed(_) => EXITCODE_PRIV_DROP_FAILED,
|
||||
PrebakeError::UserNotFound(_) => EXITCODE_PRIV_DROP_FAILED,
|
||||
PrebakeError::InvalidPrivilegeState { .. } => EXITCODE_PRIV_DROP_FAILED,
|
||||
PrebakeError::ValidateError() => EXITCODE_PARSE_ERROR,
|
||||
PrebakeError::InvalidStage { .. } => EXITCODE_PRIV_DROP_FAILED,
|
||||
PrebakeError::IoError(_) => EXITCODE_IO_ERROR,
|
||||
PrebakeError::YamlParseError(_) => EXITCODE_PARSE_ERROR,
|
||||
PrebakeError::ConfigError(_) => EXITCODE_PARSE_ERROR,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum BootstrapError {
|
||||
#[error("Failed to write bootstrap script: {0}")]
|
||||
IOError(#[from] std::io::Error),
|
||||
|
||||
#[error("Failed to find bootstrap.sh: {0}")]
|
||||
ScriptNotFound(PathBuf),
|
||||
|
||||
#[error("Failed to execute bootstrap.sh: {0}")]
|
||||
ExecutionError(#[from] ExecutionError),
|
||||
|
||||
#[error("Failed to create user {0}: {1}")]
|
||||
UserCreationFailed(String, String),
|
||||
}
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum DependencyError {
|
||||
#[error("No supported package manager found for {0}")]
|
||||
UnsupportedPlatform(Info),
|
||||
|
||||
#[error("Unknown package manager: {0}")]
|
||||
UnknownManager(String),
|
||||
|
||||
#[error("Failed to parse config")]
|
||||
ParseError(),
|
||||
|
||||
#[error("Stage {stage} failed: {source}")]
|
||||
StageFailed {
|
||||
stage: &'static str,
|
||||
#[source]
|
||||
source: ExecutionError,
|
||||
},
|
||||
|
||||
#[error("Custom command execution failed: {0}")]
|
||||
CustomFailed(#[from] ExecutionError),
|
||||
}
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum ExecutionError {
|
||||
#[error("Invalid command: {0}")]
|
||||
InvalidCommand(String),
|
||||
|
||||
#[error("Execution failed: {0}")]
|
||||
ExecutionFailed(String),
|
||||
|
||||
#[error("Permission denied: uid={euid}, gid={egid}: {reason}")]
|
||||
PermissionDenied {
|
||||
euid: u32,
|
||||
egid: u32,
|
||||
reason: String,
|
||||
},
|
||||
|
||||
#[error("Timeout")]
|
||||
Timeout,
|
||||
|
||||
#[error("IO error: {0}")]
|
||||
IoError(String),
|
||||
|
||||
#[error("Script not found: {0}")]
|
||||
ScriptNotFound(PathBuf),
|
||||
|
||||
#[error("Package manager not found")]
|
||||
PackageManagerNotFound,
|
||||
}
|
||||
|
||||
impl ExecutionError {
|
||||
pub fn exit_code(&self) -> i32 {
|
||||
match self {
|
||||
ExecutionError::InvalidCommand(_) => EXITCODE_EXECUTION_ERROR,
|
||||
ExecutionError::ExecutionFailed(_) => EXITCODE_EXECUTION_ERROR,
|
||||
ExecutionError::PermissionDenied { .. } => EXITCODE_GENERAL_ERROR,
|
||||
ExecutionError::Timeout => EXITCODE_TIMEOUT,
|
||||
ExecutionError::IoError(_) => EXITCODE_IO_ERROR,
|
||||
ExecutionError::ScriptNotFound(_) => EXITCODE_INVALID_ARGS,
|
||||
ExecutionError::PackageManagerNotFound => EXITCODE_GENERAL_ERROR,
|
||||
CliError::Prebake(e) => e.exit_code(),
|
||||
CliError::Bake(e) => e.exit_code(),
|
||||
CliError::Finalize(e) => e.exit_code(),
|
||||
CliError::General(_) => 1,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,23 +1,18 @@
|
||||
pub mod config;
|
||||
pub mod constant;
|
||||
pub mod error;
|
||||
pub mod event;
|
||||
pub mod plugin;
|
||||
pub mod stage;
|
||||
pub mod template;
|
||||
pub mod types;
|
||||
|
||||
use crate::cli::Cli;
|
||||
use crate::constant::FINALIZE_STANDALONE_PLUGIN_PATH;
|
||||
use crate::constant::STANDALONE_STATUS_DIR;
|
||||
use crate::engine::EventSender;
|
||||
use crate::finalize::plugin::fetch::{self, FetchArgument};
|
||||
use crate::types::buildstatus::{BuildStatus, StageResult, read_build_status};
|
||||
use crate::types::buildstatus::{StageInfo, StagePhase};
|
||||
use chrono::Utc;
|
||||
use std::path::PathBuf;
|
||||
// use crate::finalize::error::PluginError;
|
||||
use crate::types::resource::ResourceRegistry;
|
||||
pub use config::*;
|
||||
pub use error::FinalizeError;
|
||||
use std::path::Path;
|
||||
use workshop_engine::EventSender;
|
||||
|
||||
pub fn parse(path: &Path) -> Result<FinalizeConfig, FinalizeError> {
|
||||
let content = std::fs::read_to_string(path).map_err(FinalizeError::IoError)?;
|
||||
@@ -27,9 +22,10 @@ pub fn parse(path: &Path) -> Result<FinalizeConfig, FinalizeError> {
|
||||
|
||||
pub async fn finalize(
|
||||
finalize_path: &Path,
|
||||
cli: &Cli,
|
||||
ctx: &mut crate::ExecutionContext,
|
||||
_cli: &Cli,
|
||||
ctx: &mut workshop_engine::ExecutionContext,
|
||||
event_tx: EventSender,
|
||||
registry: &ResourceRegistry,
|
||||
) -> Result<(), FinalizeError> {
|
||||
let mut finalize = parse(finalize_path)?;
|
||||
let validate_result = finalize.validate();
|
||||
@@ -39,33 +35,10 @@ pub async fn finalize(
|
||||
}
|
||||
return Err(FinalizeError::ValidateError(errors));
|
||||
};
|
||||
if ctx.standalone {
|
||||
// In standalone mode, finalize should download plugin by itself
|
||||
log::debug!("Downloading {} plugins", finalize.plugin.len());
|
||||
let plugin_path = PathBuf::from(FINALIZE_STANDALONE_PLUGIN_PATH);
|
||||
// if plugin_path.exists() {
|
||||
// std::fs::remove_dir_all(&plugin_path).map_err(FinalizeError::IoError)?;
|
||||
// }
|
||||
for (name, data) in finalize.plugin.iter_mut() {
|
||||
log::info!("Downloading plugin {}", name);
|
||||
let (source, config) = data.iter_mut().next().unwrap_or_else(|| {
|
||||
panic!(
|
||||
"Internal Error: Plugin {} has no source configured, this should have been caught by validation!",
|
||||
name
|
||||
)
|
||||
});
|
||||
let fetch_argument = FetchArgument {
|
||||
checksum: config.checksum.clone(),
|
||||
shallow: config.shallow,
|
||||
};
|
||||
fetch::fetch_plugin(&source, &name, &plugin_path, fetch_argument).await?;
|
||||
config.runtime.fspath = Some(plugin_path.join(name));
|
||||
}
|
||||
}
|
||||
|
||||
// Register Plugins
|
||||
// Register Plugins (paths resolved from registry)
|
||||
log::info!("Registering {} plugins", finalize.plugin.len());
|
||||
let registered_plugins = plugin::register(finalize.plugin, None)?;
|
||||
let registered_plugins = plugin::register(finalize.plugin, None, &mut finalize.notification, registry)?;
|
||||
|
||||
// Early hook
|
||||
let earlyhook_result = stage::hook::hook(
|
||||
@@ -76,13 +49,9 @@ pub async fn finalize(
|
||||
)
|
||||
.await;
|
||||
|
||||
// Read build status from prior stages
|
||||
let build_status = read_build_status(STANDALONE_STATUS_DIR).unwrap_or_default();
|
||||
let final_result = build_status.final_result();
|
||||
|
||||
// Determine if we should continue with artifacts/latehook
|
||||
// If prior stages failed, we still send notification but skip artifacts
|
||||
let prior_failed = build_status.has_failures() || earlyhook_result.is_err();
|
||||
let _prior_failed = earlyhook_result.is_err();
|
||||
|
||||
// Prepare artifacts
|
||||
// TODO: artifact stage implementation
|
||||
@@ -96,20 +65,5 @@ pub async fn finalize(
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Write finalize success status
|
||||
let _ = crate::types::buildstatus::write_stage_status(
|
||||
STANDALONE_STATUS_DIR,
|
||||
StageInfo {
|
||||
name: "finalize".to_string(),
|
||||
phase: StagePhase::Finalize,
|
||||
substage: "clean".to_string(),
|
||||
result: StageResult::Success,
|
||||
duration_ms: 0,
|
||||
started_at: Utc::now(),
|
||||
finished_at: Utc::now(),
|
||||
error_message: None,
|
||||
},
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user