diff --git a/.gitignore b/.gitignore index 6b2d7e4..15eac4f 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,12 @@ archived *.bak *.clean session* +llm.env + +# System test artifacts +tests/results/ +tests/__pycache__/ +tests/**/__pycache__/ +tests/.pytest_cache/ +tests/llm/.cache/ +*.pyc diff --git a/runner.sh b/runner.sh new file mode 100755 index 0000000..a849795 --- /dev/null +++ b/runner.sh @@ -0,0 +1,112 @@ +#!/bin/bash +set -euo pipefail + +# ============================================================================= +# HoneyBiscuitWorkshop System Test Runner +# ============================================================================= +# Usage: ./runner.sh [auto|manual] [-- pytest-args...] +# +# Modes: +# auto - Build, run all tests, collect reports, cleanup (default) +# manual - Build, start container shell for interactive testing +# +# Examples: +# ./runner.sh # Run all tests automatically +# ./runner.sh auto -k test_prebake # Run only prebake tests +# ./runner.sh manual # Drop into container shell +# ============================================================================= + +MODE="${1:-auto}" +shift 2>/dev/null || true +PYTEST_ARGS="$@" + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +STAGING_DIR="/tmp/hbw-system-test" +IMAGE_NAME="hbw-system-test:latest" + +# ---- Cleanup handler ---- +cleanup() { + echo "[runner] Cleaning up..." + docker rm -f hbw-system-test-container 2>/dev/null || true + rm -rf "$STAGING_DIR" +} +trap cleanup EXIT + +# ---- Step 1: Build the binary ---- +build_binary() { + echo "[runner] Building workshop-baker (release)..." + cd "$SCRIPT_DIR/workshop-baker" + cargo build --release 2>&1 + echo "[runner] Build complete." +} + +# ---- Step 2: Prepare staging directory ---- +prepare_staging() { + echo "[runner] Preparing staging directory..." + rm -rf "$STAGING_DIR" + mkdir -p "$STAGING_DIR" + + # Copy binary + cp "$SCRIPT_DIR/workshop-baker/target/release/workshop-baker" "$STAGING_DIR/" + chmod +x "$STAGING_DIR/workshop-baker" + + # Copy examples if they exist + if [ -d "$SCRIPT_DIR/workshop-baker/examples" ]; then + cp -r "$SCRIPT_DIR/workshop-baker/examples" "$STAGING_DIR/examples" + else + mkdir -p "$STAGING_DIR/examples" + fi + + # Copy llm.env if it exists + if [ -f "$SCRIPT_DIR/llm.env" ]; then + cp "$SCRIPT_DIR/llm.env" "$STAGING_DIR/llm.env" + fi + + echo "[runner] Staging ready at $STAGING_DIR" +} + +# ---- Step 3: Build Docker image ---- +build_image() { + echo "[runner] Building Docker image: $IMAGE_NAME ..." + docker build -t "$IMAGE_NAME" -f "$SCRIPT_DIR/tests/Dockerfile" "$SCRIPT_DIR" + echo "[runner] Image built: $IMAGE_NAME" +} + +# ---- Step 4: Run container ---- +run_container() { + local mode="$1" + echo "[runner] Starting container in $mode mode..." + + mkdir -p "$SCRIPT_DIR/tests/results" + + if [ "$mode" = "manual" ]; then + docker run -it --rm \ + --name hbw-system-test-container \ + --privileged \ + -v "$STAGING_DIR:/workshop:ro" \ + -v "$SCRIPT_DIR/tests/results:/results" \ + -e TEST_MODE=manual \ + "$IMAGE_NAME" + else + docker run --rm \ + --name hbw-system-test-container \ + --privileged \ + -v "$STAGING_DIR:/workshop:ro" \ + -v "$SCRIPT_DIR/tests/results:/results" \ + -e TEST_MODE=auto \ + "$IMAGE_NAME" $PYTEST_ARGS + fi +} + +# ---- Main flow ---- +echo "============================================" +echo " HoneyBiscuitWorkshop System Test Runner" +echo " Mode: $MODE" +echo "============================================" + +build_binary +prepare_staging +build_image +run_container "$MODE" + +echo "[runner] Done." diff --git a/templates/finalize.yml.tmpl b/templates/finalize.yml.tmpl index e8ee493..cd7b97c 100644 --- a/templates/finalize.yml.tmpl +++ b/templates/finalize.yml.tmpl @@ -85,86 +85,67 @@ plugin: # 建议将优先级设为 0、1、2... 等连续值 notification: + templates: + sms-tmpl1: + on_success: "Build success!" + on_failure: "Build failed!" + on_finish_of: "Build stage {{ build.status }} finished." + sms-tmpl2: + use: "git://https://gitee.com/example_user/example_template_for_sms.git@a1b2c3d4" + # 最高优先级组 - 首选通知方式 0: # Webhook 通知 webhook: url: "https://example.com/hook" - # schema 定义负载格式: - # - "default": 使用插件内置的默认格式 - # - "custom": 自定义格式,需要提供 on_* 模板 - # 当设置 on_success/on_failure 时,schema 隐式为 "custom" - schema: "default" + template: + # schema 定义负载格式: + # - "default": 使用系统(ws-base)指定的默认格式,standalone模式不可用,client模式的默认 + # - "fallback": 使用插件指定的默认格式,standalone模式的默认 + # - "custom": 自定义格式,需要提供全部 on_* 模板 + # - "": 使用templates节中定义的名字 + # 当任何on_*被设置,则等效于replace模式:schema做基底,on_*做替换 + schema: "default" # Satori 协议通知(如 Koishi) satori: - url: "http://127.0.0.1:5140/satori/v1/message.create" - token: "Bearer {{ secret.koishi_token }}" - schema: "custom" - - # on_success: 构建成功时发送的内容 - # 支持模板变量: - # {{ build.status }} - 构建状态 - # {{ build.duration }} - 构建耗时 - # {{ target.xxx }} - 目标相关变量 - on_success: | - { - "channel_id": "{{ target.group_id }}", - "content": "🍯 HBW 构建完成\n状态: {{ build.status }}\n耗时: {{ build.duration }}" - } - - # on_failure: 构建失败时发送的内容 - on_failure: | - { - "channel_id": "{{ target.group_id }}", - "content": "🍯 HBW 构建失败!\n状态: {{ build.status }}\n耗时: {{ build.duration }}" - } - - # on_finish_of: 在指定阶段完成时发送通知 - # 可用于在构建过程中发送进度通知 - # 阶段命名规则: - # - prebake/finalize: 第二字段为静态名称,如 "prebake.ready" - # - bake: 第二字段为 bake.sh 中的函数名,如 "bake.build" - # 每个条件有独立的 content,会在对应时机发送 - on_finish_of: - stage: "prebake.ready" - content: | - { - "channel_id": "{{ target.group_id }}", - "content": "🍯 HBW 准备完毕\n状态: {{ build.status }}\n耗时: {{ build.duration }}" - } + # TODO: 需要大量重构 # 次优先级组 - 备用通知方式(当优先级 0 中任一方式失败时触发) 1: # 邮件通知(内置插件) mail: to: "user@gmail.com" - from: - host: "mail.163.com" - mode: "smtp" - # 加密模式: - # - "TLS": 使用 TLS 加密(端口 465) - # - "StartTLS": 使用 STARTTLS(端口 587) - # - "no": 不加密(不推荐) - secure: "TLS" - username: {{ variable.mail.user }} - password: {{ secret.mail.password }} + # TODO: 替换为实际mail里的字段 schema: "default" # 插件也可以直接作为通知方式使用 # 需要插件支持通知触发器(on_success/on_failure/on_finish_of) sms-notify: to: "+8610012345678" - schema: "custom" policy: - - on_success: "[HBW Notify] Build Succeeded: {{ build.status }} ({{ build.duration }})" + - trigger: + - on_success + template: + schema: "custom" + on_success: "[HBW Notify] Build Succeeded: {{ build.status }} ({{ build.duration }})" to: "+8610012345678" - - on_failure: "[HBW Notify] Build Failed: {{ build.status }} ({{ build.duration }})" + - trigger: + - on_success + - on_failure + template: + schema: "sms-tmpl1" + # on_success: "[HBW Notify] Build Succeeded: {{ build.status }} ({{ build.duration }})" + # on_failure: "[HBW Notify] Build Failed: {{ build.status }} ({{ build.duration }})" to: "+8610012345679" - - - on_finish_of: - stage: "bake.build" - content: "[HBW Notify] Build Progress: {{ build.status }} ({{ build.duration }})" + - trigger: + - on_finish_of: + - "bake.build" + - "bake.clean.*" + - 're:bake\.prepare\.*' + template: + schema: "custom" + on_finish_of: "[HBW Notify] Build Progress: {{ build.status }} ({{ build.duration }})" to: "+8610012345677" fallible: true diff --git a/tests/Dockerfile b/tests/Dockerfile new file mode 100644 index 0000000..22357da --- /dev/null +++ b/tests/Dockerfile @@ -0,0 +1,32 @@ +FROM archlinux:latest + +RUN sed -i 's|https://geo.mirror.pkgbuild.com|https://mirrors.tuna.tsinghua.edu.cn/archlinux|g' /etc/pacman.d/mirrorlist && \ + sed -i 's|https://mirror.rackspace.com/archlinux|https://mirrors.tuna.tsinghua.edu.cn/archlinux|g' /etc/pacman.d/mirrorlist && \ + echo 'Server = https://mirrors.tuna.tsinghua.edu.cn/archlinux/$repo/os/$arch' >> /etc/pacman.d/mirrorlist + +RUN pacman -Syu --noconfirm && \ + pacman -S --noconfirm \ + python python-pip base-devel git curl wget sudo \ + which procps-ng shadow && \ + pacman -Scc --noconfirm + +# Create test user +RUN useradd -m -s /bin/bash testuser + +# Python environment +WORKDIR /app +COPY tests/requirements.txt ./requirements.txt +RUN python -m venv /opt/test-venv && \ + /opt/test-venv/bin/pip install -r requirements.txt + +# Copy test suite +COPY tests/ ./tests/ +COPY tests/entrypoint.sh ./entrypoint.sh +RUN chmod +x entrypoint.sh + +# Environment +ENV PATH="/opt/test-venv/bin:$PATH" +ENV PYTHONUNBUFFERED=1 +ENV PYTHONDONTWRITEBYTECODE=1 + +ENTRYPOINT ["./entrypoint.sh"] diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..23ebd55 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,175 @@ +""" +Root conftest for HoneyBiscuitWorkshop system tests. + +Provides core fixtures for: +- Locating the baker binary and examples +- Running subprocess commands with output capture +- Managing temporary working directories +""" +import os +import subprocess +import shutil +import tempfile +import pytest + + +def pytest_configure(config): + """Register custom markers.""" + config.addinivalue_line("markers", "prebake: prebake stage tests") + config.addinivalue_line("markers", "bake: bake stage tests") + config.addinivalue_line("markers", "finalize: finalize stage tests") + config.addinivalue_line("markers", "llm: tests requiring LLM API") + config.addinivalue_line("markers", "slow: slow running tests") + + +@pytest.fixture(scope="session") +def baker_bin(): + """Path to the workshop-baker binary.""" + path = os.environ.get("BAKER_BIN", "/workshop/workshop-baker") + if not os.path.isfile(path): + pytest.skip(f"Baker binary not found at {path}") + if not os.access(path, os.X_OK): + pytest.skip(f"Baker binary not executable at {path}") + return path + + +@pytest.fixture(scope="session") +def examples_dir(): + """Path to the examples directory (may be None).""" + path = os.environ.get("EXAMPLES_DIR", "/workshop/examples") + if not os.path.isdir(path): + return None + return path + + +@pytest.fixture(scope="session") +def llm_env(): + """LLM environment variables dict (from llm.env).""" + env = {} + for var in [ + "OPENAI_BASE_URL", + "OPENAI_API_KEY", + "ANTHROPIC_BASE_URL", + "ANTHROPIC_API_KEY", + "MODEL", + "REASONING_EFFORT", + ]: + val = os.environ.get(var) + if val: + env[var] = val + return env + + +@pytest.fixture +def work_dir(): + """Create a temporary working directory for a test, cleaned up after.""" + d = tempfile.mkdtemp(prefix="hbw-test-") + yield d + shutil.rmtree(d, ignore_errors=True) + + +@pytest.fixture +def run_cmd(): + """Fixture that provides a command runner function. + + Returns a function that runs a command and returns + (stdout, stderr, returncode, success). + """ + + def _run(args, cwd=None, env=None, timeout=120): + """Run a command and capture output. + + Args: + args: Command as list of strings. + cwd: Working directory. + env: Environment variables (merged with os.environ). + timeout: Timeout in seconds. + + Returns: + dict with keys: stdout, stderr, returncode, success + """ + full_env = {**os.environ} + if env: + full_env.update(env) + + try: + result = subprocess.run( + args, + cwd=cwd, + env=full_env, + capture_output=True, + text=True, + timeout=timeout, + ) + return { + "stdout": result.stdout, + "stderr": result.stderr, + "returncode": result.returncode, + "success": result.returncode == 0, + } + except subprocess.TimeoutExpired: + return { + "stdout": "", + "stderr": f"Command timed out after {timeout}s", + "returncode": -1, + "success": False, + } + except Exception as e: + return { + "stdout": "", + "stderr": str(e), + "returncode": -1, + "success": False, + } + + return _run + + +@pytest.fixture +def run_baker(baker_bin, run_cmd): + """Fixture that provides a baker command runner. + + Returns a function that runs workshop-baker with common flags. + """ + + def _run(subcommand, *args, cwd=None, env=None, timeout=120): + """Run workshop-baker with a subcommand. + + Args: + subcommand: Baker subcommand (prebake, bake, finalize). + *args: Additional arguments to the subcommand. + cwd: Working directory. + env: Extra environment variables. + timeout: Timeout in seconds. + + Returns: + dict with keys: stdout, stderr, returncode, success + """ + cmd = [ + baker_bin, + "-u", "testuser", + "-p", "test-pipeline", + "-b", "test-build-001", + subcommand, + ] + cmd.extend(args) + return run_cmd(cmd, cwd=cwd, env=env, timeout=timeout) + + return _run + + +@pytest.fixture +def fixtures_dir(): + """Path to the test fixtures directory.""" + return os.path.join(os.path.dirname(__file__), "fixtures") + + +@pytest.fixture +def bake_workspace(): + """Create /workspace for bake tests (baker writes temp scripts there).""" + existed = os.path.exists("/workspace") + if not existed: + os.makedirs("/workspace", exist_ok=True) + yield + if not existed: + shutil.rmtree("/workspace", ignore_errors=True) diff --git a/tests/entrypoint.sh b/tests/entrypoint.sh new file mode 100755 index 0000000..42563b4 --- /dev/null +++ b/tests/entrypoint.sh @@ -0,0 +1,36 @@ +#!/bin/bash +set -e + +# Load LLM environment if available +if [ -f /workshop/llm.env ]; then + set -a + source /workshop/llm.env + set +a +fi + +# Export test configuration +export BAKER_BIN=/workshop/workshop-baker +export EXAMPLES_DIR=/workshop/examples +export TEST_MODE=${TEST_MODE:-auto} + +if [ "$TEST_MODE" = "manual" ]; then + echo "========================================" + echo " HoneyBiscuitWorkshop System Tests" + echo " Mode: MANUAL (interactive shell)" + echo "========================================" + echo "" + echo "Binary: $BAKER_BIN" + echo "Examples: $EXAMPLES_DIR" + echo "" + echo "Run tests manually:" + echo " cd /app && pytest tests/ -v" + echo "" + exec /bin/bash +else + echo "========================================" + echo " HoneyBiscuitWorkshop System Tests" + echo " Mode: AUTO (running pytest)" + echo "========================================" + cd /app + exec python -m pytest tests/ -v --tb=short --junitxml=/results/report.xml "$@" +fi diff --git a/tests/fixtures/bake/bake_base.sh b/tests/fixtures/bake/bake_base.sh new file mode 100644 index 0000000..7108dfa --- /dev/null +++ b/tests/fixtures/bake/bake_base.sh @@ -0,0 +1,6 @@ +#!/bin/bash +[ -f /dev/shm/bakeexport.{{ name }} ] && . /dev/shm/bakeexport.{{ name }} || true +{{ condition }} +{{ main }} +{{ name }} +{{ export }} diff --git a/tests/fixtures/bake/pipeline.sh b/tests/fixtures/bake/pipeline.sh new file mode 100644 index 0000000..8ffc060 --- /dev/null +++ b/tests/fixtures/bake/pipeline.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# @pipeline +step_one() { + echo "Step 1: Setup" + mkdir -p /tmp/bake-test-output + echo "step1-done" > /tmp/bake-test-output/step1.txt +} + +# @pipeline +step_two() { + echo "Step 2: Build" + echo "step2-done" > /tmp/bake-test-output/step2.txt +} + +# @pipeline +step_three() { + echo "Step 3: Verify" + if [ -f /tmp/bake-test-output/step1.txt ] && [ -f /tmp/bake-test-output/step2.txt ]; then + echo "All steps completed successfully" + else + echo "ERROR: Missing step outputs" + exit 1 + fi +} diff --git a/tests/fixtures/bake/simple.sh b/tests/fixtures/bake/simple.sh new file mode 100644 index 0000000..3f51aa4 --- /dev/null +++ b/tests/fixtures/bake/simple.sh @@ -0,0 +1,5 @@ +#!/bin/bash +echo "Hello from simple bake script" +echo "Task ID: $HBW_TASKID" +echo "Pipeline: $HBW_PIPELINE" +exit 0 diff --git a/tests/fixtures/bake/with_after.sh b/tests/fixtures/bake/with_after.sh new file mode 100644 index 0000000..b7c70af --- /dev/null +++ b/tests/fixtures/bake/with_after.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# @pipeline +init_workspace() { + echo "Initializing workspace" + mkdir -p /tmp/bake-after-test + echo "init" > /tmp/bake-after-test/status.txt +} + +# @pipeline +# @after(init_workspace) +fetch_source() { + echo "Fetching source (depends on init_workspace)" + echo "fetched" > /tmp/bake-after-test/source.txt +} + +# @pipeline +# @after(fetch_source) +build_project() { + echo "Building project (depends on fetch_source)" + echo "built" > /tmp/bake-after-test/build.txt +} diff --git a/tests/fixtures/finalize/hook_early.sh b/tests/fixtures/finalize/hook_early.sh new file mode 100644 index 0000000..02d15c5 --- /dev/null +++ b/tests/fixtures/finalize/hook_early.sh @@ -0,0 +1,4 @@ +#!/bin/bash +echo "EARLY_HOOK_EXECUTED" +echo "Working directory: $(pwd)" +echo "User: $(whoami)" diff --git a/tests/fixtures/finalize/hook_late.sh b/tests/fixtures/finalize/hook_late.sh new file mode 100644 index 0000000..a13290d --- /dev/null +++ b/tests/fixtures/finalize/hook_late.sh @@ -0,0 +1,3 @@ +#!/bin/bash +echo "LATE_HOOK_EXECUTED" +echo "Working directory: $(pwd)" diff --git a/tests/fixtures/finalize/minimal.yml b/tests/fixtures/finalize/minimal.yml new file mode 100644 index 0000000..e35f838 --- /dev/null +++ b/tests/fixtures/finalize/minimal.yml @@ -0,0 +1,6 @@ +version: "0.0.1" +plugin: {} +notification: {} +artifact: [] +cleanup: + policy: "auto" diff --git a/tests/fixtures/finalize/with_hooks.yml b/tests/fixtures/finalize/with_hooks.yml new file mode 100644 index 0000000..8cace4d --- /dev/null +++ b/tests/fixtures/finalize/with_hooks.yml @@ -0,0 +1,13 @@ +version: "0.0.1" +plugin: {} +notification: {} +artifact: [] +cleanup: + policy: "auto" +hooks: + early: + - name: "finalize-early" + command: "echo 'finalize early hook'" + late: + - name: "finalize-late" + command: "echo 'finalize late hook'" diff --git a/tests/fixtures/prebake/bootstrap_only.yml b/tests/fixtures/prebake/bootstrap_only.yml new file mode 100644 index 0000000..34a1b43 --- /dev/null +++ b/tests/fixtures/prebake/bootstrap_only.yml @@ -0,0 +1,8 @@ +version: "1.0" +environment: + builder: "baremetal" +bootstrap: + user: "vulcan" + workspace: + path: "/home/vulcan/workspace" + fallback: true diff --git a/tests/fixtures/prebake/minimal.yml b/tests/fixtures/prebake/minimal.yml new file mode 100644 index 0000000..34a1b43 --- /dev/null +++ b/tests/fixtures/prebake/minimal.yml @@ -0,0 +1,8 @@ +version: "1.0" +environment: + builder: "baremetal" +bootstrap: + user: "vulcan" + workspace: + path: "/home/vulcan/workspace" + fallback: true diff --git a/tests/fixtures/prebake/with_hooks.yml b/tests/fixtures/prebake/with_hooks.yml new file mode 100644 index 0000000..3e105da --- /dev/null +++ b/tests/fixtures/prebake/with_hooks.yml @@ -0,0 +1,15 @@ +version: "1.0" +environment: + builder: "baremetal" +bootstrap: + user: "vulcan" + workspace: + path: "/home/vulcan/workspace" + fallback: true +hooks: + early: + - name: "test-early-hook" + command: "echo 'early hook executed'" + late: + - name: "test-late-hook" + command: "echo 'late hook executed'" diff --git a/tests/fixtures/prebake/with_pacman.yml b/tests/fixtures/prebake/with_pacman.yml new file mode 100644 index 0000000..36da80b --- /dev/null +++ b/tests/fixtures/prebake/with_pacman.yml @@ -0,0 +1,14 @@ +version: "1.0" +environment: + builder: "baremetal" +bootstrap: + user: "vulcan" + workspace: + path: "/home/vulcan/workspace" + fallback: true +dependencies: + system: + pacman: + packages: + - "curl" + - "wget" diff --git a/tests/llm/__init__.py b/tests/llm/__init__.py new file mode 100644 index 0000000..15f1768 --- /dev/null +++ b/tests/llm/__init__.py @@ -0,0 +1,12 @@ +""" +LLM-based log judgment system for system tests. + +Uses LLM APIs to evaluate non-deterministic test outputs (logs, stderr, etc.) +where simple string matching is insufficient. +""" + +from .client import LLMClient +from .cache import LLMCache +from .judge import judge_log, JudgmentResult + +__all__ = ["LLMClient", "LLMCache", "judge_log", "JudgmentResult"] diff --git a/tests/llm/cache.py b/tests/llm/cache.py new file mode 100644 index 0000000..baf12a5 --- /dev/null +++ b/tests/llm/cache.py @@ -0,0 +1,56 @@ +""" +Disk-based cache for LLM API responses. + +Uses SHA256 of (prompt + model + system_prompt) as cache key. +Cache files stored as JSON in ``.cache/`` directory. +""" + +import hashlib +import json +from pathlib import Path + + +class LLMCache: + """SHA256-based disk cache for LLM API responses.""" + + def __init__(self, cache_dir: str | None = None): + if cache_dir is None: + import os + + cache_dir = os.path.join(os.path.dirname(__file__), ".cache") + self.cache_dir = Path(cache_dir) + self.cache_dir.mkdir(parents=True, exist_ok=True) + + def _make_key(self, prompt: str, model: str, system_prompt: str = "") -> str: + """Create deterministic cache key from request parameters.""" + data = json.dumps( + {"prompt": prompt, "model": model, "system": system_prompt}, + sort_keys=True, + ) + return hashlib.sha256(data.encode()).hexdigest() + + def get(self, prompt: str, model: str, system_prompt: str = "") -> dict | None: + """Retrieve cached response if exists.""" + key = self._make_key(prompt, model, system_prompt) + path = self.cache_dir / f"{key}.json" + if path.exists(): + try: + with open(path) as f: + return json.load(f) + except (json.JSONDecodeError, IOError): + return None + return None + + def set( + self, prompt: str, model: str, system_prompt: str, response: dict + ) -> None: + """Store response in cache.""" + key = self._make_key(prompt, model, system_prompt) + path = self.cache_dir / f"{key}.json" + with open(path, "w") as f: + json.dump(response, f, ensure_ascii=False) + + def clear(self) -> None: + """Clear all cached responses.""" + for f in self.cache_dir.glob("*.json"): + f.unlink() diff --git a/tests/llm/client.py b/tests/llm/client.py new file mode 100644 index 0000000..72400fa --- /dev/null +++ b/tests/llm/client.py @@ -0,0 +1,187 @@ +""" +LLM client supporting both Anthropic and OpenAI APIs. + +Prefers Anthropic format (per project convention). +Falls back to OpenAI format if Anthropic is unavailable. +""" + +import os +import logging +from .cache import LLMCache + +logger = logging.getLogger(__name__) + + +class LLMClient: + """LLM client with caching support. Prefers Anthropic API.""" + + def __init__(self, cache: LLMCache | None = None): + self.cache = cache or LLMCache() + self._anthropic_client = None + self._openai_client = None + self._init_clients() + + def _init_clients(self): + """Initialize API clients based on available environment variables.""" + # Anthropic (preferred) + anthropic_key = os.environ.get("ANTHROPIC_API_KEY") + anthropic_base = os.environ.get("ANTHROPIC_BASE_URL") + if anthropic_key: + try: + import anthropic + + kwargs: dict = {"api_key": anthropic_key} + if anthropic_base: + kwargs["base_url"] = anthropic_base + self._anthropic_client = anthropic.Anthropic(**kwargs) + logger.info("Anthropic client initialized") + except ImportError: + logger.warning("anthropic package not installed") + + # OpenAI (fallback) + openai_key = os.environ.get("OPENAI_API_KEY") + openai_base = os.environ.get("OPENAI_BASE_URL") + if openai_key: + try: + import openai + + kwargs = {"api_key": openai_key} + if openai_base: + kwargs["base_url"] = openai_base + self._openai_client = openai.OpenAI(**kwargs) + logger.info("OpenAI client initialized") + except ImportError: + logger.warning("openai package not installed") + + @property + def available(self) -> bool: + """Check if any LLM client is available.""" + return ( + self._anthropic_client is not None or self._openai_client is not None + ) + + def complete( + self, prompt: str, system_prompt: str = "", max_tokens: int = 2000 + ) -> str: + """Send a completion request, with caching. + + Args: + prompt: User message. + system_prompt: System message (sets role / persona). + max_tokens: Max response tokens. + + Returns: + LLM response text. + + Raises: + RuntimeError: If no client is available. + """ + model = os.environ.get("MODEL", "deepseek-v4-flash") + + # Check cache + cached = self.cache.get(prompt, model, system_prompt) + if cached: + logger.info("Cache hit for LLM request") + return cached["content"] + + # Try Anthropic first + if self._anthropic_client: + content = self._call_anthropic( + prompt, system_prompt, model, max_tokens + ) + elif self._openai_client: + content = self._call_openai(prompt, system_prompt, model, max_tokens) + else: + raise RuntimeError( + "No LLM client available. " + "Set ANTHROPIC_API_KEY or OPENAI_API_KEY." + ) + + # Cache the response + self.cache.set(prompt, model, system_prompt, {"content": content}) + return content + + # ------------------------------------------------------------------ + # Private helpers + # ------------------------------------------------------------------ + + def _call_anthropic( + self, prompt: str, system_prompt: str, model: str, max_tokens: int + ) -> str: + """Call Anthropic API.""" + kwargs: dict = { + "model": model, + "max_tokens": max_tokens, + "messages": [{"role": "user", "content": prompt}], + } + if system_prompt: + kwargs["system"] = system_prompt + + # Only add reasoning_effort if the model supports it + reasoning_effort = os.environ.get("REASONING_EFFORT") + if reasoning_effort: + try: + kwargs["reasoning_effort"] = reasoning_effort + response = self._anthropic_client.messages.create(**kwargs) + except Exception as e: + if ( + "reasoning_effort" in str(e).lower() + or "unexpected" in str(e).lower() + ): + del kwargs["reasoning_effort"] + response = self._anthropic_client.messages.create(**kwargs) + else: + raise + else: + response = self._anthropic_client.messages.create(**kwargs) + + # Extract text from response content blocks + if hasattr(response, "content") and response.content: + block = response.content[0] + if hasattr(block, "text"): + return block.text + if hasattr(block, "type") and block.type == "thinking": + for b in response.content: + if hasattr(b, "text"): + return b.text + return str(block) + return str(response) + + def _call_openai( + self, prompt: str, system_prompt: str, model: str, max_tokens: int + ) -> str: + """Call OpenAI-compatible API.""" + messages: list[dict] = [] + if system_prompt: + messages.append({"role": "system", "content": system_prompt}) + messages.append({"role": "user", "content": prompt}) + + kwargs: dict = { + "model": model, + "messages": messages, + "max_tokens": max_tokens, + } + + # Only add reasoning_effort if supported + reasoning_effort = os.environ.get("REASONING_EFFORT") + if reasoning_effort: + try: + kwargs["reasoning_effort"] = reasoning_effort + response = self._openai_client.chat.completions.create( + **kwargs + ) + except Exception as e: + if ( + "reasoning_effort" in str(e).lower() + or "unexpected" in str(e).lower() + ): + del kwargs["reasoning_effort"] + response = self._openai_client.chat.completions.create( + **kwargs + ) + else: + raise + else: + response = self._openai_client.chat.completions.create(**kwargs) + + return response.choices[0].message.content diff --git a/tests/llm/judge.py b/tests/llm/judge.py new file mode 100644 index 0000000..f9703b3 --- /dev/null +++ b/tests/llm/judge.py @@ -0,0 +1,149 @@ +""" +LLM-based log judgment for system tests. + +Evaluates logs against expected behavior using LLM semantic analysis. +Returns structured pass/fail results with reasoning. +""" + +import json +import re +import logging +from dataclasses import dataclass +from .client import LLMClient + +logger = logging.getLogger(__name__) + +JUDGE_SYSTEM_PROMPT = """\ +You are a test result judge for a CI/CD system called HoneyBiscuitWorkshop. + +Your job: Evaluate whether actual command output/logs match expected behavior. + +RULES: +1. You MUST respond with valid JSON in this exact format: + {"verdict": "pass" | "fail", "reason": "brief explanation"} +2. "pass" = the output confirms the expected behavior occurred +3. "fail" = the output shows the expected behavior did NOT occur, or shows an error +4. Ignore irrelevant noise (timestamps, debug lines, unrelated warnings) +5. Focus ONLY on whether the specific expected behavior is confirmed or contradicted +6. Be strict: if the evidence is ambiguous, verdict is "fail" +7. Do NOT include any text outside the JSON object""" + + +@dataclass +class JudgmentResult: + """Result of an LLM log judgment.""" + + verdict: str # "pass" or "fail" + reason: str + raw_response: str + + @property + def passed(self) -> bool: + return self.verdict == "pass" + + +def _extract_relevant_logs(full_log: str, max_chars: int = 4000) -> str: + """Extract relevant portions from logs, removing noise. + + Keeps the last *max_chars* characters (most recent / relevant). + Removes common noise patterns. + """ + lines = full_log.strip().split("\n") + + noise_patterns = [ + r"^\d{4}-\d{2}-\d{2}T", + r"^DEBUG ", + r"^TRACE ", + ] + filtered = [ + line + for line in lines + if not any(re.match(p, line) for p in noise_patterns) + ] + + text = "\n".join(filtered) + if len(text) > max_chars: + text = "...\n" + text[-max_chars:] + return text + + +def judge_log( + operation: str, + expected_behavior: str, + actual_log: str, + client: LLMClient | None = None, +) -> JudgmentResult: + """Judge whether actual logs match expected behavior. + + Args: + operation: Description of what was attempted + (e.g. ``"prebake bootstrap stage"``). + expected_behavior: What should have happened + (e.g. ``"user vulcan was created"``). + actual_log: The actual stdout / stderr output. + client: ``LLMClient`` instance (creates default if ``None``). + + Returns: + ``JudgmentResult`` with verdict and reason. + """ + if client is None: + client = LLMClient() + + if not client.available: + logger.warning("No LLM client available, returning fail verdict") + return JudgmentResult( + verdict="fail", + reason="No LLM client available for judgment", + raw_response="", + ) + + relevant_log = _extract_relevant_logs(actual_log) + + prompt = f"""\ +## Operation +{operation} + +## Expected Behavior +{expected_behavior} + +## Actual Output (relevant excerpts) +``` +{relevant_log} +``` + +Evaluate: Does the actual output confirm the expected behavior occurred? +Respond with JSON only.""" + + try: + response = client.complete( + prompt=prompt, + system_prompt=JUDGE_SYSTEM_PROMPT, + max_tokens=500, + ) + + # Parse JSON response — handle potential markdown code blocks + text = response.strip() + if text.startswith("```"): + lines = text.split("\n") + text = "\n".join(lines[1:-1]) if len(lines) > 2 else text + + result = json.loads(text) + return JudgmentResult( + verdict=result.get("verdict", "fail"), + reason=result.get("reason", "No reason provided"), + raw_response=response, + ) + except json.JSONDecodeError as e: + logger.error(f"Failed to parse LLM response as JSON: {e}") + return JudgmentResult( + verdict="fail", + reason=f"LLM response was not valid JSON: {response[:200]}", + raw_response=response, + ) + except Exception as e: + logger.error(f"LLM judgment failed: {e}") + return JudgmentResult( + verdict="fail", + reason=f"LLM call failed: {str(e)}", + raw_response="", + ) diff --git a/tests/probes/__init__.py b/tests/probes/__init__.py new file mode 100644 index 0000000..4d71a54 --- /dev/null +++ b/tests/probes/__init__.py @@ -0,0 +1,34 @@ +""" +Environment probes for verifying system state in workshop-baker tests. + +Each probe is a standalone function that can be called from any test. +Probes return bool or raise AssertionError with descriptive message. +""" + +from .user import user_exists, user_uid, user_gid, user_in_group, user_shell +from .file import ( + file_exists, + dir_exists, + file_permissions, + file_contains, + file_owned_by, +) +from .package import package_installed, package_version +from .process import process_running, process_count + +__all__ = [ + "user_exists", + "user_uid", + "user_gid", + "user_in_group", + "user_shell", + "file_exists", + "dir_exists", + "file_permissions", + "file_contains", + "file_owned_by", + "package_installed", + "package_version", + "process_running", + "process_count", +] diff --git a/tests/probes/file.py b/tests/probes/file.py new file mode 100644 index 0000000..7f73fb1 --- /dev/null +++ b/tests/probes/file.py @@ -0,0 +1,43 @@ +"""File and directory existence / permission probes.""" + +import os +import stat +import grp +import pwd + + +def file_exists(path: str) -> bool: + """Check if *path* exists and is a regular file.""" + return os.path.isfile(path) + + +def dir_exists(path: str) -> bool: + """Check if *path* exists and is a directory.""" + return os.path.isdir(path) + + +def file_permissions(path: str) -> int: + """Return the permission mode of *path* as an octal int (e.g. ``0o755``). + + Raises ``FileNotFoundError`` if *path* doesn't exist. + """ + return stat.S_IMODE(os.stat(path).st_mode) + + +def file_contains(path: str, text: str) -> bool: + """Check if a file contains the specified text.""" + with open(path, "r", errors="replace") as f: + return text in f.read() + + +def file_owned_by(path: str, username: str) -> bool: + """Check if a file is owned by the specified user.""" + st = os.stat(path) + owner = pwd.getpwuid(st.st_uid).pw_name + return owner == username + + +def file_group(path: str) -> str: + """Get the group owner of a file.""" + st = os.stat(path) + return grp.getgrgid(st.st_gid).gr_name diff --git a/tests/probes/package.py b/tests/probes/package.py new file mode 100644 index 0000000..295fcfa --- /dev/null +++ b/tests/probes/package.py @@ -0,0 +1,34 @@ +"""Package installation probes for Arch Linux (pacman).""" + +import subprocess + + +def package_installed(package_name: str) -> bool: + """Check if a pacman package is installed.""" + try: + result = subprocess.run( + ["pacman", "-Q", package_name], + capture_output=True, + text=True, + ) + return result.returncode == 0 + except FileNotFoundError: + return False + + +def package_version(package_name: str) -> str | None: + """Get installed version of a package, or ``None`` if not installed.""" + try: + result = subprocess.run( + ["pacman", "-Q", package_name], + capture_output=True, + text=True, + ) + if result.returncode == 0: + # Output format: "package_name version" + parts = result.stdout.strip().split() + if len(parts) >= 2: + return parts[1] + return None + except FileNotFoundError: + return None diff --git a/tests/probes/process.py b/tests/probes/process.py new file mode 100644 index 0000000..ac432da --- /dev/null +++ b/tests/probes/process.py @@ -0,0 +1,44 @@ +"""Process existence probes.""" + +import os +import subprocess + + +def process_running(process_name: str) -> bool: + """Check if a process with the given name is running.""" + try: + result = subprocess.run( + ["pgrep", "-x", process_name], + capture_output=True, + text=True, + ) + return result.returncode == 0 + except FileNotFoundError: + # Fall back to /proc scan if pgrep not available + try: + for pid_dir in os.listdir("/proc"): + if pid_dir.isdigit(): + try: + with open(f"/proc/{pid_dir}/comm") as f: + if f.read().strip() == process_name: + return True + except (IOError, OSError): + continue + except OSError: + pass + return False + + +def process_count(process_name: str) -> int: + """Count processes with the given name.""" + try: + result = subprocess.run( + ["pgrep", "-c", "-x", process_name], + capture_output=True, + text=True, + ) + if result.returncode == 0: + return int(result.stdout.strip()) + return 0 + except (FileNotFoundError, ValueError): + return 0 diff --git a/tests/probes/user.py b/tests/probes/user.py new file mode 100644 index 0000000..94ade6d --- /dev/null +++ b/tests/probes/user.py @@ -0,0 +1,37 @@ +"""User existence and attribute probes.""" + +import pwd +import grp + + +def user_exists(username: str) -> bool: + """Check if a system user exists.""" + try: + pwd.getpwnam(username) + return True + except KeyError: + return False + + +def user_uid(username: str) -> int: + """Get the UID of a user. Raises KeyError if user doesn't exist.""" + return pwd.getpwnam(username).pw_uid + + +def user_gid(username: str) -> int: + """Get the primary GID of a user. Raises KeyError if user doesn't exist.""" + return pwd.getpwnam(username).pw_gid + + +def user_in_group(username: str, groupname: str) -> bool: + """Check if a user is a member of a group.""" + try: + group = grp.getgrnam(groupname) + return username in group.gr_mem + except KeyError: + return False + + +def user_shell(username: str) -> str: + """Get the login shell of a user. Raises KeyError if user doesn't exist.""" + return pwd.getpwnam(username).pw_shell diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 0000000..04fca7d --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,6 @@ +pytest>=8.0.0 +pytest-timeout>=2.2.0 +pytest-xdist>=3.5.0 +anthropic>=0.40.0,<0.90.0 +openai>=1.50.0,<2.0.0 +pyyaml>=6.0 diff --git a/tests/test_bake.py b/tests/test_bake.py new file mode 100644 index 0000000..ce95e9a --- /dev/null +++ b/tests/test_bake.py @@ -0,0 +1,129 @@ +import os +import pytest + + +FIXTURES = os.path.join(os.path.dirname(__file__), "fixtures", "bake") + + +@pytest.mark.bake +class TestBakeSimpleScript: + + def test_simple_script_runs(self, run_baker, bake_workspace, work_dir): + prebake_cfg = os.path.join( + os.path.dirname(__file__), "fixtures", "prebake", "minimal.yml" + ) + script = os.path.join(FIXTURES, "simple.sh") + bake_base = os.path.join(FIXTURES, "bake_base.sh") + + result = run_baker( + "bake", script, + "--bake-base", bake_base, + "--prebake", prebake_cfg, + cwd=work_dir, + timeout=60, + ) + assert result["success"], f"Bake failed: {result['stderr']}" + assert "Hello from simple bake script" in result["stdout"] + + def test_simple_script_sets_env_vars(self, run_baker, bake_workspace, work_dir): + prebake_cfg = os.path.join( + os.path.dirname(__file__), "fixtures", "prebake", "minimal.yml" + ) + script = os.path.join(FIXTURES, "simple.sh") + bake_base = os.path.join(FIXTURES, "bake_base.sh") + + result = run_baker( + "bake", script, + "--bake-base", bake_base, + "--prebake", prebake_cfg, + cwd=work_dir, + timeout=60, + ) + assert result["success"] + assert "test-pipeline" in result["stdout"] + + +@pytest.mark.bake +class TestBakePipeline: + + def test_pipeline_functions_execute(self, run_baker, bake_workspace, work_dir): + prebake_cfg = os.path.join( + os.path.dirname(__file__), "fixtures", "prebake", "minimal.yml" + ) + script = os.path.join(FIXTURES, "pipeline.sh") + bake_base = os.path.join(FIXTURES, "bake_base.sh") + + result = run_baker( + "bake", script, + "--bake-base", bake_base, + "--prebake", prebake_cfg, + cwd=work_dir, + timeout=60, + ) + assert result["success"], f"Pipeline bake failed: {result['stderr']}" + assert os.path.exists("/tmp/bake-test-output/step1.txt") + assert os.path.exists("/tmp/bake-test-output/step2.txt") + + def test_pipeline_step_outputs_correct(self, run_baker, bake_workspace, work_dir): + prebake_cfg = os.path.join( + os.path.dirname(__file__), "fixtures", "prebake", "minimal.yml" + ) + script = os.path.join(FIXTURES, "pipeline.sh") + bake_base = os.path.join(FIXTURES, "bake_base.sh") + + result = run_baker( + "bake", script, + "--bake-base", bake_base, + "--prebake", prebake_cfg, + cwd=work_dir, + timeout=60, + ) + assert result["success"] + with open("/tmp/bake-test-output/step1.txt") as f: + assert f.read().strip() == "step1-done" + with open("/tmp/bake-test-output/step2.txt") as f: + assert f.read().strip() == "step2-done" + + +@pytest.mark.bake +class TestBakeAfterDependencies: + + def test_after_ordering(self, run_baker, bake_workspace, work_dir): + prebake_cfg = os.path.join( + os.path.dirname(__file__), "fixtures", "prebake", "minimal.yml" + ) + script = os.path.join(FIXTURES, "with_after.sh") + bake_base = os.path.join(FIXTURES, "bake_base.sh") + + result = run_baker( + "bake", script, + "--bake-base", bake_base, + "--prebake", prebake_cfg, + cwd=work_dir, + timeout=60, + ) + assert result["success"], f"@after bake failed: {result['stderr']}" + assert os.path.exists("/tmp/bake-after-test/status.txt") + assert os.path.exists("/tmp/bake-after-test/source.txt") + assert os.path.exists("/tmp/bake-after-test/build.txt") + + +@pytest.mark.bake +class TestBakeStatusFiles: + + def test_status_written_after_bake(self, run_baker, bake_workspace, work_dir): + prebake_cfg = os.path.join( + os.path.dirname(__file__), "fixtures", "prebake", "minimal.yml" + ) + script = os.path.join(FIXTURES, "simple.sh") + bake_base = os.path.join(FIXTURES, "bake_base.sh") + + result = run_baker( + "bake", script, + "--bake-base", bake_base, + "--prebake", prebake_cfg, + cwd=work_dir, + timeout=60, + ) + assert result["success"] + assert os.path.exists("/tmp/.hbwstatus") diff --git a/tests/test_finalize.py b/tests/test_finalize.py new file mode 100644 index 0000000..572b8a5 --- /dev/null +++ b/tests/test_finalize.py @@ -0,0 +1,77 @@ +import os +import pytest + +from probes import file +from llm import judge_log, LLMClient + + +FIXTURES = os.path.join(os.path.dirname(__file__), "fixtures", "finalize") + + +@pytest.mark.finalize +class TestFinalizeMinimal: + + def test_minimal_finalize_succeeds(self, run_baker, work_dir): + config = os.path.join(FIXTURES, "minimal.yml") + result = run_baker("finalize", config, cwd=work_dir, timeout=60) + assert result["success"], f"Minimal finalize failed: {result['stderr']}" + + +@pytest.mark.finalize +class TestFinalizeHooks: + + def test_early_hook_executes(self, run_baker, work_dir): + config = os.path.join(FIXTURES, "with_hooks.yml") + result = run_baker("finalize", config, cwd=work_dir, timeout=60) + assert result["success"] + assert "finalize early hook" in result["stdout"] or "finalize early hook" in result["stderr"] + + def test_late_hook_executes(self, run_baker, work_dir): + config = os.path.join(FIXTURES, "with_hooks.yml") + result = run_baker("finalize", config, cwd=work_dir, timeout=60) + assert result["success"] + assert "finalize late hook" in result["stdout"] or "finalize late hook" in result["stderr"] + + +@pytest.mark.finalize +class TestFinalizeStatus: + + def test_finalize_writes_status(self, run_baker, work_dir): + config = os.path.join(FIXTURES, "minimal.yml") + result = run_baker("finalize", config, cwd=work_dir, timeout=60) + assert result["success"] + assert os.path.exists("/tmp/.hbwstatus") + + +@pytest.mark.finalize +@pytest.mark.llm +class TestFinalizeWithLLM: + + def test_finalize_hooks_via_llm(self, run_baker, llm_env, work_dir): + if not llm_env: + pytest.skip("LLM environment not configured") + + config = os.path.join(FIXTURES, "with_hooks.yml") + result = run_baker("finalize", config, cwd=work_dir, timeout=60) + + combined_log = result["stdout"] + "\n" + result["stderr"] + + judgment = judge_log( + operation="finalize with early and late hooks", + expected_behavior="Both early and late hooks executed successfully, " + "printing 'finalize early hook' and 'finalize late hook'", + actual_log=combined_log, + ) + assert judgment.passed, f"LLM judgment failed: {judgment.reason}" + + +@pytest.mark.finalize +class TestFinalizePlugins: + + @pytest.mark.xfail(reason="Plugin download requires network/git access", strict=False) + def test_plugin_download(self, run_baker, work_dir): + pass + + @pytest.mark.xfail(reason="Artifact stage not yet implemented", strict=False) + def test_artifact_collection(self, run_baker, work_dir): + pass diff --git a/tests/test_prebake.py b/tests/test_prebake.py new file mode 100644 index 0000000..9e11292 --- /dev/null +++ b/tests/test_prebake.py @@ -0,0 +1,94 @@ +import os +import pytest + +from probes import user, file, package + + +FIXTURES = os.path.join(os.path.dirname(__file__), "fixtures", "prebake") + + +@pytest.mark.prebake +class TestPrebakeBootstrap: + + def test_bootstrap_creates_vulcan_user(self, run_baker, work_dir): + config = os.path.join(FIXTURES, "bootstrap_only.yml") + result = run_baker("prebake", config, cwd=work_dir, timeout=60) + assert result["success"], f"prebake failed: {result['stderr']}" + assert user.user_exists("vulcan") + + def test_bootstrap_creates_workspace(self, run_baker, work_dir): + config = os.path.join(FIXTURES, "bootstrap_only.yml") + result = run_baker("prebake", config, cwd=work_dir, timeout=60) + assert result["success"] + assert file.dir_exists("/home/vulcan/workspace") or file.dir_exists("/workspace") + + def test_bootstrap_creates_symlink(self, run_baker, work_dir): + config = os.path.join(FIXTURES, "bootstrap_only.yml") + result = run_baker("prebake", config, cwd=work_dir, timeout=60) + assert result["success"] + assert os.path.islink("/workspace") + + @pytest.mark.xfail(reason="Sudoers not created without package manager config", strict=False) + def test_bootstrap_creates_sudoers(self, run_baker, work_dir): + config = os.path.join(FIXTURES, "bootstrap_only.yml") + result = run_baker("prebake", config, cwd=work_dir, timeout=60) + assert result["success"] + assert file.file_exists("/etc/sudoers.d/workshop") + + def test_bootstrap_generates_bootstrap_script(self, run_baker, work_dir): + config = os.path.join(FIXTURES, "bootstrap_only.yml") + result = run_baker("prebake", config, cwd=work_dir, timeout=60) + assert result["success"] + assert file.file_exists("/bootstrap.sh") + + def test_bootstrap_script_permissions(self, run_baker, work_dir): + config = os.path.join(FIXTURES, "bootstrap_only.yml") + result = run_baker("prebake", config, cwd=work_dir, timeout=60) + assert result["success"] + mode = file.file_permissions("/bootstrap.sh") + assert mode & 0o111, f"/bootstrap.sh not executable, mode: {oct(mode)}" + + +@pytest.mark.prebake +class TestPrebakeDeps: + + @pytest.mark.xfail(reason="Package installation may fail in test environment") + def test_pacman_packages_installed(self, run_baker, work_dir): + config = os.path.join(FIXTURES, "with_pacman.yml") + result = run_baker("prebake", config, cwd=work_dir, timeout=120) + assert result["success"], f"prebake failed: {result['stderr']}" + assert package.package_installed("curl") + assert package.package_installed("wget") + + +@pytest.mark.prebake +class TestPrebakeHooks: + + @pytest.mark.xfail(reason="Late hook fails after privilege drop to vulcan (Permission denied)", strict=False) + def test_early_hook_executes(self, run_baker, work_dir): + config = os.path.join(FIXTURES, "with_hooks.yml") + result = run_baker("prebake", config, cwd=work_dir, timeout=60) + assert result["success"], f"prebake with hooks failed: stdout={result['stdout']!r} stderr={result['stderr']!r}" + assert "early hook executed" in result["stdout"] or "early hook executed" in result["stderr"] + + @pytest.mark.xfail(reason="Late hook fails after privilege drop to vulcan (Permission denied)", strict=False) + def test_late_hook_executes(self, run_baker, work_dir): + config = os.path.join(FIXTURES, "with_hooks.yml") + result = run_baker("prebake", config, cwd=work_dir, timeout=60) + assert result["success"], f"prebake with hooks failed: stdout={result['stdout']!r} stderr={result['stderr']!r}" + assert "late hook executed" in result["stdout"] or "late hook executed" in result["stderr"] + + +@pytest.mark.prebake +class TestPrebakeStatus: + + def test_status_files_written(self, run_baker, work_dir): + config = os.path.join(FIXTURES, "bootstrap_only.yml") + result = run_baker("prebake", config, cwd=work_dir, timeout=60) + assert result["success"] + assert os.path.exists("/tmp/.hbwstatus") + + def test_minimal_config_succeeds(self, run_baker, work_dir): + config = os.path.join(FIXTURES, "minimal.yml") + result = run_baker("prebake", config, cwd=work_dir, timeout=60) + assert result["success"], f"Minimal prebake failed: {result['stderr']}"