84b3b7d8e8
Adds a complete pytest system test suite for the HoneyBiscuitWorkshop CI/CD system: - runner.sh: orchestrates build, Docker image creation, and test execution - Dockerfile: Arch Linux-based test image with Python venv - conftest.py: shared fixtures (baker_bin, run_baker, work_dir, llm_env) - probes/: environment probes for checking users, files, packages, processes - llm/: LLM-based log judgment system with caching for semantic test evaluation - fixtures/: test configs and scripts for prebake, bake, and finalize stages - test_prebake.py, test_bake.py, test_finalize.py: test suites for each pipeline stage Also refactors finalize.yml.tmpl notification config to use a templates system.
176 lines
4.8 KiB
Python
176 lines
4.8 KiB
Python
"""
|
|
Root conftest for HoneyBiscuitWorkshop system tests.
|
|
|
|
Provides core fixtures for:
|
|
- Locating the baker binary and examples
|
|
- Running subprocess commands with output capture
|
|
- Managing temporary working directories
|
|
"""
|
|
import os
|
|
import subprocess
|
|
import shutil
|
|
import tempfile
|
|
import pytest
|
|
|
|
|
|
def pytest_configure(config):
|
|
"""Register custom markers."""
|
|
config.addinivalue_line("markers", "prebake: prebake stage tests")
|
|
config.addinivalue_line("markers", "bake: bake stage tests")
|
|
config.addinivalue_line("markers", "finalize: finalize stage tests")
|
|
config.addinivalue_line("markers", "llm: tests requiring LLM API")
|
|
config.addinivalue_line("markers", "slow: slow running tests")
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def baker_bin():
|
|
"""Path to the workshop-baker binary."""
|
|
path = os.environ.get("BAKER_BIN", "/workshop/workshop-baker")
|
|
if not os.path.isfile(path):
|
|
pytest.skip(f"Baker binary not found at {path}")
|
|
if not os.access(path, os.X_OK):
|
|
pytest.skip(f"Baker binary not executable at {path}")
|
|
return path
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def examples_dir():
|
|
"""Path to the examples directory (may be None)."""
|
|
path = os.environ.get("EXAMPLES_DIR", "/workshop/examples")
|
|
if not os.path.isdir(path):
|
|
return None
|
|
return path
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def llm_env():
|
|
"""LLM environment variables dict (from llm.env)."""
|
|
env = {}
|
|
for var in [
|
|
"OPENAI_BASE_URL",
|
|
"OPENAI_API_KEY",
|
|
"ANTHROPIC_BASE_URL",
|
|
"ANTHROPIC_API_KEY",
|
|
"MODEL",
|
|
"REASONING_EFFORT",
|
|
]:
|
|
val = os.environ.get(var)
|
|
if val:
|
|
env[var] = val
|
|
return env
|
|
|
|
|
|
@pytest.fixture
|
|
def work_dir():
|
|
"""Create a temporary working directory for a test, cleaned up after."""
|
|
d = tempfile.mkdtemp(prefix="hbw-test-")
|
|
yield d
|
|
shutil.rmtree(d, ignore_errors=True)
|
|
|
|
|
|
@pytest.fixture
|
|
def run_cmd():
|
|
"""Fixture that provides a command runner function.
|
|
|
|
Returns a function that runs a command and returns
|
|
(stdout, stderr, returncode, success).
|
|
"""
|
|
|
|
def _run(args, cwd=None, env=None, timeout=120):
|
|
"""Run a command and capture output.
|
|
|
|
Args:
|
|
args: Command as list of strings.
|
|
cwd: Working directory.
|
|
env: Environment variables (merged with os.environ).
|
|
timeout: Timeout in seconds.
|
|
|
|
Returns:
|
|
dict with keys: stdout, stderr, returncode, success
|
|
"""
|
|
full_env = {**os.environ}
|
|
if env:
|
|
full_env.update(env)
|
|
|
|
try:
|
|
result = subprocess.run(
|
|
args,
|
|
cwd=cwd,
|
|
env=full_env,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=timeout,
|
|
)
|
|
return {
|
|
"stdout": result.stdout,
|
|
"stderr": result.stderr,
|
|
"returncode": result.returncode,
|
|
"success": result.returncode == 0,
|
|
}
|
|
except subprocess.TimeoutExpired:
|
|
return {
|
|
"stdout": "",
|
|
"stderr": f"Command timed out after {timeout}s",
|
|
"returncode": -1,
|
|
"success": False,
|
|
}
|
|
except Exception as e:
|
|
return {
|
|
"stdout": "",
|
|
"stderr": str(e),
|
|
"returncode": -1,
|
|
"success": False,
|
|
}
|
|
|
|
return _run
|
|
|
|
|
|
@pytest.fixture
|
|
def run_baker(baker_bin, run_cmd):
|
|
"""Fixture that provides a baker command runner.
|
|
|
|
Returns a function that runs workshop-baker with common flags.
|
|
"""
|
|
|
|
def _run(subcommand, *args, cwd=None, env=None, timeout=120):
|
|
"""Run workshop-baker with a subcommand.
|
|
|
|
Args:
|
|
subcommand: Baker subcommand (prebake, bake, finalize).
|
|
*args: Additional arguments to the subcommand.
|
|
cwd: Working directory.
|
|
env: Extra environment variables.
|
|
timeout: Timeout in seconds.
|
|
|
|
Returns:
|
|
dict with keys: stdout, stderr, returncode, success
|
|
"""
|
|
cmd = [
|
|
baker_bin,
|
|
"-u", "testuser",
|
|
"-p", "test-pipeline",
|
|
"-b", "test-build-001",
|
|
subcommand,
|
|
]
|
|
cmd.extend(args)
|
|
return run_cmd(cmd, cwd=cwd, env=env, timeout=timeout)
|
|
|
|
return _run
|
|
|
|
|
|
@pytest.fixture
|
|
def fixtures_dir():
|
|
"""Path to the test fixtures directory."""
|
|
return os.path.join(os.path.dirname(__file__), "fixtures")
|
|
|
|
|
|
@pytest.fixture
|
|
def bake_workspace():
|
|
"""Create /workspace for bake tests (baker writes temp scripts there)."""
|
|
existed = os.path.exists("/workspace")
|
|
if not existed:
|
|
os.makedirs("/workspace", exist_ok=True)
|
|
yield
|
|
if not existed:
|
|
shutil.rmtree("/workspace", ignore_errors=True)
|