Catty Steve
84b3b7d8e8
Adds a complete pytest system test suite for the HoneyBiscuitWorkshop CI/CD system: - runner.sh: orchestrates build, Docker image creation, and test execution - Dockerfile: Arch Linux-based test image with Python venv - conftest.py: shared fixtures (baker_bin, run_baker, work_dir, llm_env) - probes/: environment probes for checking users, files, packages, processes - llm/: LLM-based log judgment system with caching for semantic test evaluation - fixtures/: test configs and scripts for prebake, bake, and finalize stages - test_prebake.py, test_bake.py, test_finalize.py: test suites for each pipeline stage Also refactors finalize.yml.tmpl notification config to use a templates system.
78 lines
2.6 KiB
Python
78 lines
2.6 KiB
Python
import os
|
|
import pytest
|
|
|
|
from probes import file
|
|
from llm import judge_log, LLMClient
|
|
|
|
|
|
FIXTURES = os.path.join(os.path.dirname(__file__), "fixtures", "finalize")
|
|
|
|
|
|
@pytest.mark.finalize
|
|
class TestFinalizeMinimal:
|
|
|
|
def test_minimal_finalize_succeeds(self, run_baker, work_dir):
|
|
config = os.path.join(FIXTURES, "minimal.yml")
|
|
result = run_baker("finalize", config, cwd=work_dir, timeout=60)
|
|
assert result["success"], f"Minimal finalize failed: {result['stderr']}"
|
|
|
|
|
|
@pytest.mark.finalize
|
|
class TestFinalizeHooks:
|
|
|
|
def test_early_hook_executes(self, run_baker, work_dir):
|
|
config = os.path.join(FIXTURES, "with_hooks.yml")
|
|
result = run_baker("finalize", config, cwd=work_dir, timeout=60)
|
|
assert result["success"]
|
|
assert "finalize early hook" in result["stdout"] or "finalize early hook" in result["stderr"]
|
|
|
|
def test_late_hook_executes(self, run_baker, work_dir):
|
|
config = os.path.join(FIXTURES, "with_hooks.yml")
|
|
result = run_baker("finalize", config, cwd=work_dir, timeout=60)
|
|
assert result["success"]
|
|
assert "finalize late hook" in result["stdout"] or "finalize late hook" in result["stderr"]
|
|
|
|
|
|
@pytest.mark.finalize
|
|
class TestFinalizeStatus:
|
|
|
|
def test_finalize_writes_status(self, run_baker, work_dir):
|
|
config = os.path.join(FIXTURES, "minimal.yml")
|
|
result = run_baker("finalize", config, cwd=work_dir, timeout=60)
|
|
assert result["success"]
|
|
assert os.path.exists("/tmp/.hbwstatus")
|
|
|
|
|
|
@pytest.mark.finalize
|
|
@pytest.mark.llm
|
|
class TestFinalizeWithLLM:
|
|
|
|
def test_finalize_hooks_via_llm(self, run_baker, llm_env, work_dir):
|
|
if not llm_env:
|
|
pytest.skip("LLM environment not configured")
|
|
|
|
config = os.path.join(FIXTURES, "with_hooks.yml")
|
|
result = run_baker("finalize", config, cwd=work_dir, timeout=60)
|
|
|
|
combined_log = result["stdout"] + "\n" + result["stderr"]
|
|
|
|
judgment = judge_log(
|
|
operation="finalize with early and late hooks",
|
|
expected_behavior="Both early and late hooks executed successfully, "
|
|
"printing 'finalize early hook' and 'finalize late hook'",
|
|
actual_log=combined_log,
|
|
)
|
|
assert judgment.passed, f"LLM judgment failed: {judgment.reason}"
|
|
|
|
|
|
@pytest.mark.finalize
|
|
class TestFinalizePlugins:
|
|
|
|
@pytest.mark.xfail(reason="Plugin download requires network/git access", strict=False)
|
|
def test_plugin_download(self, run_baker, work_dir):
|
|
pass
|
|
|
|
@pytest.mark.xfail(reason="Artifact stage not yet implemented", strict=False)
|
|
def test_artifact_collection(self, run_baker, work_dir):
|
|
pass
|