Catty Steve
84b3b7d8e8
Adds a complete pytest system test suite for the HoneyBiscuitWorkshop CI/CD system: - runner.sh: orchestrates build, Docker image creation, and test execution - Dockerfile: Arch Linux-based test image with Python venv - conftest.py: shared fixtures (baker_bin, run_baker, work_dir, llm_env) - probes/: environment probes for checking users, files, packages, processes - llm/: LLM-based log judgment system with caching for semantic test evaluation - fixtures/: test configs and scripts for prebake, bake, and finalize stages - test_prebake.py, test_bake.py, test_finalize.py: test suites for each pipeline stage Also refactors finalize.yml.tmpl notification config to use a templates system.
150 lines
4.2 KiB
Python
150 lines
4.2 KiB
Python
"""
|
|
LLM-based log judgment for system tests.
|
|
|
|
Evaluates logs against expected behavior using LLM semantic analysis.
|
|
Returns structured pass/fail results with reasoning.
|
|
"""
|
|
|
|
import json
|
|
import re
|
|
import logging
|
|
from dataclasses import dataclass
|
|
from .client import LLMClient
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
JUDGE_SYSTEM_PROMPT = """\
|
|
You are a test result judge for a CI/CD system called HoneyBiscuitWorkshop.
|
|
|
|
Your job: Evaluate whether actual command output/logs match expected behavior.
|
|
|
|
RULES:
|
|
1. You MUST respond with valid JSON in this exact format:
|
|
{"verdict": "pass" | "fail", "reason": "brief explanation"}
|
|
2. "pass" = the output confirms the expected behavior occurred
|
|
3. "fail" = the output shows the expected behavior did NOT occur, or shows an error
|
|
4. Ignore irrelevant noise (timestamps, debug lines, unrelated warnings)
|
|
5. Focus ONLY on whether the specific expected behavior is confirmed or contradicted
|
|
6. Be strict: if the evidence is ambiguous, verdict is "fail"
|
|
7. Do NOT include any text outside the JSON object"""
|
|
|
|
|
|
@dataclass
|
|
class JudgmentResult:
|
|
"""Result of an LLM log judgment."""
|
|
|
|
verdict: str # "pass" or "fail"
|
|
reason: str
|
|
raw_response: str
|
|
|
|
@property
|
|
def passed(self) -> bool:
|
|
return self.verdict == "pass"
|
|
|
|
|
|
def _extract_relevant_logs(full_log: str, max_chars: int = 4000) -> str:
|
|
"""Extract relevant portions from logs, removing noise.
|
|
|
|
Keeps the last *max_chars* characters (most recent / relevant).
|
|
Removes common noise patterns.
|
|
"""
|
|
lines = full_log.strip().split("\n")
|
|
|
|
noise_patterns = [
|
|
r"^\d{4}-\d{2}-\d{2}T",
|
|
r"^DEBUG ",
|
|
r"^TRACE ",
|
|
]
|
|
filtered = [
|
|
line
|
|
for line in lines
|
|
if not any(re.match(p, line) for p in noise_patterns)
|
|
]
|
|
|
|
text = "\n".join(filtered)
|
|
if len(text) > max_chars:
|
|
text = "...\n" + text[-max_chars:]
|
|
return text
|
|
|
|
|
|
def judge_log(
|
|
operation: str,
|
|
expected_behavior: str,
|
|
actual_log: str,
|
|
client: LLMClient | None = None,
|
|
) -> JudgmentResult:
|
|
"""Judge whether actual logs match expected behavior.
|
|
|
|
Args:
|
|
operation: Description of what was attempted
|
|
(e.g. ``"prebake bootstrap stage"``).
|
|
expected_behavior: What should have happened
|
|
(e.g. ``"user vulcan was created"``).
|
|
actual_log: The actual stdout / stderr output.
|
|
client: ``LLMClient`` instance (creates default if ``None``).
|
|
|
|
Returns:
|
|
``JudgmentResult`` with verdict and reason.
|
|
"""
|
|
if client is None:
|
|
client = LLMClient()
|
|
|
|
if not client.available:
|
|
logger.warning("No LLM client available, returning fail verdict")
|
|
return JudgmentResult(
|
|
verdict="fail",
|
|
reason="No LLM client available for judgment",
|
|
raw_response="",
|
|
)
|
|
|
|
relevant_log = _extract_relevant_logs(actual_log)
|
|
|
|
prompt = f"""\
|
|
## Operation
|
|
{operation}
|
|
|
|
## Expected Behavior
|
|
{expected_behavior}
|
|
|
|
## Actual Output (relevant excerpts)
|
|
```
|
|
{relevant_log}
|
|
```
|
|
|
|
Evaluate: Does the actual output confirm the expected behavior occurred?
|
|
Respond with JSON only."""
|
|
|
|
try:
|
|
response = client.complete(
|
|
prompt=prompt,
|
|
system_prompt=JUDGE_SYSTEM_PROMPT,
|
|
max_tokens=500,
|
|
)
|
|
|
|
# Parse JSON response — handle potential markdown code blocks
|
|
text = response.strip()
|
|
if text.startswith("```"):
|
|
lines = text.split("\n")
|
|
text = "\n".join(lines[1:-1]) if len(lines) > 2 else text
|
|
|
|
result = json.loads(text)
|
|
return JudgmentResult(
|
|
verdict=result.get("verdict", "fail"),
|
|
reason=result.get("reason", "No reason provided"),
|
|
raw_response=response,
|
|
)
|
|
except json.JSONDecodeError as e:
|
|
logger.error(f"Failed to parse LLM response as JSON: {e}")
|
|
return JudgmentResult(
|
|
verdict="fail",
|
|
reason=f"LLM response was not valid JSON: {response[:200]}",
|
|
raw_response=response,
|
|
)
|
|
except Exception as e:
|
|
logger.error(f"LLM judgment failed: {e}")
|
|
return JudgmentResult(
|
|
verdict="fail",
|
|
reason=f"LLM call failed: {str(e)}",
|
|
raw_response="",
|
|
)
|