INNER CODE UNIT · Python
grade_output
wgpsec/AboutSecurity · scripts/bench-skill.py:164
def grade_output(output: str, expectations: list[str]) -> dict:
"""
Grade an output against expectations using keyword/pattern matching.
For more sophisticated grading, use grade_eval.py with an LLM grader.
This is a fast heuristic that checks if expectation keywords appear in the output.
"""
results = []
for expectation in expectations:
# Extract key phrases from expectation for matching
# Simple heuristic: check if the core terms appear in output
passed = check_expectation(output, expectation)
results.append({
"text": expectation,
"passed": passed,
"evidence": f"{'Found' if passed else 'Not found'} in output ({len(output)} chars)",
})