INNER CODE UNIT · Python
grade_run_dir
wgpsec/AboutSecurity · scripts/grade_eval.py:136
def grade_run_dir(run_dir: Path, evals_data: dict, model: str = None) -> dict:
"""Grade a single run directory."""
output_file = run_dir / "output.txt"
if not output_file.exists():
return {"error": f"No output.txt in {run_dir}"}
output_text = output_file.read_text(encoding="utf-8")
# Find corresponding eval expectations
grading_file = run_dir / "grading.json"
if grading_file.exists():
existing = json.loads(grading_file.read_text(encoding="utf-8"))
expectations = existing.get("expectations", [])
else:
expectations = []
if not expectations:
return {"error": "No expectations to grade against"}