INNER CODE UNIT · Python

run_skill_benchmark

wgpsec/AboutSecurity · scripts/bench-skill.py:369

def run_skill_benchmark(
    skill_path: Path,
    runs_per_config: int = 1,
    model: str = None,
    timeout: int = 120,
    grade_only: bool = False,
    verbose: bool = True,
) -> Path:
    """Run full benchmark for a skill. Returns workspace path."""
    name, description, body = parse_skill_md(skill_path)
    evals_data = load_evals(skill_path)
    evals = evals_data.get("evals", [])

    if not evals:
        raise ValueError(f"No evals defined in {skill_path / 'evals' / 'evals.json'}")

    # Create workspace
    timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")

View source record →