INNER CODE UNIT · Python
grade_with_llm
wgpsec/AboutSecurity · scripts/grade_eval.py:30
def grade_with_llm(
output_text: str,
expectations: list[dict],
model: str = None,
timeout: int = 60,
) -> dict:
"""Grade output against expectations using an LLM."""
expectations_text = "\n".join(
f"{i+1}. {exp['text']}" for i, exp in enumerate(expectations)
)
prompt = f"""你是一个评估器。请评估以下 AI Agent 输出是否满足给定的期望。
## 待评估的输出
<output>
{output_text[:8000]}