feat: PLANNING体系首版（00路线图/02任务总表/03执行协议/I-03/I-04/registry42口径）+M1止血交付
This commit is contained in:
@@ -0,0 +1,841 @@
|
||||
#!/usr/bin/env python3
|
||||
"""evals runner —— 把六维题库串成一次可复现的模型评估（标准库，零依赖）。
|
||||
|
||||
用法：
|
||||
# 0) 题库自检：题面/满分/判分器/语料/fixture 一致性
|
||||
python run_eval.py --audit
|
||||
|
||||
# 1) 只看题面清单（确认解析正确、且不含评分点）
|
||||
python run_eval.py --list-questions
|
||||
|
||||
# 2) 干跑：产出将发送的 prompt 全文，不发网络请求
|
||||
python run_eval.py --model glm-5.3-flash --transport dry-run
|
||||
|
||||
# 3) 真跑（需网关可达 + 环境变量里的 key）
|
||||
set DSH_API_KEY=sk-xxx
|
||||
python run_eval.py --model glm-5.3-flash --transport openai \
|
||||
--base-url http://<dsh-gateway>/v1 --api-key-env DSH_API_KEY
|
||||
|
||||
# 4) 离线重放（用已录制的回答目录，answers/<qid>.txt）
|
||||
python run_eval.py --model demo --transport replay --answers-dir ../fixtures/selftest/good
|
||||
|
||||
# 5) 评委补判后定稿（产出 grade）
|
||||
python run_eval.py --model glm-5.3-flash --manual results/glm-5.3-flash-2026-09-12.manual.json
|
||||
|
||||
设计要点：
|
||||
- 只发「给模型的题面」段；**评分点段永不进入 prompt**（解析时硬校验）。
|
||||
- 自动判分复用 validate.py；人工项不自动给分，登记为 pending。
|
||||
- 评估协议（evals/README.md）：全新会话、逐题单发、每维 ≤10 分钟。
|
||||
- 密钥只从环境变量读取，本脚本不读任何配置文件里的凭据。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
EVALS = HERE.parent # evals/
|
||||
PLANNING = EVALS.parent # PLANNING/
|
||||
RESULTS = EVALS / "results"
|
||||
FIXTURE_SRC = EVALS / "fixtures" / "tooltask"
|
||||
CORPUS = EVALS / "assets" / "longctx-corpus.md"
|
||||
PROTOCOL = PLANNING / "03-执行协议.md"
|
||||
SCORING = EVALS / "scoring.md"
|
||||
REGISTRY = PLANNING / "model-registry.json"
|
||||
|
||||
sys.path.insert(0, str(HERE))
|
||||
import validate as V # noqa: E402
|
||||
|
||||
DIM_FILES = {
|
||||
"dim1": "dim1-instruction.md",
|
||||
"dim2": "dim2-code.md",
|
||||
"dim3": "dim3-context.md",
|
||||
"dim4": "dim4-tools.md",
|
||||
"dim5": "dim5-hallucination.md",
|
||||
"dim6": "dim6-delivery.md",
|
||||
}
|
||||
# 维度满分（= scoring.md 的维度表，与题面文件的「（N 分）」声明交叉校验）
|
||||
EXPECTED_DIM_MAX = {"dim1": 11, "dim2": 7, "dim3": 6, "dim4": 6, "dim5": 6, "dim6": 6}
|
||||
# 各维题量：dim4/dim6 是 2 题（见 evals/README 目录说明与 I-04 卡的资产表），其余 3 题 → 共 16 题
|
||||
EXPECTED_DIM_QTY = {"dim1": 3, "dim2": 3, "dim3": 3, "dim4": 2, "dim5": 3, "dim6": 2}
|
||||
# 总分上限 = 维度表求和。注意：README/scoring.md/registry 的标题写「36 分」，
|
||||
# 与维度表合计 42 不符 —— 这是规格内部的算术矛盾，登记为 SPEC-DEFECT-1，由规划者裁定；
|
||||
# 本 runner 一律以题面文件为准（题面是唯一同时被评分点和判分器约束的源）。
|
||||
EXPECTED_TOTAL_MAX = sum(EXPECTED_DIM_MAX.values()) # 42
|
||||
SPEC_HEADLINE_TOTAL = 36 # scoring.md / README / registry 的标题数字
|
||||
SPEC_DEFECTS = {
|
||||
"SPEC-DEFECT-1": (
|
||||
f"README §四 / evals/README §四 / scoring.md §一 称“满分 {SPEC_HEADLINE_TOTAL} 分”，"
|
||||
f"但 scoring.md 维度表各行相加 = {EXPECTED_TOTAL_MAX}，且与 16 道题面声明的分值合计一致。"
|
||||
f"等级换算用的是百分比，故不影响评级；但绝对总分口径需规划者裁定后统一。"
|
||||
),
|
||||
}
|
||||
DIM_TITLES = {
|
||||
"dim1": "指令遵循", "dim2": "代码修改", "dim3": "长上下文",
|
||||
"dim4": "工具调用", "dim5": "幻觉抵抗", "dim6": "交付规范",
|
||||
}
|
||||
HEAD_RE = re.compile(r"^##\s*Q(\d)\.(\d)\s*[·:：]?\s*(.*)$", re.M)
|
||||
POINTS_RE = re.compile(r"[（(]\s*(\d+)\s*分\s*[）)]")
|
||||
|
||||
API_KEY_HINT = "DSH_API_KEY"
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ 题面解析
|
||||
|
||||
class Question:
|
||||
def __init__(self, qid: str, dim: str, title: str, declared: int | None, prompt: str, src: Path):
|
||||
self.qid = qid
|
||||
self.dim = dim
|
||||
self.title = title.strip()
|
||||
self.declared = declared
|
||||
self.prompt = prompt.strip()
|
||||
self.src = src
|
||||
|
||||
def __repr__(self) -> str: # pragma: no cover
|
||||
return f"<Q {self.qid} {self.title!r} declared={self.declared}>"
|
||||
|
||||
|
||||
def parse_question_file(path: Path) -> list[Question]:
|
||||
"""从一份维度题面里抽出全部问题；只取「给模型的题面」围栏块。"""
|
||||
text = path.read_text(encoding="utf-8")
|
||||
marks = list(HEAD_RE.finditer(text))
|
||||
out: list[Question] = []
|
||||
for i, m in enumerate(marks):
|
||||
dim_no, q_no = m.group(1), m.group(2)
|
||||
end = marks[i + 1].start() if i + 1 < len(marks) else len(text)
|
||||
block = text[m.start():end]
|
||||
qid = f"dim{dim_no}-q{q_no}"
|
||||
prompt = _extract_prompt(block, path.name, qid)
|
||||
out.append(Question(qid, f"dim{dim_no}", m.group(3), _declared_points(m.group(3)),
|
||||
prompt, path))
|
||||
return out
|
||||
|
||||
|
||||
def _declared_points(title: str) -> int | None:
|
||||
m = POINTS_RE.search(title)
|
||||
return int(m.group(1)) if m else None
|
||||
|
||||
|
||||
def _extract_prompt(block: str, fname: str, qid: str) -> str:
|
||||
seg = re.search(r"###\s*给模型的题面\s*\n(.*?)(?=\n###|\Z)", block, re.S)
|
||||
if not seg:
|
||||
raise ValueError(f"{fname} {qid}: 找不到「给模型的题面」段")
|
||||
fences = re.findall(r"```[a-zA-Z0-9_+-]*\n(.*?)```", seg.group(1), re.S)
|
||||
if not fences:
|
||||
raise ValueError(f"{fname} {qid}: 「给模型的题面」段里没有围栏块")
|
||||
prompt = fences[0].strip()
|
||||
# 硬校验：评分点绝不允许进入 prompt
|
||||
for banned in ("评分点", "勿发给被测模型", "判定方式", "自动（", "人工/评委"):
|
||||
if banned in prompt:
|
||||
raise ValueError(f"{fname} {qid}: prompt 含禁用内容 {banned!r}")
|
||||
return prompt
|
||||
|
||||
|
||||
def load_questions() -> list[Question]:
|
||||
qs: list[Question] = []
|
||||
for dim, fname in DIM_FILES.items():
|
||||
qs.extend(parse_question_file(EVALS / "questions" / fname))
|
||||
return qs
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ 传输层
|
||||
|
||||
class TransportError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class DryRunTransport:
|
||||
name = "dry-run"
|
||||
|
||||
def ask(self, qid, messages, model, timeout) -> tuple[str, dict]:
|
||||
return "", {"note": "dry-run：未发送请求"}
|
||||
|
||||
|
||||
class ReplayTransport:
|
||||
"""离线重放：从目录读 <qid>.txt|md 作为「模型回答」。"""
|
||||
name = "replay"
|
||||
|
||||
def __init__(self, answers_dir: Path):
|
||||
self.dir = answers_dir
|
||||
|
||||
def ask(self, qid, messages, model, timeout) -> tuple[str, dict]:
|
||||
for ext in (".txt", ".md"):
|
||||
f = self.dir / f"{qid}{ext}"
|
||||
if f.exists():
|
||||
return f.read_text(encoding="utf-8"), {"note": f"replay:{f.name}"}
|
||||
raise TransportError(f"replay 目录里没有 {qid}.txt|.md —— {self.dir}")
|
||||
|
||||
|
||||
class OpenAITransport:
|
||||
"""OpenAI 兼容 /chat/completions。密钥只从环境变量取，绝不落盘、绝不打印。"""
|
||||
name = "openai"
|
||||
|
||||
def __init__(self, base_url: str, api_key: str, temperature: float | None = None,
|
||||
max_tokens: int | None = None, retries: int = 2):
|
||||
self.url = base_url.rstrip("/") + "/chat/completions"
|
||||
self.key = api_key
|
||||
self.temperature = temperature
|
||||
self.max_tokens = max_tokens
|
||||
self.retries = retries
|
||||
self.usage: list[dict] = []
|
||||
|
||||
def ask(self, qid, messages, model, timeout) -> tuple[str, dict]:
|
||||
body: dict = {"model": model, "messages": messages}
|
||||
if self.temperature is not None:
|
||||
body["temperature"] = self.temperature
|
||||
if self.max_tokens:
|
||||
body["max_tokens"] = self.max_tokens
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
last = ""
|
||||
for attempt in range(self.retries + 1):
|
||||
req = urllib.request.Request(
|
||||
self.url, data=data, method="POST",
|
||||
headers={"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {self.key}"},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
payload = json.loads(resp.read().decode("utf-8", "replace"))
|
||||
text = _pick_content(payload)
|
||||
meta = {"note": "openai", "usage": payload.get("usage") or {}}
|
||||
self.usage.append({"qid": qid, **meta["usage"]})
|
||||
return text, meta
|
||||
except urllib.error.HTTPError as e:
|
||||
detail = _safe_err(e)
|
||||
last = f"HTTP {e.code}: {detail}"
|
||||
if e.code in (408, 409, 429, 500, 502, 503, 504) and attempt < self.retries:
|
||||
time.sleep(2 * (attempt + 1))
|
||||
continue
|
||||
raise TransportError(last) from None
|
||||
except Exception as e: # noqa: BLE001
|
||||
last = f"{type(e).__name__}: {e}"
|
||||
if attempt < self.retries:
|
||||
time.sleep(2 * (attempt + 1))
|
||||
continue
|
||||
raise TransportError(last) from None
|
||||
raise TransportError(last)
|
||||
|
||||
|
||||
def _pick_content(payload: dict) -> str:
|
||||
try:
|
||||
msg = payload["choices"][0]["message"]
|
||||
except Exception: # noqa: BLE001
|
||||
raise TransportError(f"响应结构异常：{json.dumps(payload, ensure_ascii=False)[:300]}") from None
|
||||
content = msg.get("content")
|
||||
if isinstance(content, list): # 多段 content
|
||||
content = "".join(seg.get("text", "") for seg in content if isinstance(seg, dict))
|
||||
return (content or "").strip()
|
||||
|
||||
|
||||
def _safe_err(e: urllib.error.HTTPError) -> str:
|
||||
try:
|
||||
raw = e.read().decode("utf-8", "replace")
|
||||
except Exception: # noqa: BLE001
|
||||
return "(无法读取响应体)"
|
||||
raw = re.sub(r"(sk-|Bearer\s+)[A-Za-z0-9._\-]{6,}", r"\1<redacted>", raw)
|
||||
return raw[:300]
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ prompt 组装
|
||||
|
||||
def progress_template() -> str:
|
||||
"""抽 03-执行协议.md 的 PROGRESS 章节，作为 dim6-q1 的题面附件。"""
|
||||
if not PROTOCOL.exists():
|
||||
return ""
|
||||
text = PROTOCOL.read_text(encoding="utf-8")
|
||||
blocks = [b for b in re.split(r"\n(?=## )", text) if b.startswith("## 二、PROGRESS")]
|
||||
return blocks[0].strip() if blocks else ""
|
||||
|
||||
|
||||
def build_messages(q: Question, workdir: Path | None, corpus: str) -> list[dict]:
|
||||
prompt = q.prompt
|
||||
if q.qid in ("dim4-q1", "dim4-q2"):
|
||||
if workdir is None:
|
||||
raise TransportError("dim4 需要副本目录")
|
||||
prompt = prompt.replace("<fixtures 副本路径>", str(workdir))
|
||||
prompt = prompt.replace("<fixtures 副本路径>", str(workdir))
|
||||
if q.qid == "dim3-q1":
|
||||
pass # 语料单独给
|
||||
if q.qid == "dim6-q1":
|
||||
tpl = progress_template()
|
||||
if tpl:
|
||||
prompt += "\n\n（以下是 PLANNING/03-执行协议.md 中规定的 PROGRESS 格式章节）\n\n" + tpl
|
||||
if q.dim == "dim3":
|
||||
return [
|
||||
{"role": "user", "content": "这是一份运维文档，请先阅读，随后我会提问。\n\n" + corpus},
|
||||
{"role": "assistant", "content": "已读完这份文档，请提问。"},
|
||||
{"role": "user", "content": prompt},
|
||||
]
|
||||
return [{"role": "user", "content": prompt}]
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ 判分汇总
|
||||
|
||||
def _pct(got: int, mx: int) -> float:
|
||||
return (got / mx) if mx else 0.0
|
||||
|
||||
|
||||
def compute_grade(total_got: int, total_max: int, dim_scores: dict, veto_flags: list[str]):
|
||||
if veto_flags:
|
||||
return "C", "硬否决：" + "；".join(veto_flags)
|
||||
pct = _pct(total_got, total_max)
|
||||
d2 = _pct(dim_scores.get("dim2", {}).get("got", 0), dim_scores.get("dim2", {}).get("max", 1))
|
||||
d4 = _pct(dim_scores.get("dim4", {}).get("got", 0), dim_scores.get("dim4", {}).get("max", 1))
|
||||
if pct >= 0.90 and d2 >= 0.90 and d4 >= 0.90:
|
||||
return "S", f"总分 {pct:.0%}，dim2 {d2:.0%} / dim4 {d4:.0%}"
|
||||
if pct >= 0.75:
|
||||
return "A", f"总分 {pct:.0%}"
|
||||
if pct >= 0.60:
|
||||
return "B", f"总分 {pct:.0%}"
|
||||
return "C", f"总分 {pct:.0%} < 60%"
|
||||
|
||||
|
||||
def score_answers(answers: dict[str, str], questions: list[Question],
|
||||
workdir: Path | None) -> tuple[dict, dict, list[dict]]:
|
||||
per_q: dict[str, dict] = {}
|
||||
dim_scores: dict[str, dict] = {}
|
||||
pending: list[dict] = []
|
||||
for q in questions:
|
||||
ans = answers.get(q.qid)
|
||||
if ans is None:
|
||||
continue
|
||||
rep = V.score_report(q.qid, ans, workdir=str(workdir) if q.qid.startswith("dim4") else None)
|
||||
per_q[q.qid] = rep
|
||||
d = dim_scores.setdefault(q.dim, {"got": 0, "max": 0, "pending": 0})
|
||||
d["got"] += rep["auto_got"]
|
||||
d["max"] += rep["max"]
|
||||
d["pending"] += rep["manual_pending"]
|
||||
for item in rep["manual_items"]:
|
||||
pending.append({"qid": q.qid, **item})
|
||||
return per_q, dim_scores, pending
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ 主流程
|
||||
|
||||
def sanitize(model: str) -> str:
|
||||
return re.sub(r"[^A-Za-z0-9._-]", "_", model)
|
||||
|
||||
|
||||
def prepare_workdir(model: str, reuse: bool) -> Path:
|
||||
wd = RESULTS / "work" / sanitize(model) / "tooltask"
|
||||
if wd.exists() and reuse:
|
||||
return wd
|
||||
if wd.exists():
|
||||
shutil.rmtree(wd)
|
||||
wd.mkdir(parents=True)
|
||||
for f in ("calc.py", "test_calc.py"):
|
||||
src = FIXTURE_SRC / f
|
||||
if src.exists():
|
||||
shutil.copy2(src, wd / f)
|
||||
return wd
|
||||
|
||||
|
||||
def run_eval(args) -> int:
|
||||
questions = load_questions()
|
||||
if args.only:
|
||||
# 同时接受维度（dim1）与题号（dim1-q1），避免静默选空
|
||||
sel = {s.strip() for s in args.only.split(",") if s.strip()}
|
||||
unknown = {s for s in sel if s not in DIM_FILES and not re.fullmatch(r"dim\d-q\d", s)}
|
||||
if unknown:
|
||||
print(f"--only 里有无法识别的选择器：{sorted(unknown)}")
|
||||
print("可用：" + "、".join(DIM_FILES) + " 或 dim1-q1 形式")
|
||||
return 1
|
||||
questions = [q for q in questions if q.dim in sel or q.qid in sel]
|
||||
if args.questions:
|
||||
want = {s.strip() for s in args.questions.split(",") if s.strip()}
|
||||
unknown = {s for s in want if not re.fullmatch(r"dim\d-q\d", s)}
|
||||
if unknown:
|
||||
print(f"--questions 里有无法识别的题号：{sorted(unknown)}")
|
||||
return 1
|
||||
questions = [q for q in questions if q.qid in want]
|
||||
if not questions:
|
||||
print("选中的题目为空 —— 检查 --only / --questions 的选择器")
|
||||
return 1
|
||||
|
||||
if args.list_questions:
|
||||
print(f"{'QID':<10} {'满分':<4} 标题")
|
||||
print("-" * 70)
|
||||
for q in questions:
|
||||
print(f"{q.qid:<10} {str(q.declared or '?'):<4} {q.title}")
|
||||
print("-" * 70)
|
||||
print(f"共 {len(questions)} 题；题面长度合计 "
|
||||
f"{sum(len(q.prompt) for q in questions)} 字符")
|
||||
return 0
|
||||
|
||||
if args.audit:
|
||||
return run_audit(questions)
|
||||
|
||||
if args.manual:
|
||||
return merge_manual(Path(args.manual))
|
||||
|
||||
model = args.model
|
||||
if not model:
|
||||
print("需要 --model（或 --audit / --list-questions / --manual）")
|
||||
return 1
|
||||
|
||||
RESULTS.mkdir(parents=True, exist_ok=True)
|
||||
corpus = CORPUS.read_text(encoding="utf-8") if any(q.dim == "dim3" for q in questions) else ""
|
||||
if not any(q.dim == "dim4" for q in questions):
|
||||
workdir = None
|
||||
elif args.workdir:
|
||||
workdir = Path(args.workdir).resolve()
|
||||
if not (workdir / "calc.py").exists():
|
||||
print(f"--workdir 里没有 calc.py：{workdir}")
|
||||
return 1
|
||||
else:
|
||||
workdir = prepare_workdir(model, args.reuse_workdir)
|
||||
|
||||
# ---- 传输层选择
|
||||
if args.transport == "dry-run":
|
||||
transport = DryRunTransport()
|
||||
if args.api_key_from_env:
|
||||
print(f"[WARN] dry-run 不需要密钥，忽略 --api-key-env {args.api_key_from_env}")
|
||||
elif args.transport == "replay":
|
||||
if not args.answers_dir:
|
||||
print("--transport replay 需要 --answers-dir")
|
||||
return 1
|
||||
transport = ReplayTransport(Path(args.answers_dir))
|
||||
else:
|
||||
base_url = args.base_url or _config_get(args, "base_url")
|
||||
if not base_url:
|
||||
print("--transport openai 需要 --base-url（或在 eval-config.json 里配 base_url）")
|
||||
return 1
|
||||
env_name = args.api_key_env or _config_get(args, "api_key_env") or API_KEY_HINT
|
||||
import os
|
||||
key = os.environ.get(env_name, "")
|
||||
if not key:
|
||||
print(f"环境变量 {env_name} 未设置 —— 密钥只从环境变量读取（本工具不读配置文件里的凭据）")
|
||||
print(f" PowerShell: $env:{env_name}='<your-key>'")
|
||||
print(f" bash: export {env_name}='<your-key>'")
|
||||
return 1
|
||||
transport = OpenAITransport(base_url, key,
|
||||
temperature=args.temperature,
|
||||
max_tokens=args.max_tokens)
|
||||
|
||||
stamp = date.today().isoformat()
|
||||
raw_dir = RESULTS / "raw" / sanitize(model)
|
||||
raw_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
print(f"模型={model} 传输={transport.name} 题目={len(questions)} "
|
||||
f"超时={args.timeout}s 分数上限={sum(q.declared or 0 for q in questions)}")
|
||||
answers: dict[str, str] = {}
|
||||
sent: dict[str, str] = {}
|
||||
for q in questions:
|
||||
try:
|
||||
messages = build_messages(q, workdir, corpus)
|
||||
except TransportError as e:
|
||||
print(f" [SKIP] {q.qid}: {e}")
|
||||
continue
|
||||
sent[q.qid] = messages[-1]["content"]
|
||||
if args.transport == "dry-run":
|
||||
(raw_dir / f"{q.qid}.prompt.md").write_text(
|
||||
"\n\n=== role boundary ===\n\n".join(
|
||||
f"[{m['role']}]\n{m['content']}" for m in messages), encoding="utf-8")
|
||||
print(f" [DRY ] {q.qid} prompt={len(messages[-1]['content'])} 字符 → {q.qid}.prompt.md")
|
||||
continue
|
||||
t0 = time.time()
|
||||
try:
|
||||
text, meta = transport.ask(q.qid, messages, model, args.timeout)
|
||||
except TransportError as e:
|
||||
print(f" [FAIL] {q.qid}: {e}")
|
||||
answers[q.qid] = ""
|
||||
(raw_dir / f"{q.qid}.error.txt").write_text(str(e), encoding="utf-8")
|
||||
continue
|
||||
dt = time.time() - t0
|
||||
answers[q.qid] = text
|
||||
(raw_dir / f"{q.qid}.txt").write_text(text, encoding="utf-8")
|
||||
note = meta.get("note", "")
|
||||
print(f" [ OK ] {q.qid} {dt:5.1f}s {len(text):6d} 字符 {note}")
|
||||
|
||||
if args.transport == "dry-run":
|
||||
print(f"\n干跑完成：prompt 已写入 {raw_dir}")
|
||||
return 0
|
||||
|
||||
got_any = {k: v for k, v in answers.items() if v.strip()}
|
||||
if not got_any:
|
||||
print("\n全部题目都没有拿到回答（网关不可达 / 凭据无效 / 题目全被跳过）——"
|
||||
"不产出结果文件，避免留下 0/0 的空成绩。")
|
||||
return 1
|
||||
|
||||
per_q, dim_scores, pending = score_answers(answers, questions, workdir)
|
||||
|
||||
auto_got = sum(v["auto_got"] for v in per_q.values())
|
||||
total_max = sum(v["max"] for v in per_q.values())
|
||||
total_pending = sum(v["manual_pending"] for v in per_q.values())
|
||||
dim5 = dim_scores.get("dim5", {"got": 0, "max": 0})
|
||||
veto_hint: list[str] = []
|
||||
if dim5["max"] and _pct(dim5["got"], dim5["max"]) < 0.5:
|
||||
veto_hint.append(f"dim5 自动项 {dim5['got']}/{dim5['max']} <50%（待人工复核后裁定）")
|
||||
if any(v == "" for v in answers.values()):
|
||||
failed = [k for k, v in answers.items() if v == ""]
|
||||
veto_hint.append(f"存在无回答题：{failed}")
|
||||
|
||||
grade, basis = (None, "待评委补判人工项")
|
||||
if total_pending == 0:
|
||||
grade, basis = compute_grade(auto_got, total_max, dim_scores, [])
|
||||
|
||||
payload = {
|
||||
"model": model,
|
||||
"group": args.group or _registry_group(model),
|
||||
"date": stamp,
|
||||
"runner": "run_eval.py",
|
||||
"transport": transport.name,
|
||||
"base_url": (args.base_url or _config_get(args, "base_url") or None) if transport.name == "openai" else None,
|
||||
"scores": {d: {"got": s["got"], "max": s["max"],
|
||||
"manual_pending": s["pending"],
|
||||
"notes": _dim_notes(per_q, d)}
|
||||
for d, s in sorted(dim_scores.items())},
|
||||
"per_question": {q: {"auto_got": r["auto_got"], "max": r["max"],
|
||||
"manual_pending": r["manual_pending"], "logs": r["logs"]}
|
||||
for q, r in sorted(per_q.items())},
|
||||
"total": {"auto_got": auto_got, "manual_pending": total_pending,
|
||||
"max": total_max, "got": auto_got if total_pending == 0 else None},
|
||||
"hard_veto": bool(veto_hint),
|
||||
"hard_veto_hint": veto_hint,
|
||||
"grade": grade,
|
||||
"grade_basis": basis,
|
||||
"transcript": f"evals/results/{sanitize(model)}-{stamp}.log.md",
|
||||
"score_scale": {"expected_total_max": EXPECTED_TOTAL_MAX,
|
||||
"expected_dims": EXPECTED_DIM_MAX},
|
||||
}
|
||||
out_json = RESULTS / f"{sanitize(model)}-{stamp}.json"
|
||||
out_json.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
_write_transcript(RESULTS / f"{sanitize(model)}-{stamp}.log.md", model, stamp, questions,
|
||||
answers, per_q, transport, sent)
|
||||
_write_report(RESULTS / f"{sanitize(model)}-{stamp}.md", payload)
|
||||
|
||||
if total_pending:
|
||||
tpl = RESULTS / f"{sanitize(model)}-{stamp}.manual.template.json"
|
||||
tpl.write_text(json.dumps({
|
||||
"model": model, "judge": "", "date": stamp,
|
||||
"items": {p["qid"]: None for p in pending},
|
||||
"criteria": [{"qid": p["qid"], "points": p["points"], "criterion": p["criterion"]}
|
||||
for p in pending],
|
||||
"veto_flags": [],
|
||||
"notes": {},
|
||||
}, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(f"\n人工项待补判 {total_pending} 分 → 模板：{tpl}")
|
||||
print(f"补判后执行：python run_eval.py --manual {tpl.name}")
|
||||
|
||||
print(f"\n自动项 {auto_got}/{total_max}（待评委 {total_pending} 分）")
|
||||
for d, s in sorted(dim_scores.items()):
|
||||
flag = f" 待人工 {s['pending']}" if s["pending"] else ""
|
||||
print(f" {d} {DIM_TITLES[d]:<6} {s['got']:>2}/{s['max']:<2}{flag}")
|
||||
print(f"\n结果：{out_json}")
|
||||
if veto_hint:
|
||||
print("[硬否决提示] " + "；".join(veto_hint))
|
||||
return 0
|
||||
|
||||
|
||||
def _dim_notes(per_q: dict, dim: str) -> str:
|
||||
bits = []
|
||||
for qid, rep in per_q.items():
|
||||
if qid.startswith(dim + "-") and rep["auto_got"] < rep["max"]:
|
||||
failed = [ln for ln in rep["logs"] if ln.startswith("[FAIL]")]
|
||||
if failed:
|
||||
bits.append(f"{qid}: " + failed[0][7:][:60])
|
||||
return "；".join(bits)
|
||||
|
||||
|
||||
def _write_transcript(path: Path, model: str, stamp: str, questions: list[Question],
|
||||
answers: dict[str, str], per_q: dict, transport,
|
||||
sent: dict[str, str]) -> None:
|
||||
lines = [f"# 评估 transcript — {model}（{stamp}）",
|
||||
"", f"- 传输：`{transport.name}`", f"- 题目数：{len(answers)}",
|
||||
"- 协议：全新会话 / 逐题单发 / 不预告下一题（见 evals/README.md）", "", "---", ""]
|
||||
for q in questions:
|
||||
if q.qid not in answers:
|
||||
continue
|
||||
lines += [f"## {q.qid} · {q.title}", "",
|
||||
"### 题面（实际发送，末条 user 消息）", "", "```text",
|
||||
sent.get(q.qid, ""), "```", "",
|
||||
"### 模型回答", "", "```text", answers[q.qid], "```", "",
|
||||
"### 自动判分", ""]
|
||||
for ln in per_q.get(q.qid, {}).get("logs", []):
|
||||
lines.append(f"- {ln}")
|
||||
lines.append("")
|
||||
path.write_text("\n".join(lines), encoding="utf-8")
|
||||
|
||||
|
||||
def _write_report(path: Path, payload: dict) -> None:
|
||||
t = payload["total"]
|
||||
lines = [f"# 评估报告 — {payload['model']}（{payload['date']}）", "",
|
||||
f"- 传输：`{payload['transport']}`",
|
||||
f"- 自动得分：**{t['auto_got']}/{t['max']}**，待评委：**{t['manual_pending']}**",
|
||||
f"- 等级：**{payload['grade'] or '待评委补判'}**（{payload['grade_basis']}）", "",
|
||||
"| 维度 | 自动 | 满分 | 待人工 |", "|---|---|---|---|"]
|
||||
for d, s in sorted(payload["scores"].items()):
|
||||
lines.append(f"| {d} {DIM_TITLES.get(d, '')} | {s['got']} | {s['max']} | {s['manual_pending']} |")
|
||||
lines += ["", "## 逐题", "", "| 题 | 自动 | 满分 | 待人工 |", "|---|---|---|---|"]
|
||||
for q, r in sorted(payload["per_question"].items()):
|
||||
lines.append(f"| {q} | {r['auto_got']} | {r['max']} | {r['manual_pending']} |")
|
||||
if payload["hard_veto_hint"]:
|
||||
lines += ["", "## 硬否决提示", ""] + [f"- {x}" for x in payload["hard_veto_hint"]]
|
||||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ 评委补判
|
||||
|
||||
def merge_manual(manual_path: Path) -> int:
|
||||
m = json.loads(manual_path.read_text(encoding="utf-8"))
|
||||
model = m.get("model")
|
||||
if not model:
|
||||
print("manual 文件缺 model 字段")
|
||||
return 1
|
||||
# 排除评委产物（.manual.json / .manual.template.json），只留 runner 主结果
|
||||
cands = sorted(c for c in RESULTS.glob(f"{sanitize(model)}-*.json") if ".manual" not in c.name)
|
||||
if not cands:
|
||||
print(f"找不到 {model} 的评估结果 json")
|
||||
return 1
|
||||
payload = json.loads(cands[-1].read_text(encoding="utf-8"))
|
||||
|
||||
items = m.get("items") or {}
|
||||
if any(v is None for v in items.values()):
|
||||
print("manual.items 里还有 null —— 补判未完成")
|
||||
return 1
|
||||
# 校验不超过各题待判分值
|
||||
cap: dict[str, int] = {}
|
||||
for qid, rep in payload["per_question"].items():
|
||||
cap[qid] = rep["manual_pending"]
|
||||
for qid, val in items.items():
|
||||
if qid not in cap:
|
||||
print(f"manual.items 有未知题号 {qid}")
|
||||
return 1
|
||||
if val > cap[qid]:
|
||||
print(f"{qid} 人工给分 {val} 超过待判上限 {cap[qid]}")
|
||||
return 1
|
||||
if sum(items.values()) < sum(cap.values()):
|
||||
missing = [q for q, c in cap.items() if items.get(q) is None]
|
||||
print(f"仍有未判题：{missing}")
|
||||
return 1
|
||||
|
||||
manual_got = sum(items.values())
|
||||
for qid, rep in payload["per_question"].items():
|
||||
rep["manual_got"] = items.get(qid, 0)
|
||||
for d, s in payload["scores"].items():
|
||||
s["manual_got"] = sum(items.get(q, 0) for q in payload["scores"] and
|
||||
[x for x in payload["per_question"] if x.startswith(d + "-")])
|
||||
s["got_total"] = s["got"] + s["manual_got"]
|
||||
veto = m.get("veto_flags") or []
|
||||
total_max = payload["total"]["max"]
|
||||
got = payload["total"]["auto_got"] + manual_got
|
||||
grade, basis = compute_grade(got, total_max, {
|
||||
d: {"got": payload["scores"][d]["got_total"], "max": payload["scores"][d]["max"]}
|
||||
for d in payload["scores"]}, veto)
|
||||
payload["total"].update({"got": got, "manual_got": manual_got, "manual_pending": 0})
|
||||
payload["grade"] = grade
|
||||
payload["grade_basis"] = basis
|
||||
payload["hard_veto"] = bool(veto)
|
||||
payload["hard_veto_flags"] = veto
|
||||
payload["judge"] = {"name": m.get("judge"), "date": m.get("date"), "notes": m.get("notes")}
|
||||
|
||||
out = cands[-1]
|
||||
out.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
_write_report(out.with_suffix(".md"), payload)
|
||||
print(f"已定稿：{out}")
|
||||
print(f"总分 {got}/{total_max}（自动 {payload['total']['auto_got']} + 人工 {manual_got}）"
|
||||
f" → 等级 {grade}（{basis}）")
|
||||
print("提醒：成绩回填 PLANNING/model-registry.json 由规划者执行，本工具不自动写注册表。")
|
||||
return 0
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ 题库自检
|
||||
|
||||
def _parse_scoring_md() -> dict[str, tuple[int, int]]:
|
||||
"""从 scoring.md 的维度表里读出 {dim: (题数, 满分)}。"""
|
||||
if not SCORING.exists():
|
||||
return {}
|
||||
out: dict[str, tuple[int, int]] = {}
|
||||
for ln in SCORING.read_text(encoding="utf-8").splitlines():
|
||||
m = re.match(r"\|\s*(dim\d)\s*[^|]*\|\s*(\d+)\s*\|\s*(\d+)\s*\|", ln)
|
||||
if m:
|
||||
out[m.group(1)] = (int(m.group(2)), int(m.group(3)))
|
||||
return out
|
||||
|
||||
|
||||
def run_audit(questions: list[Question]) -> int:
|
||||
checks: list[tuple[str, bool, str]] = []
|
||||
|
||||
def add(name: str, ok: bool, detail: str = "") -> None:
|
||||
checks.append((name, ok, detail))
|
||||
|
||||
# 1 题面解析 + 声明满分
|
||||
by_dim: dict[str, list[Question]] = {}
|
||||
for q in questions:
|
||||
by_dim.setdefault(q.dim, []).append(q)
|
||||
add(f"题量合计 16", len(questions) == sum(EXPECTED_DIM_QTY.values()),
|
||||
f"实际 {len(questions)} 题")
|
||||
for dim, qty in EXPECTED_DIM_QTY.items():
|
||||
qs = by_dim.get(dim, [])
|
||||
got = sum(q.declared or 0 for q in qs)
|
||||
add(f"{dim} 题量 {qty}", len(qs) == qty, f"实际 {len(qs)} 题")
|
||||
add(f"{dim} 满分 {EXPECTED_DIM_MAX[dim]}", got == EXPECTED_DIM_MAX[dim],
|
||||
f"题面声明合计 {got}")
|
||||
total_declared = sum(q.declared or 0 for q in questions)
|
||||
add(f"总满分 {EXPECTED_TOTAL_MAX}（维度表求和）", total_declared == EXPECTED_TOTAL_MAX,
|
||||
f"题面声明合计 {total_declared}")
|
||||
|
||||
# 2 与 scoring.md 对照
|
||||
sm = _parse_scoring_md()
|
||||
add("scoring.md 可解析", len(sm) == 6, f"解析到 {len(sm)} 行维度")
|
||||
for dim, (n, mx) in sm.items():
|
||||
add(f"scoring.md {dim} 满分 {mx} 与题面一致",
|
||||
mx == sum(q.declared or 0 for q in by_dim.get(dim, [])),
|
||||
f"题面 {sum(q.declared or 0 for q in by_dim.get(dim, []))}")
|
||||
# 规格已知缺陷：标题分 ≠ 维度表合计。不静默、不改规格，登记为 FAIL 提请裁定。
|
||||
add(f"[规格缺陷 SPEC-DEFECT-1] 标题分 {SPEC_HEADLINE_TOTAL} 与维度表合计 {EXPECTED_TOTAL_MAX} 不一致",
|
||||
False, SPEC_DEFECTS["SPEC-DEFECT-1"])
|
||||
|
||||
# 3 判分器覆盖 + 自动/人工配平
|
||||
for q in questions:
|
||||
has = q.qid in V.CHECKERS
|
||||
rep_max = None
|
||||
if has:
|
||||
dummy = "" if q.qid != "dim4-q1" else ""
|
||||
rep = V.score_report(q.qid, dummy, workdir=None)
|
||||
rep_max = rep["max"]
|
||||
add(f"{q.qid} 有自动判分器", has, "" if has else "缺 checker")
|
||||
if has and q.declared is not None:
|
||||
pending = sum(p for p, _ in V.MANUAL_POINTS.get(q.qid, []))
|
||||
add(f"{q.qid} 自动+人工={q.declared}",
|
||||
rep_max == q.declared and rep_max == (q.declared - pending) + pending,
|
||||
f"checker.max={rep_max} 声明={q.declared} 人工登记={pending}")
|
||||
|
||||
# 4 语料
|
||||
if CORPUS.exists():
|
||||
t = CORPUS.read_text(encoding="utf-8")
|
||||
add("语料 51,483 字符", len(t) == 51483, f"实际 {len(t)}")
|
||||
for name, key, lo, hi in [("针 XN-7742", "XN-7742", 0, 0.10),
|
||||
("针 300 秒", "300 秒", 0.45, 0.55),
|
||||
("针 NSM-7(末次)", "NSM-7", 0.88, 0.96),
|
||||
("诱饵 周四01:00", "周四 01:00", 0.15, 0.25),
|
||||
("诱饵 86400", "86400", 0.30, 0.40)]:
|
||||
idx = t.rfind(key) if name.endswith("(末次)") else t.find(key)
|
||||
pct = idx / len(t) if idx >= 0 else -1
|
||||
add(f"{name} 位置 {lo:.0%}–{hi:.0%}", 0 <= idx and lo <= pct <= hi,
|
||||
f"实测 {pct:.2%}" if idx >= 0 else "未找到")
|
||||
else:
|
||||
add("语料存在", False, str(CORPUS))
|
||||
|
||||
# 5 fixture 基线
|
||||
import subprocess
|
||||
import tempfile
|
||||
if FIXTURE_SRC.exists():
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
for f in ("calc.py", "test_calc.py"):
|
||||
shutil.copy2(FIXTURE_SRC / f, Path(td) / f)
|
||||
p = subprocess.run([sys.executable, "-m", "unittest", "-v"], cwd=td,
|
||||
capture_output=True, text=True, timeout=90)
|
||||
out = p.stdout + p.stderr
|
||||
m = re.search(r"Ran (\d+) tests", out)
|
||||
f_m = re.search(r"failures=(\d+)", out)
|
||||
e_m = re.search(r"errors=(\d+)", out)
|
||||
add("fixture 基线 9 用例", bool(m) and m.group(1) == "9",
|
||||
f"Ran={m.group(1) if m else '?'}")
|
||||
add("fixture 基线 failures=2, errors=2",
|
||||
bool(f_m) and bool(e_m) and f_m.group(1) == "2" and e_m.group(1) == "2",
|
||||
f"failures={f_m.group(1) if f_m else '?'} errors={e_m.group(1) if e_m else '?'}")
|
||||
else:
|
||||
add("fixture 存在", False, str(FIXTURE_SRC))
|
||||
|
||||
# 6 registry 刻度一致
|
||||
if REGISTRY.exists():
|
||||
reg = json.loads(REGISTRY.read_text(encoding="utf-8"))
|
||||
scale = reg.get("scale", {})
|
||||
add("registry total_max 与规格缺陷一致（应为待裁定的 36）",
|
||||
scale.get("total_max") == SPEC_HEADLINE_TOTAL,
|
||||
f"实际 {scale.get('total_max')}；裁定后需与 scoring.md 一并改口径")
|
||||
add("registry dims 与 scoring 一致", scale.get("dims") == EXPECTED_DIM_MAX,
|
||||
f"实际 {json.dumps(scale.get('dims'), ensure_ascii=False)}")
|
||||
add("registry 覆盖 13 模型", len(reg.get("models", [])) >= 13,
|
||||
f"实际 {len(reg.get('models', []))}")
|
||||
else:
|
||||
add("model-registry.json 存在", False, str(REGISTRY))
|
||||
|
||||
# 7 题面不含评分点
|
||||
for q in questions:
|
||||
add(f"{q.qid} 题面不含评分点",
|
||||
"评分点" not in q.prompt and "判定方式" not in q.prompt, "")
|
||||
|
||||
hard = [c for c in checks if "SPEC-DEFECT" not in c[0] and not c[0].startswith("[规格缺陷")]
|
||||
known = [c for c in checks if c not in hard]
|
||||
ok = sum(1 for _, o, _ in hard if o)
|
||||
print(f"{'检查项':<50} 结果")
|
||||
print("-" * 90)
|
||||
for name, o, detail in checks:
|
||||
mark = "[PASS]" if o else ("[KNOWN]" if name in [k[0] for k in known] else "[FAIL]")
|
||||
print(f"{mark} {name:<50} {detail}")
|
||||
print("-" * 90)
|
||||
print(f"AUDIT: {ok}/{len(hard)} structural checks passed"
|
||||
f"（另有 {len(known)} 项已登记规格缺陷，需规划者裁定）")
|
||||
return 0 if ok == len(hard) else 2
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ 配置读取
|
||||
|
||||
def _load_config() -> dict:
|
||||
f = HERE / "eval-config.json"
|
||||
if f.exists():
|
||||
try:
|
||||
return json.loads(f.read_text(encoding="utf-8"))
|
||||
except Exception: # noqa: BLE001
|
||||
return {}
|
||||
return {}
|
||||
|
||||
|
||||
def _config_get(args, field: str):
|
||||
cfg = _load_config()
|
||||
model_cfg = (cfg.get("models") or {}).get(args.model or "", {}) or {}
|
||||
return model_cfg.get(field) or cfg.get(field)
|
||||
|
||||
|
||||
def _registry_group(model: str) -> str | None:
|
||||
if not REGISTRY.exists():
|
||||
return None
|
||||
try:
|
||||
reg = json.loads(REGISTRY.read_text(encoding="utf-8"))
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
for m in reg.get("models", []):
|
||||
if m.get("id") == model:
|
||||
g = m.get("groups") or []
|
||||
return g[0] if g else None
|
||||
return None
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description="evals runner（六维评估流水线）")
|
||||
ap.add_argument("--model")
|
||||
ap.add_argument("--group", default=None)
|
||||
ap.add_argument("--transport", choices=["dry-run", "openai", "replay"], default="dry-run")
|
||||
ap.add_argument("--base-url", default=None)
|
||||
ap.add_argument("--api-key-env", default=None)
|
||||
ap.add_argument("--api-key-from-env", action="store_true",
|
||||
help="显式声明「密钥只从环境变量来」")
|
||||
ap.add_argument("--answers-dir", default=None)
|
||||
ap.add_argument("--temperature", type=float, default=None)
|
||||
ap.add_argument("--max-tokens", type=int, default=None)
|
||||
ap.add_argument("--timeout", type=int, default=600, help="单题超时秒数（协议：每维 ≤10 分钟）")
|
||||
ap.add_argument("--only", default=None, help="只跑某些维度，如 dim1,dim3")
|
||||
ap.add_argument("--questions", default=None, help="只跑某些题，如 dim1-q1,dim4-q2")
|
||||
ap.add_argument("--reuse-workdir", action="store_true", help="dim4 复用上次的副本目录")
|
||||
ap.add_argument("--workdir", default=None, help="dim4 指定副本目录（自检/复核用）")
|
||||
ap.add_argument("--list-questions", action="store_true")
|
||||
ap.add_argument("--audit", action="store_true")
|
||||
ap.add_argument("--manual", default=None, help="评委补判结果 json，合并后定稿等级")
|
||||
args = ap.parse_args()
|
||||
return run_eval(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user