#!/usr/bin/env python3
"""evals runner —— 把六维题库串成一次可复现的模型评估（标准库，零依赖）。

用法：
    # 0) 题库自检：题面/满分/判分器/语料/fixture 一致性
    python run_eval.py --audit

    # 1) 只看题面清单（确认解析正确、且不含评分点）
    python run_eval.py --list-questions

    # 2) 干跑：产出将发送的 prompt 全文，不发网络请求
    python run_eval.py --model glm-5.3-flash --transport dry-run

    # 3) 真跑（需网关可达 + 环境变量里的 key）
    set DSH_API_KEY=sk-xxx
    python run_eval.py --model glm-5.3-flash --transport openai \
        --base-url http://<dsh-gateway>/v1 --api-key-env DSH_API_KEY

    # 4) 离线重放（用已录制的回答目录，answers/<qid>.txt）
    python run_eval.py --model demo --transport replay --answers-dir ../fixtures/selftest/good

    # 5) 评委补判后定稿（产出 grade）
    python run_eval.py --model glm-5.3-flash --manual results/glm-5.3-flash-2026-09-12.manual.json

设计要点：
- 只发「给模型的题面」段；**评分点段永不进入 prompt**（解析时硬校验）。
- 自动判分复用 validate.py；人工项不自动给分，登记为 pending。
- 评估协议（evals/README.md）：全新会话、逐题单发、每维 ≤10 分钟。
- 密钥只从环境变量读取，本脚本不读任何配置文件里的凭据。
"""

from __future__ import annotations

import argparse
import json
import re
import shutil
import sys
import time
import urllib.error
import urllib.request
from datetime import date
from pathlib import Path

HERE = Path(__file__).resolve().parent
EVALS = HERE.parent                     # evals/
PLANNING = EVALS.parent                 # PLANNING/
RESULTS = EVALS / "results"
FIXTURE_SRC = EVALS / "fixtures" / "tooltask"
CORPUS = EVALS / "assets" / "longctx-corpus.md"
PROTOCOL = PLANNING / "03-执行协议.md"
SCORING = EVALS / "scoring.md"
REGISTRY = PLANNING / "model-registry.json"

sys.path.insert(0, str(HERE))
import validate as V  # noqa: E402

DIM_FILES = {
    "dim1": "dim1-instruction.md",
    "dim2": "dim2-code.md",
    "dim3": "dim3-context.md",
    "dim4": "dim4-tools.md",
    "dim5": "dim5-hallucination.md",
    "dim6": "dim6-delivery.md",
}
# 维度满分（= scoring.md 的维度表，与题面文件的「（N 分）」声明交叉校验）
EXPECTED_DIM_MAX = {"dim1": 11, "dim2": 7, "dim3": 6, "dim4": 6, "dim5": 6, "dim6": 6}
# 各维题量：dim4/dim6 是 2 题（见 evals/README 目录说明与 I-04 卡的资产表），其余 3 题 → 共 16 题
EXPECTED_DIM_QTY = {"dim1": 3, "dim2": 3, "dim3": 3, "dim4": 2, "dim5": 3, "dim6": 2}
# 总分上限 = 维度表求和。注意：README/scoring.md/registry 的标题写「36 分」，
# 与维度表合计 42 不符 —— 这是规格内部的算术矛盾，登记为 SPEC-DEFECT-1，由规划者裁定；
# 本 runner 一律以题面文件为准（题面是唯一同时被评分点和判分器约束的源）。
EXPECTED_TOTAL_MAX = sum(EXPECTED_DIM_MAX.values())          # 42
SPEC_HEADLINE_TOTAL = 36                                      # scoring.md / README / registry 的标题数字
SPEC_DEFECTS = {
    "SPEC-DEFECT-1": (
        f"README §四 / evals/README §四 / scoring.md §一 称“满分 {SPEC_HEADLINE_TOTAL} 分”，"
        f"但 scoring.md 维度表各行相加 = {EXPECTED_TOTAL_MAX}，且与 16 道题面声明的分值合计一致。"
        f"等级换算用的是百分比，故不影响评级；但绝对总分口径需规划者裁定后统一。"
    ),
}
DIM_TITLES = {
    "dim1": "指令遵循", "dim2": "代码修改", "dim3": "长上下文",
    "dim4": "工具调用", "dim5": "幻觉抵抗", "dim6": "交付规范",
}
HEAD_RE = re.compile(r"^##\s*Q(\d)\.(\d)\s*[·:：]?\s*(.*)$", re.M)
POINTS_RE = re.compile(r"[（(]\s*(\d+)\s*分\s*[）)]")

API_KEY_HINT = "DSH_API_KEY"


# ------------------------------------------------------------------ 题面解析

class Question:
    def __init__(self, qid: str, dim: str, title: str, declared: int | None, prompt: str, src: Path):
        self.qid = qid
        self.dim = dim
        self.title = title.strip()
        self.declared = declared
        self.prompt = prompt.strip()
        self.src = src

    def __repr__(self) -> str:  # pragma: no cover
        return f"<Q {self.qid} {self.title!r} declared={self.declared}>"


def parse_question_file(path: Path) -> list[Question]:
    """从一份维度题面里抽出全部问题；只取「给模型的题面」围栏块。"""
    text = path.read_text(encoding="utf-8")
    marks = list(HEAD_RE.finditer(text))
    out: list[Question] = []
    for i, m in enumerate(marks):
        dim_no, q_no = m.group(1), m.group(2)
        end = marks[i + 1].start() if i + 1 < len(marks) else len(text)
        block = text[m.start():end]
        qid = f"dim{dim_no}-q{q_no}"
        prompt = _extract_prompt(block, path.name, qid)
        out.append(Question(qid, f"dim{dim_no}", m.group(3), _declared_points(m.group(3)),
                            prompt, path))
    return out


def _declared_points(title: str) -> int | None:
    m = POINTS_RE.search(title)
    return int(m.group(1)) if m else None


def _extract_prompt(block: str, fname: str, qid: str) -> str:
    seg = re.search(r"###\s*给模型的题面\s*\n(.*?)(?=\n###|\Z)", block, re.S)
    if not seg:
        raise ValueError(f"{fname} {qid}: 找不到「给模型的题面」段")
    fences = re.findall(r"```[a-zA-Z0-9_+-]*\n(.*?)```", seg.group(1), re.S)
    if not fences:
        raise ValueError(f"{fname} {qid}: 「给模型的题面」段里没有围栏块")
    prompt = fences[0].strip()
    # 硬校验：评分点绝不允许进入 prompt
    for banned in ("评分点", "勿发给被测模型", "判定方式", "自动（", "人工/评委"):
        if banned in prompt:
            raise ValueError(f"{fname} {qid}: prompt 含禁用内容 {banned!r}")
    return prompt


def load_questions() -> list[Question]:
    qs: list[Question] = []
    for dim, fname in DIM_FILES.items():
        qs.extend(parse_question_file(EVALS / "questions" / fname))
    return qs


# ------------------------------------------------------------------ 传输层

class TransportError(RuntimeError):
    pass


class DryRunTransport:
    name = "dry-run"

    def ask(self, qid, messages, model, timeout) -> tuple[str, dict]:
        return "", {"note": "dry-run：未发送请求"}


class ReplayTransport:
    """离线重放：从目录读 <qid>.txt|md 作为「模型回答」。"""
    name = "replay"

    def __init__(self, answers_dir: Path):
        self.dir = answers_dir

    def ask(self, qid, messages, model, timeout) -> tuple[str, dict]:
        for ext in (".txt", ".md"):
            f = self.dir / f"{qid}{ext}"
            if f.exists():
                return f.read_text(encoding="utf-8"), {"note": f"replay:{f.name}"}
        raise TransportError(f"replay 目录里没有 {qid}.txt|.md —— {self.dir}")


class OpenAITransport:
    """OpenAI 兼容 /chat/completions。密钥只从环境变量取，绝不落盘、绝不打印。"""
    name = "openai"

    def __init__(self, base_url: str, api_key: str, temperature: float | None = None,
                 max_tokens: int | None = None, retries: int = 2):
        self.url = base_url.rstrip("/") + "/chat/completions"
        self.key = api_key
        self.temperature = temperature
        self.max_tokens = max_tokens
        self.retries = retries
        self.usage: list[dict] = []

    def ask(self, qid, messages, model, timeout) -> tuple[str, dict]:
        body: dict = {"model": model, "messages": messages}
        if self.temperature is not None:
            body["temperature"] = self.temperature
        if self.max_tokens:
            body["max_tokens"] = self.max_tokens
        data = json.dumps(body).encode("utf-8")
        last = ""
        for attempt in range(self.retries + 1):
            req = urllib.request.Request(
                self.url, data=data, method="POST",
                headers={"Content-Type": "application/json",
                         "Authorization": f"Bearer {self.key}"},
            )
            try:
                with urllib.request.urlopen(req, timeout=timeout) as resp:
                    payload = json.loads(resp.read().decode("utf-8", "replace"))
                text = _pick_content(payload)
                meta = {"note": "openai", "usage": payload.get("usage") or {}}
                self.usage.append({"qid": qid, **meta["usage"]})
                return text, meta
            except urllib.error.HTTPError as e:
                detail = _safe_err(e)
                last = f"HTTP {e.code}: {detail}"
                if e.code in (408, 409, 429, 500, 502, 503, 504) and attempt < self.retries:
                    time.sleep(2 * (attempt + 1))
                    continue
                raise TransportError(last) from None
            except Exception as e:  # noqa: BLE001
                last = f"{type(e).__name__}: {e}"
                if attempt < self.retries:
                    time.sleep(2 * (attempt + 1))
                    continue
                raise TransportError(last) from None
        raise TransportError(last)


def _pick_content(payload: dict) -> str:
    try:
        msg = payload["choices"][0]["message"]
    except Exception:  # noqa: BLE001
        raise TransportError(f"响应结构异常：{json.dumps(payload, ensure_ascii=False)[:300]}") from None
    content = msg.get("content")
    if isinstance(content, list):  # 多段 content
        content = "".join(seg.get("text", "") for seg in content if isinstance(seg, dict))
    return (content or "").strip()


def _safe_err(e: urllib.error.HTTPError) -> str:
    try:
        raw = e.read().decode("utf-8", "replace")
    except Exception:  # noqa: BLE001
        return "(无法读取响应体)"
    raw = re.sub(r"(sk-|Bearer\s+)[A-Za-z0-9._\-]{6,}", r"\1<redacted>", raw)
    return raw[:300]


# ------------------------------------------------------------------ prompt 组装

def progress_template() -> str:
    """抽 03-执行协议.md 的 PROGRESS 章节，作为 dim6-q1 的题面附件。"""
    if not PROTOCOL.exists():
        return ""
    text = PROTOCOL.read_text(encoding="utf-8")
    blocks = [b for b in re.split(r"\n(?=## )", text) if b.startswith("## 二、PROGRESS")]
    return blocks[0].strip() if blocks else ""


def build_messages(q: Question, workdir: Path | None, corpus: str) -> list[dict]:
    prompt = q.prompt
    if q.qid in ("dim4-q1", "dim4-q2"):
        if workdir is None:
            raise TransportError("dim4 需要副本目录")
        prompt = prompt.replace("<fixtures 副本路径>", str(workdir))
        prompt = prompt.replace("<fixtures 副本路径>", str(workdir))
    if q.qid == "dim3-q1":
        pass  # 语料单独给
    if q.qid == "dim6-q1":
        tpl = progress_template()
        if tpl:
            prompt += "\n\n（以下是 PLANNING/03-执行协议.md 中规定的 PROGRESS 格式章节）\n\n" + tpl
    if q.dim == "dim3":
        return [
            {"role": "user", "content": "这是一份运维文档，请先阅读，随后我会提问。\n\n" + corpus},
            {"role": "assistant", "content": "已读完这份文档，请提问。"},
            {"role": "user", "content": prompt},
        ]
    return [{"role": "user", "content": prompt}]


# ------------------------------------------------------------------ 判分汇总

def _pct(got: int, mx: int) -> float:
    return (got / mx) if mx else 0.0


def compute_grade(total_got: int, total_max: int, dim_scores: dict, veto_flags: list[str]):
    if veto_flags:
        return "C", "硬否决：" + "；".join(veto_flags)
    pct = _pct(total_got, total_max)
    d2 = _pct(dim_scores.get("dim2", {}).get("got", 0), dim_scores.get("dim2", {}).get("max", 1))
    d4 = _pct(dim_scores.get("dim4", {}).get("got", 0), dim_scores.get("dim4", {}).get("max", 1))
    if pct >= 0.90 and d2 >= 0.90 and d4 >= 0.90:
        return "S", f"总分 {pct:.0%}，dim2 {d2:.0%} / dim4 {d4:.0%}"
    if pct >= 0.75:
        return "A", f"总分 {pct:.0%}"
    if pct >= 0.60:
        return "B", f"总分 {pct:.0%}"
    return "C", f"总分 {pct:.0%} < 60%"


def score_answers(answers: dict[str, str], questions: list[Question],
                  workdir: Path | None) -> tuple[dict, dict, list[dict]]:
    per_q: dict[str, dict] = {}
    dim_scores: dict[str, dict] = {}
    pending: list[dict] = []
    for q in questions:
        ans = answers.get(q.qid)
        if ans is None:
            continue
        rep = V.score_report(q.qid, ans, workdir=str(workdir) if q.qid.startswith("dim4") else None)
        per_q[q.qid] = rep
        d = dim_scores.setdefault(q.dim, {"got": 0, "max": 0, "pending": 0})
        d["got"] += rep["auto_got"]
        d["max"] += rep["max"]
        d["pending"] += rep["manual_pending"]
        for item in rep["manual_items"]:
            pending.append({"qid": q.qid, **item})
    return per_q, dim_scores, pending


# ------------------------------------------------------------------ 主流程

def sanitize(model: str) -> str:
    return re.sub(r"[^A-Za-z0-9._-]", "_", model)


def prepare_workdir(model: str, reuse: bool) -> Path:
    wd = RESULTS / "work" / sanitize(model) / "tooltask"
    if wd.exists() and reuse:
        return wd
    if wd.exists():
        shutil.rmtree(wd)
    wd.mkdir(parents=True)
    for f in ("calc.py", "test_calc.py"):
        src = FIXTURE_SRC / f
        if src.exists():
            shutil.copy2(src, wd / f)
    return wd


def run_eval(args) -> int:
    questions = load_questions()
    if args.only:
        # 同时接受维度（dim1）与题号（dim1-q1），避免静默选空
        sel = {s.strip() for s in args.only.split(",") if s.strip()}
        unknown = {s for s in sel if s not in DIM_FILES and not re.fullmatch(r"dim\d-q\d", s)}
        if unknown:
            print(f"--only 里有无法识别的选择器：{sorted(unknown)}")
            print("可用：" + "、".join(DIM_FILES) + "  或  dim1-q1 形式")
            return 1
        questions = [q for q in questions if q.dim in sel or q.qid in sel]
    if args.questions:
        want = {s.strip() for s in args.questions.split(",") if s.strip()}
        unknown = {s for s in want if not re.fullmatch(r"dim\d-q\d", s)}
        if unknown:
            print(f"--questions 里有无法识别的题号：{sorted(unknown)}")
            return 1
        questions = [q for q in questions if q.qid in want]
    if not questions:
        print("选中的题目为空 —— 检查 --only / --questions 的选择器")
        return 1

    if args.list_questions:
        print(f"{'QID':<10} {'满分':<4} 标题")
        print("-" * 70)
        for q in questions:
            print(f"{q.qid:<10} {str(q.declared or '?'):<4} {q.title}")
        print("-" * 70)
        print(f"共 {len(questions)} 题；题面长度合计 "
              f"{sum(len(q.prompt) for q in questions)} 字符")
        return 0

    if args.audit:
        return run_audit(questions)

    if args.manual:
        return merge_manual(Path(args.manual))

    model = args.model
    if not model:
        print("需要 --model（或 --audit / --list-questions / --manual）")
        return 1

    RESULTS.mkdir(parents=True, exist_ok=True)
    corpus = CORPUS.read_text(encoding="utf-8") if any(q.dim == "dim3" for q in questions) else ""
    if not any(q.dim == "dim4" for q in questions):
        workdir = None
    elif args.workdir:
        workdir = Path(args.workdir).resolve()
        if not (workdir / "calc.py").exists():
            print(f"--workdir 里没有 calc.py：{workdir}")
            return 1
    else:
        workdir = prepare_workdir(model, args.reuse_workdir)

    # ---- 传输层选择
    if args.transport == "dry-run":
        transport = DryRunTransport()
        if args.api_key_from_env:
            print(f"[WARN] dry-run 不需要密钥，忽略 --api-key-env {args.api_key_from_env}")
    elif args.transport == "replay":
        if not args.answers_dir:
            print("--transport replay 需要 --answers-dir")
            return 1
        transport = ReplayTransport(Path(args.answers_dir))
    else:
        base_url = args.base_url or _config_get(args, "base_url")
        if not base_url:
            print("--transport openai 需要 --base-url（或在 eval-config.json 里配 base_url）")
            return 1
        env_name = args.api_key_env or _config_get(args, "api_key_env") or API_KEY_HINT
        import os
        key = os.environ.get(env_name, "")
        if not key:
            print(f"环境变量 {env_name} 未设置 —— 密钥只从环境变量读取（本工具不读配置文件里的凭据）")
            print(f"  PowerShell:  $env:{env_name}='<your-key>'")
            print(f"  bash:        export {env_name}='<your-key>'")
            return 1
        transport = OpenAITransport(base_url, key,
                                    temperature=args.temperature,
                                    max_tokens=args.max_tokens)

    stamp = date.today().isoformat()
    raw_dir = RESULTS / "raw" / sanitize(model)
    raw_dir.mkdir(parents=True, exist_ok=True)

    print(f"模型={model}  传输={transport.name}  题目={len(questions)}  "
          f"超时={args.timeout}s  分数上限={sum(q.declared or 0 for q in questions)}")
    answers: dict[str, str] = {}
    sent: dict[str, str] = {}
    for q in questions:
        try:
            messages = build_messages(q, workdir, corpus)
        except TransportError as e:
            print(f"  [SKIP] {q.qid}: {e}")
            continue
        sent[q.qid] = messages[-1]["content"]
        if args.transport == "dry-run":
            (raw_dir / f"{q.qid}.prompt.md").write_text(
                "\n\n=== role boundary ===\n\n".join(
                    f"[{m['role']}]\n{m['content']}" for m in messages), encoding="utf-8")
            print(f"  [DRY ] {q.qid}  prompt={len(messages[-1]['content'])} 字符 → {q.qid}.prompt.md")
            continue
        t0 = time.time()
        try:
            text, meta = transport.ask(q.qid, messages, model, args.timeout)
        except TransportError as e:
            print(f"  [FAIL] {q.qid}: {e}")
            answers[q.qid] = ""
            (raw_dir / f"{q.qid}.error.txt").write_text(str(e), encoding="utf-8")
            continue
        dt = time.time() - t0
        answers[q.qid] = text
        (raw_dir / f"{q.qid}.txt").write_text(text, encoding="utf-8")
        note = meta.get("note", "")
        print(f"  [ OK ] {q.qid}  {dt:5.1f}s  {len(text):6d} 字符  {note}")

    if args.transport == "dry-run":
        print(f"\n干跑完成：prompt 已写入 {raw_dir}")
        return 0

    got_any = {k: v for k, v in answers.items() if v.strip()}
    if not got_any:
        print("\n全部题目都没有拿到回答（网关不可达 / 凭据无效 / 题目全被跳过）——"
              "不产出结果文件，避免留下 0/0 的空成绩。")
        return 1

    per_q, dim_scores, pending = score_answers(answers, questions, workdir)

    auto_got = sum(v["auto_got"] for v in per_q.values())
    total_max = sum(v["max"] for v in per_q.values())
    total_pending = sum(v["manual_pending"] for v in per_q.values())
    dim5 = dim_scores.get("dim5", {"got": 0, "max": 0})
    veto_hint: list[str] = []
    if dim5["max"] and _pct(dim5["got"], dim5["max"]) < 0.5:
        veto_hint.append(f"dim5 自动项 {dim5['got']}/{dim5['max']} <50%（待人工复核后裁定）")
    if any(v == "" for v in answers.values()):
        failed = [k for k, v in answers.items() if v == ""]
        veto_hint.append(f"存在无回答题：{failed}")

    grade, basis = (None, "待评委补判人工项")
    if total_pending == 0:
        grade, basis = compute_grade(auto_got, total_max, dim_scores, [])

    payload = {
        "model": model,
        "group": args.group or _registry_group(model),
        "date": stamp,
        "runner": "run_eval.py",
        "transport": transport.name,
        "base_url": (args.base_url or _config_get(args, "base_url") or None) if transport.name == "openai" else None,
        "scores": {d: {"got": s["got"], "max": s["max"],
                       "manual_pending": s["pending"],
                       "notes": _dim_notes(per_q, d)}
                   for d, s in sorted(dim_scores.items())},
        "per_question": {q: {"auto_got": r["auto_got"], "max": r["max"],
                             "manual_pending": r["manual_pending"], "logs": r["logs"]}
                         for q, r in sorted(per_q.items())},
        "total": {"auto_got": auto_got, "manual_pending": total_pending,
                  "max": total_max, "got": auto_got if total_pending == 0 else None},
        "hard_veto": bool(veto_hint),
        "hard_veto_hint": veto_hint,
        "grade": grade,
        "grade_basis": basis,
        "transcript": f"evals/results/{sanitize(model)}-{stamp}.log.md",
        "score_scale": {"expected_total_max": EXPECTED_TOTAL_MAX,
                        "expected_dims": EXPECTED_DIM_MAX},
    }
    out_json = RESULTS / f"{sanitize(model)}-{stamp}.json"
    out_json.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")

    _write_transcript(RESULTS / f"{sanitize(model)}-{stamp}.log.md", model, stamp, questions,
                      answers, per_q, transport, sent)
    _write_report(RESULTS / f"{sanitize(model)}-{stamp}.md", payload)

    if total_pending:
        tpl = RESULTS / f"{sanitize(model)}-{stamp}.manual.template.json"
        tpl.write_text(json.dumps({
            "model": model, "judge": "", "date": stamp,
            "items": {p["qid"]: None for p in pending},
            "criteria": [{"qid": p["qid"], "points": p["points"], "criterion": p["criterion"]}
                         for p in pending],
            "veto_flags": [],
            "notes": {},
        }, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
        print(f"\n人工项待补判 {total_pending} 分 → 模板：{tpl}")
        print(f"补判后执行：python run_eval.py --manual {tpl.name}")

    print(f"\n自动项 {auto_got}/{total_max}（待评委 {total_pending} 分）")
    for d, s in sorted(dim_scores.items()):
        flag = f"  待人工 {s['pending']}" if s["pending"] else ""
        print(f"  {d} {DIM_TITLES[d]:<6} {s['got']:>2}/{s['max']:<2}{flag}")
    print(f"\n结果：{out_json}")
    if veto_hint:
        print("[硬否决提示] " + "；".join(veto_hint))
    return 0


def _dim_notes(per_q: dict, dim: str) -> str:
    bits = []
    for qid, rep in per_q.items():
        if qid.startswith(dim + "-") and rep["auto_got"] < rep["max"]:
            failed = [ln for ln in rep["logs"] if ln.startswith("[FAIL]")]
            if failed:
                bits.append(f"{qid}: " + failed[0][7:][:60])
    return "；".join(bits)


def _write_transcript(path: Path, model: str, stamp: str, questions: list[Question],
                      answers: dict[str, str], per_q: dict, transport,
                      sent: dict[str, str]) -> None:
    lines = [f"# 评估 transcript — {model}（{stamp}）",
             "", f"- 传输：`{transport.name}`", f"- 题目数：{len(answers)}",
             "- 协议：全新会话 / 逐题单发 / 不预告下一题（见 evals/README.md）", "", "---", ""]
    for q in questions:
        if q.qid not in answers:
            continue
        lines += [f"## {q.qid} · {q.title}", "",
                  "### 题面（实际发送，末条 user 消息）", "", "```text",
                  sent.get(q.qid, ""), "```", "",
                  "### 模型回答", "", "```text", answers[q.qid], "```", "",
                  "### 自动判分", ""]
        for ln in per_q.get(q.qid, {}).get("logs", []):
            lines.append(f"- {ln}")
        lines.append("")
    path.write_text("\n".join(lines), encoding="utf-8")


def _write_report(path: Path, payload: dict) -> None:
    t = payload["total"]
    lines = [f"# 评估报告 — {payload['model']}（{payload['date']}）", "",
             f"- 传输：`{payload['transport']}`",
             f"- 自动得分：**{t['auto_got']}/{t['max']}**，待评委：**{t['manual_pending']}**",
             f"- 等级：**{payload['grade'] or '待评委补判'}**（{payload['grade_basis']}）", "",
             "| 维度 | 自动 | 满分 | 待人工 |", "|---|---|---|---|"]
    for d, s in sorted(payload["scores"].items()):
        lines.append(f"| {d} {DIM_TITLES.get(d, '')} | {s['got']} | {s['max']} | {s['manual_pending']} |")
    lines += ["", "## 逐题", "", "| 题 | 自动 | 满分 | 待人工 |", "|---|---|---|---|"]
    for q, r in sorted(payload["per_question"].items()):
        lines.append(f"| {q} | {r['auto_got']} | {r['max']} | {r['manual_pending']} |")
    if payload["hard_veto_hint"]:
        lines += ["", "## 硬否决提示", ""] + [f"- {x}" for x in payload["hard_veto_hint"]]
    path.write_text("\n".join(lines) + "\n", encoding="utf-8")


# ------------------------------------------------------------------ 评委补判

def merge_manual(manual_path: Path) -> int:
    m = json.loads(manual_path.read_text(encoding="utf-8"))
    model = m.get("model")
    if not model:
        print("manual 文件缺 model 字段")
        return 1
    # 排除评委产物（.manual.json / .manual.template.json），只留 runner 主结果
    cands = sorted(c for c in RESULTS.glob(f"{sanitize(model)}-*.json") if ".manual" not in c.name)
    if not cands:
        print(f"找不到 {model} 的评估结果 json")
        return 1
    payload = json.loads(cands[-1].read_text(encoding="utf-8"))

    items = m.get("items") or {}
    if any(v is None for v in items.values()):
        print("manual.items 里还有 null —— 补判未完成")
        return 1
    # 校验不超过各题待判分值
    cap: dict[str, int] = {}
    for qid, rep in payload["per_question"].items():
        cap[qid] = rep["manual_pending"]
    for qid, val in items.items():
        if qid not in cap:
            print(f"manual.items 有未知题号 {qid}")
            return 1
        if val > cap[qid]:
            print(f"{qid} 人工给分 {val} 超过待判上限 {cap[qid]}")
            return 1
    if sum(items.values()) < sum(cap.values()):
        missing = [q for q, c in cap.items() if items.get(q) is None]
        print(f"仍有未判题：{missing}")
        return 1

    manual_got = sum(items.values())
    for qid, rep in payload["per_question"].items():
        rep["manual_got"] = items.get(qid, 0)
    for d, s in payload["scores"].items():
        s["manual_got"] = sum(items.get(q, 0) for q in payload["scores"] and
                              [x for x in payload["per_question"] if x.startswith(d + "-")])
        s["got_total"] = s["got"] + s["manual_got"]
    veto = m.get("veto_flags") or []
    total_max = payload["total"]["max"]
    got = payload["total"]["auto_got"] + manual_got
    grade, basis = compute_grade(got, total_max, {
        d: {"got": payload["scores"][d]["got_total"], "max": payload["scores"][d]["max"]}
        for d in payload["scores"]}, veto)
    payload["total"].update({"got": got, "manual_got": manual_got, "manual_pending": 0})
    payload["grade"] = grade
    payload["grade_basis"] = basis
    payload["hard_veto"] = bool(veto)
    payload["hard_veto_flags"] = veto
    payload["judge"] = {"name": m.get("judge"), "date": m.get("date"), "notes": m.get("notes")}

    out = cands[-1]
    out.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    _write_report(out.with_suffix(".md"), payload)
    print(f"已定稿：{out}")
    print(f"总分 {got}/{total_max}（自动 {payload['total']['auto_got']} + 人工 {manual_got}）"
          f" → 等级 {grade}（{basis}）")
    print("提醒：成绩回填 PLANNING/model-registry.json 由规划者执行，本工具不自动写注册表。")
    return 0


# ------------------------------------------------------------------ 题库自检

def _parse_scoring_md() -> dict[str, tuple[int, int]]:
    """从 scoring.md 的维度表里读出 {dim: (题数, 满分)}。"""
    if not SCORING.exists():
        return {}
    out: dict[str, tuple[int, int]] = {}
    for ln in SCORING.read_text(encoding="utf-8").splitlines():
        m = re.match(r"\|\s*(dim\d)\s*[^|]*\|\s*(\d+)\s*\|\s*(\d+)\s*\|", ln)
        if m:
            out[m.group(1)] = (int(m.group(2)), int(m.group(3)))
    return out


def run_audit(questions: list[Question]) -> int:
    checks: list[tuple[str, bool, str]] = []

    def add(name: str, ok: bool, detail: str = "") -> None:
        checks.append((name, ok, detail))

    # 1 题面解析 + 声明满分
    by_dim: dict[str, list[Question]] = {}
    for q in questions:
        by_dim.setdefault(q.dim, []).append(q)
    add(f"题量合计 16", len(questions) == sum(EXPECTED_DIM_QTY.values()),
        f"实际 {len(questions)} 题")
    for dim, qty in EXPECTED_DIM_QTY.items():
        qs = by_dim.get(dim, [])
        got = sum(q.declared or 0 for q in qs)
        add(f"{dim} 题量 {qty}", len(qs) == qty, f"实际 {len(qs)} 题")
        add(f"{dim} 满分 {EXPECTED_DIM_MAX[dim]}", got == EXPECTED_DIM_MAX[dim],
            f"题面声明合计 {got}")
    total_declared = sum(q.declared or 0 for q in questions)
    add(f"总满分 {EXPECTED_TOTAL_MAX}（维度表求和）", total_declared == EXPECTED_TOTAL_MAX,
        f"题面声明合计 {total_declared}")

    # 2 与 scoring.md 对照
    sm = _parse_scoring_md()
    add("scoring.md 可解析", len(sm) == 6, f"解析到 {len(sm)} 行维度")
    for dim, (n, mx) in sm.items():
        add(f"scoring.md {dim} 满分 {mx} 与题面一致",
            mx == sum(q.declared or 0 for q in by_dim.get(dim, [])),
            f"题面 {sum(q.declared or 0 for q in by_dim.get(dim, []))}")
    # 规格已知缺陷：标题分 ≠ 维度表合计。不静默、不改规格，登记为 FAIL 提请裁定。
    add(f"[规格缺陷 SPEC-DEFECT-1] 标题分 {SPEC_HEADLINE_TOTAL} 与维度表合计 {EXPECTED_TOTAL_MAX} 不一致",
        False, SPEC_DEFECTS["SPEC-DEFECT-1"])

    # 3 判分器覆盖 + 自动/人工配平
    for q in questions:
        has = q.qid in V.CHECKERS
        rep_max = None
        if has:
            dummy = "" if q.qid != "dim4-q1" else ""
            rep = V.score_report(q.qid, dummy, workdir=None)
            rep_max = rep["max"]
        add(f"{q.qid} 有自动判分器", has, "" if has else "缺 checker")
        if has and q.declared is not None:
            pending = sum(p for p, _ in V.MANUAL_POINTS.get(q.qid, []))
            add(f"{q.qid} 自动+人工={q.declared}",
                rep_max == q.declared and rep_max == (q.declared - pending) + pending,
                f"checker.max={rep_max} 声明={q.declared} 人工登记={pending}")

    # 4 语料
    if CORPUS.exists():
        t = CORPUS.read_text(encoding="utf-8")
        add("语料 51,483 字符", len(t) == 51483, f"实际 {len(t)}")
        for name, key, lo, hi in [("针 XN-7742", "XN-7742", 0, 0.10),
                                  ("针 300 秒", "300 秒", 0.45, 0.55),
                                  ("针 NSM-7(末次)", "NSM-7", 0.88, 0.96),
                                  ("诱饵 周四01:00", "周四 01:00", 0.15, 0.25),
                                  ("诱饵 86400", "86400", 0.30, 0.40)]:
            idx = t.rfind(key) if name.endswith("(末次)") else t.find(key)
            pct = idx / len(t) if idx >= 0 else -1
            add(f"{name} 位置 {lo:.0%}–{hi:.0%}", 0 <= idx and lo <= pct <= hi,
                f"实测 {pct:.2%}" if idx >= 0 else "未找到")
    else:
        add("语料存在", False, str(CORPUS))

    # 5 fixture 基线
    import subprocess
    import tempfile
    if FIXTURE_SRC.exists():
        with tempfile.TemporaryDirectory() as td:
            for f in ("calc.py", "test_calc.py"):
                shutil.copy2(FIXTURE_SRC / f, Path(td) / f)
            p = subprocess.run([sys.executable, "-m", "unittest", "-v"], cwd=td,
                               capture_output=True, text=True, timeout=90)
            out = p.stdout + p.stderr
            m = re.search(r"Ran (\d+) tests", out)
            f_m = re.search(r"failures=(\d+)", out)
            e_m = re.search(r"errors=(\d+)", out)
            add("fixture 基线 9 用例", bool(m) and m.group(1) == "9",
                f"Ran={m.group(1) if m else '?'}")
            add("fixture 基线 failures=2, errors=2",
                bool(f_m) and bool(e_m) and f_m.group(1) == "2" and e_m.group(1) == "2",
                f"failures={f_m.group(1) if f_m else '?'} errors={e_m.group(1) if e_m else '?'}")
    else:
        add("fixture 存在", False, str(FIXTURE_SRC))

    # 6 registry 刻度一致
    if REGISTRY.exists():
        reg = json.loads(REGISTRY.read_text(encoding="utf-8"))
        scale = reg.get("scale", {})
        add("registry total_max 与规格缺陷一致（应为待裁定的 36）",
            scale.get("total_max") == SPEC_HEADLINE_TOTAL,
            f"实际 {scale.get('total_max')}；裁定后需与 scoring.md 一并改口径")
        add("registry dims 与 scoring 一致", scale.get("dims") == EXPECTED_DIM_MAX,
            f"实际 {json.dumps(scale.get('dims'), ensure_ascii=False)}")
        add("registry 覆盖 13 模型", len(reg.get("models", [])) >= 13,
            f"实际 {len(reg.get('models', []))}")
    else:
        add("model-registry.json 存在", False, str(REGISTRY))

    # 7 题面不含评分点
    for q in questions:
        add(f"{q.qid} 题面不含评分点",
            "评分点" not in q.prompt and "判定方式" not in q.prompt, "")

    hard = [c for c in checks if "SPEC-DEFECT" not in c[0] and not c[0].startswith("[规格缺陷")]
    known = [c for c in checks if c not in hard]
    ok = sum(1 for _, o, _ in hard if o)
    print(f"{'检查项':<50} 结果")
    print("-" * 90)
    for name, o, detail in checks:
        mark = "[PASS]" if o else ("[KNOWN]" if name in [k[0] for k in known] else "[FAIL]")
        print(f"{mark} {name:<50} {detail}")
    print("-" * 90)
    print(f"AUDIT: {ok}/{len(hard)} structural checks passed"
          f"（另有 {len(known)} 项已登记规格缺陷，需规划者裁定）")
    return 0 if ok == len(hard) else 2


# ------------------------------------------------------------------ 配置读取

def _load_config() -> dict:
    f = HERE / "eval-config.json"
    if f.exists():
        try:
            return json.loads(f.read_text(encoding="utf-8"))
        except Exception:  # noqa: BLE001
            return {}
    return {}


def _config_get(args, field: str):
    cfg = _load_config()
    model_cfg = (cfg.get("models") or {}).get(args.model or "", {}) or {}
    return model_cfg.get(field) or cfg.get(field)


def _registry_group(model: str) -> str | None:
    if not REGISTRY.exists():
        return None
    try:
        reg = json.loads(REGISTRY.read_text(encoding="utf-8"))
    except Exception:  # noqa: BLE001
        return None
    for m in reg.get("models", []):
        if m.get("id") == model:
            g = m.get("groups") or []
            return g[0] if g else None
    return None


def main() -> int:
    ap = argparse.ArgumentParser(description="evals runner（六维评估流水线）")
    ap.add_argument("--model")
    ap.add_argument("--group", default=None)
    ap.add_argument("--transport", choices=["dry-run", "openai", "replay"], default="dry-run")
    ap.add_argument("--base-url", default=None)
    ap.add_argument("--api-key-env", default=None)
    ap.add_argument("--api-key-from-env", action="store_true",
                    help="显式声明「密钥只从环境变量来」")
    ap.add_argument("--answers-dir", default=None)
    ap.add_argument("--temperature", type=float, default=None)
    ap.add_argument("--max-tokens", type=int, default=None)
    ap.add_argument("--timeout", type=int, default=600, help="单题超时秒数（协议：每维 ≤10 分钟）")
    ap.add_argument("--only", default=None, help="只跑某些维度，如 dim1,dim3")
    ap.add_argument("--questions", default=None, help="只跑某些题，如 dim1-q1,dim4-q2")
    ap.add_argument("--reuse-workdir", action="store_true", help="dim4 复用上次的副本目录")
    ap.add_argument("--workdir", default=None, help="dim4 指定副本目录（自检/复核用）")
    ap.add_argument("--list-questions", action="store_true")
    ap.add_argument("--audit", action="store_true")
    ap.add_argument("--manual", default=None, help="评委补判结果 json，合并后定稿等级")
    args = ap.parse_args()
    return run_eval(args)


if __name__ == "__main__":
    sys.exit(main())