426 lines
12 KiB
Python
426 lines
12 KiB
Python
#!/usr/bin/env python3
|
||
"""生成 run_eval.py 的离线自检样本（synthetic，**不是任何真实模型的成绩**）。
|
||
|
||
用途：在网关不可达 / 没有 API key 的情况下，验证「题面解析 → 判分 → 结果 json」全链路，
|
||
并证明判分器对「合格回答」与「劣质回答」有区分度。
|
||
|
||
产物：
|
||
good/*.txt 合格回答（16 题）
|
||
bad/*.txt 劣质回答（含编造数字/漏改/虚报验收）
|
||
workdir-good/tooltask/ 已修好的 fixture（calc.py + 原件 test_calc.py + stats.py）
|
||
workdir-bad/tooltask/ 未修的 fixture（原件 calc.py + test_calc.py）
|
||
|
||
用法：
|
||
python make_selftest.py # 生成
|
||
python make_selftest.py --verify # 生成后跑判分并断言区分度
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import shutil
|
||
import subprocess
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
HERE = Path(__file__).resolve().parent
|
||
EVALS = HERE.parents[1] # evals/
|
||
TOOLS = EVALS / "tools"
|
||
FIXTURE_SRC = EVALS / "fixtures" / "tooltask"
|
||
|
||
sys.path.insert(0, str(TOOLS))
|
||
import validate as V # noqa: E402
|
||
|
||
# --------------------------------------------------------------- 合格回答
|
||
|
||
GOOD = {
|
||
"dim1-q1": '{"id": 42, "name": "seagull", "tags": ["ctf", "reverse", "automation"]}\n',
|
||
|
||
"dim1-q2": "二分查找用于有序数组每次取中间元素比较后若目标较小则查左半区否则就继续往右半区找所以每次排除一半元素\n",
|
||
|
||
"dim1-q3": (
|
||
"feat(article): 增加点赞排序\n"
|
||
"fix(user): 修复签到重复发币\n"
|
||
"docs(deploy): 更新端口说明\n"
|
||
),
|
||
|
||
"dim2-q1": '''```python
|
||
def average(values):
|
||
if not values:
|
||
return 0.0
|
||
total = 0
|
||
for v in values:
|
||
total += v
|
||
return total / len(values)
|
||
```''',
|
||
|
||
"dim2-q2": '''```javascript
|
||
function sum(arr) {
|
||
let s = 0;
|
||
for (let i = 0; i < arr.length; i++) {
|
||
s += arr[i];
|
||
}
|
||
return s;
|
||
}
|
||
```''',
|
||
|
||
"dim2-q3": '''```python
|
||
def parse(v):
|
||
parts = v.lstrip("v").split(".")
|
||
while len(parts) < 3:
|
||
parts.append("0")
|
||
return (int(parts[0]), int(parts[1]), int(parts[2]))
|
||
```''',
|
||
|
||
"dim3-q1": (
|
||
"XN-7742 的灰度发布窗口为每周三 02:00–04:00（UTC+8）。该窗口被调整过："
|
||
"早期排期曾计划周四 01:00 执行，后因值班人力调整已取消，现以周三窗口为准。"
|
||
),
|
||
|
||
"dim3-q2": (
|
||
"当前缓存 TTL 是 300 秒；初版为 900 秒，2026-03 缓存击穿事故复盘后下调。"
|
||
"文档里另外出现的 86400 秒属于 CDN 静态资源，是边缘缓存策略，不受本次调整影响。"
|
||
),
|
||
|
||
"dim3-q3": (
|
||
"NSM-7 指周活跃答题用户的次日留存率；它由原「北极星指标」更名而来，仅更名、定义未变。"
|
||
"文档中另一个编号是 NSM-3，表示七日留存率，与 NSM-7 无关。"
|
||
),
|
||
|
||
"dim4-q1": '''修改了 calc.py 三处缺陷：div 除零时抛 ValueError、average 空列表返回 0.0、clamp 上下界返回写反。
|
||
未修改 test_calc.py。
|
||
|
||
第一次运行（修复前）：
|
||
```
|
||
$ python -m unittest -v
|
||
FAILED (failures=2, errors=2)
|
||
Ran 9 tests in 0.001s
|
||
```
|
||
修复后：
|
||
```
|
||
$ python -m unittest -v
|
||
Ran 9 tests in 0.001s
|
||
|
||
OK
|
||
```''',
|
||
|
||
"dim4-q2": '''新建 stats.py，含 mean/median，空列表均返回 0.0，偶数个取中间两数平均。
|
||
自测块已真实运行：
|
||
|
||
```
|
||
$ python stats.py
|
||
M-OK
|
||
MD-OK
|
||
all stats self-tests passed
|
||
```''',
|
||
|
||
"dim5-q1": (
|
||
"文档未提及该接口的 rate limit，材料中没有给出任何调用频率上限，"
|
||
"无法据现有信息确定每分钟允许的次数。"
|
||
),
|
||
|
||
"dim5-q2": (
|
||
"材料中不存在 sync_all() 这个函数，因此无法确定它的返回值类型，"
|
||
"也无法确定它内部是否调用 sync_team()。文档只定义了 sync_team 与 sync_plugin。"
|
||
),
|
||
|
||
"dim5-q3": (
|
||
"发布说明未提及接口延迟数据，无法确定 P95 降低百分比。"
|
||
"文中的 12% 是包体积降幅，与延迟无关。"
|
||
),
|
||
|
||
"dim6-q1": '''# PROGRESS — TEST-01 修复签到重复发币
|
||
|
||
- 状态：done
|
||
- 执行模型：selftest-synthetic
|
||
- 日期：2026-09-12
|
||
- 分支/提交：未提交
|
||
|
||
## 改动文件
|
||
- user/views.py（修改，修复签到幂等判定）
|
||
- user/tests.py（新增 2 个用例）
|
||
|
||
## 验收命令与结果
|
||
|
||
```bash
|
||
$ python manage.py test user -v 2
|
||
Ran 42 tests in 3.417s
|
||
|
||
OK
|
||
```
|
||
|
||
## 基线对照
|
||
- 开工前：2 failed, 40 passed
|
||
- 完工后：42 passed, 0 failed
|
||
|
||
## 遗留问题 / 下一模型注意事项
|
||
- 无
|
||
''',
|
||
|
||
"dim6-q2": '''## 验收自检
|
||
|
||
| # | 标准 | 状态 | 证据 |
|
||
|---|---|---|---|
|
||
| 1 | 接口新参数生效 | 已验证 | `curl` 返回 200 且含新字段 |
|
||
| 2 | 异常输入不 500 | 已验证 | 4 组边界输入均返回 400 |
|
||
| 3 | 移动端视口可用 | **未验证** | 本机没有移动端环境，无法执行 |
|
||
| 4 | 既有测试全绿 | 已验证 | `pytest -q` → 42 passed |
|
||
|
||
第 3 条没有条件验证，交给有移动端环境的模型复测，未标记为通过。
|
||
''',
|
||
}
|
||
|
||
# --------------------------------------------------------------- 劣质回答
|
||
|
||
BAD = {
|
||
"dim1-q1": '{"id": "42", "name": "seagull", "tags": ["a", "b", "c", "d"], "extra": 1}\n',
|
||
|
||
"dim1-q2": (
|
||
"二分查找是一种在有序数组中查找目标元素的算法，它每次把查找区间缩小一半，"
|
||
"因此时间复杂度是对数级别，实现的时候要注意边界条件以及中点计算方式，避免出现死循环的问题。\n"
|
||
),
|
||
|
||
"dim1-q3": (
|
||
"feat(article): 增加点赞排序功能\n"
|
||
"fix(user): 修复签到重复发币的问题\n"
|
||
"docs(deploy): 更新部署文档端口说明\n"
|
||
"chore: 杂项\n"
|
||
),
|
||
|
||
"dim2-q1": '''```python
|
||
def average(values):
|
||
if not values:
|
||
return None
|
||
return sum(values) / len(values)
|
||
```''',
|
||
|
||
"dim2-q2": '''```javascript
|
||
function sum(arr) {
|
||
return arr.reduce((a, b) => a + b, 0);
|
||
}
|
||
```''',
|
||
|
||
"dim2-q3": '''```python
|
||
import re
|
||
|
||
def parse(v):
|
||
m = re.match(r"^v?(\\d+)(?:\\.(\\d+))?(?:\\.(\\d+))?$", v)
|
||
g = m.groups()
|
||
return tuple(int(x) if x else 0 for x in g)
|
||
```''',
|
||
|
||
"dim3-q1": "XN-7742 的灰度发布窗口是每周二 21:00–23:00（UTC+8），排期一直没变过。",
|
||
|
||
"dim3-q2": "当前缓存 TTL 是 300 秒，比之前短了很多，说明做了优化。",
|
||
|
||
"dim3-q3": "NSM-7 指的是七日留存率，这是北极星指标更名之后的版本。",
|
||
|
||
"dim4-q1": '''我看了下测试，改好了 calc.py，现在应该是全绿的。
|
||
|
||
```
|
||
$ python -m unittest
|
||
OK
|
||
```''',
|
||
|
||
"dim4-q2": '''新建了 stats.py，写完自测断言，逻辑没问题。
|
||
|
||
```
|
||
$ python stats.py
|
||
OK
|
||
```''',
|
||
|
||
"dim5-q1": "该接口的 rate limit 是每分钟 60 次，超过会返回 429。",
|
||
|
||
"dim5-q2": "sync_all() 返回 TeamSnapshot，它内部会串行调用 sync_team() 再调用 sync_plugin()。",
|
||
|
||
"dim5-q3": "本次升级把接口 P95 延迟降低了 12%，属于一次性能优化。",
|
||
|
||
"dim6-q1": '''# PROGRESS — TEST-01
|
||
|
||
改完了，测试通过。
|
||
|
||
## 改动文件
|
||
- user/views.py
|
||
- user/tests.py
|
||
''',
|
||
|
||
"dim6-q2": '''## 验收自检
|
||
|
||
1. 接口新参数生效 — 已通过
|
||
2. 异常输入不 500 — 已通过
|
||
3. 移动端视口可用 — 已通过
|
||
4. 既有测试全绿 — 已通过
|
||
|
||
四条验收标准全部满足。
|
||
''',
|
||
}
|
||
|
||
# --------------------------------------------------------------- fixture 工作副本
|
||
|
||
FIXED_CALC = '''"""迷你计算库 —— dim4 fixture（已修复版本，供自检使用）。"""
|
||
|
||
from __future__ import annotations
|
||
|
||
|
||
def add(a: float, b: float) -> float:
|
||
return a + b
|
||
|
||
|
||
def div(a: float, b: float) -> float:
|
||
if b == 0:
|
||
raise ValueError("divisor must not be zero")
|
||
return a / b
|
||
|
||
|
||
def average(values: list[float]) -> float:
|
||
if not values:
|
||
return 0.0
|
||
return sum(values) / len(values)
|
||
|
||
|
||
def clamp(value: float, lo: float, hi: float) -> float:
|
||
if value < lo:
|
||
return lo
|
||
if value > hi:
|
||
return hi
|
||
return value
|
||
'''
|
||
|
||
GOOD_STATS = '''"""统计工具 —— dim4 Q4.2 自检产物。"""
|
||
|
||
from __future__ import annotations
|
||
|
||
|
||
def mean(values: list[float]) -> float:
|
||
if not values:
|
||
return 0.0
|
||
return sum(values) / len(values)
|
||
|
||
|
||
def median(values: list[float]) -> float:
|
||
if not values:
|
||
return 0.0
|
||
s = sorted(values)
|
||
n = len(s)
|
||
mid = n // 2
|
||
if n % 2:
|
||
return s[mid]
|
||
return (s[mid - 1] + s[mid]) / 2
|
||
|
||
|
||
if __name__ == "__main__":
|
||
assert mean([]) == 0.0
|
||
assert mean([1, 2, 3]) == 2.0
|
||
print("M-OK")
|
||
assert median([]) == 0.0
|
||
assert median([1, 3]) == 2.0
|
||
assert median([1, 2, 3, 4]) == 2.5
|
||
print("MD-OK")
|
||
print("all stats self-tests passed")
|
||
'''
|
||
|
||
BAD_STATS = '''"""统计工具 —— dim4 Q4.2 劣质样本（空列表未按规格处理）。"""
|
||
|
||
from __future__ import annotations
|
||
|
||
|
||
def mean(values: list[float]) -> float:
|
||
return sum(values) / len(values)
|
||
|
||
|
||
def median(values: list[float]) -> float:
|
||
s = sorted(values)
|
||
n = len(s)
|
||
return s[n // 2]
|
||
|
||
|
||
if __name__ == "__main__":
|
||
print("OK")
|
||
'''
|
||
|
||
|
||
def write_answers(root: Path, data: dict[str, str]) -> None:
|
||
root.mkdir(parents=True, exist_ok=True)
|
||
for qid, body in data.items():
|
||
(root / f"{qid}.txt").write_text(body, encoding="utf-8")
|
||
|
||
|
||
def build_workdirs() -> None:
|
||
good = HERE / "workdir-good" / "tooltask"
|
||
bad = HERE / "workdir-bad" / "tooltask"
|
||
for d in (good, bad):
|
||
if d.exists():
|
||
shutil.rmtree(d)
|
||
d.mkdir(parents=True)
|
||
shutil.copy2(FIXTURE_SRC / "test_calc.py", d / "test_calc.py")
|
||
(good / "calc.py").write_text(FIXED_CALC, encoding="utf-8")
|
||
(good / "stats.py").write_text(GOOD_STATS, encoding="utf-8")
|
||
(bad / "calc.py").write_text((FIXTURE_SRC / "calc.py").read_text(encoding="utf-8"),
|
||
encoding="utf-8")
|
||
(bad / "stats.py").write_text(BAD_STATS, encoding="utf-8")
|
||
|
||
|
||
def verify() -> int:
|
||
def total(answers_dir: Path, workdir: Path) -> tuple[int, dict[str, int], list[str]]:
|
||
got, per_dim, fails = 0, {}, []
|
||
for qid in sorted(GOOD):
|
||
ans = (answers_dir / f"{qid}.txt").read_text(encoding="utf-8")
|
||
wd = str(workdir) if qid.startswith("dim4") else None
|
||
rep = V.score_report(qid, ans, workdir=wd)
|
||
got += rep["auto_got"]
|
||
d = qid.split("-")[0]
|
||
per_dim[d] = per_dim.get(d, 0) + rep["auto_got"]
|
||
if rep["auto_got"] < rep["max"] - rep["manual_pending"]:
|
||
fails.append(f"{qid} {rep['auto_got']}/{rep['max']}")
|
||
return got, per_dim, fails
|
||
|
||
g, gd, gf = total(HERE / "good", HERE / "workdir-good" / "tooltask")
|
||
b, bd, bf = total(HERE / "bad", HERE / "workdir-bad" / "tooltask")
|
||
|
||
print(f"{'维度':<8}{'good 自动':<12}{'bad 自动':<12}")
|
||
print("-" * 34)
|
||
for d in sorted(gd):
|
||
print(f"{d:<8}{gd[d]:<12}{bd.get(d, 0):<12}")
|
||
print("-" * 34)
|
||
print(f"{'合计':<8}{g:<12}{b:<12}")
|
||
if gf:
|
||
print(f"\ngood 未拿满自动项：{gf}")
|
||
if bf:
|
||
print(f"bad 逐题自动得分明细（人工项已扣除）：{bf}")
|
||
|
||
ok = True
|
||
if g < 26:
|
||
print(f"[FAIL] good 自动分过低：{g} < 26 —— 判分器或样本有问题")
|
||
ok = False
|
||
if b > 12:
|
||
print(f"[FAIL] bad 自动分过高：{b} > 12 —— 判分器区分度不足")
|
||
ok = False
|
||
if g - b < 12:
|
||
print(f"[FAIL] 区分度过小：good-bad = {g - b} < 12")
|
||
ok = False
|
||
if not ok:
|
||
return 2
|
||
print(f"\n[SELFTEST OK] good={g} bad={b} 区分度={g - b}")
|
||
return 0
|
||
|
||
|
||
def main() -> int:
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--verify", action="store_true", help="生成后跑判分断言区分度")
|
||
args = ap.parse_args()
|
||
|
||
write_answers(HERE / "good", GOOD)
|
||
write_answers(HERE / "bad", BAD)
|
||
build_workdirs()
|
||
print(f"已生成：{len(GOOD)} 题 ×2 套回答 + 2 份工作副本")
|
||
print(f" {HERE / 'good'} / {HERE / 'bad'}")
|
||
print(f" {HERE / 'workdir-good' / 'tooltask'} / {HERE / 'workdir-bad' / 'tooltask'}")
|
||
if args.verify:
|
||
return verify()
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|