Buckets:
| """Numbers behind the eval / benchmark report figures (analysis/fig_eval_plots.py draws them). | |
| CUDA_VISIBLE_DEVICES="" .venv/bin/python analysis/fig_eval_compute.py # CPU only, project venv (numpy) | |
| Reads saved predictions only and writes results/report_figs/fig_eval_data.json. Loaders, temperatures, margins and the | |
| routing rule are the ones analysis/final_stats.py and metrics.py use; every headline number is checked against | |
| results/final_stats.json (assertions) before anything is written. | |
| Two routing rules appear in the report and both are computed here: | |
| - "metrics": metrics_v2.py / final_stats.py. Route the lowest log-margin (log p1 - log p2 of the calibrated readout); | |
| a routed chain without an answer falls back to the readout. Headline table, H4, routing_curve figure. | |
| - "table2b": the report's 2B table (NOTEBOOK 05:30). Route the lowest probability gap (p1 - p2 of the calibrated | |
| readout); the thinking answer is the parsed letter, no fallback. Gives the 2B table's 75.6% / 253 tokens and | |
| 81.9% / 742 for V2 + RL at 30%. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import os | |
| import sys | |
| sys.dont_write_bytecode = True | |
| os.environ["CUDA_VISIBLE_DEVICES"] = "" | |
| ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| sys.path.insert(0, ROOT) | |
| sys.path.insert(0, os.path.join(ROOT, "analysis")) | |
| import numpy as np # noqa: E402 | |
| from common import ece, read_jsonl, softmax # noqa: E402 | |
| from metrics import calibrated, fit_temps, margin # noqa: E402 | |
| import final_stats as fs # noqa: E402 (loaders and file table; main() is not run) | |
| V2 = f"{ROOT}/runs/v2" | |
| P2B = f"{ROOT}/runs/v2_2b/results/predictions" | |
| OUT = f"{ROOT}/results/report_figs/fig_eval_data.json" | |
| READOUT = {**fs.READOUT_FILES, "base2b": f"{P2B}/readout_base2b.jsonl", "v2b": f"{P2B}/readout_v2b.jsonl", | |
| "v2b_ro": f"{P2B}/readout_v2b_ro.jsonl"} | |
| REASON = {**fs.REASON_FILES, "base2b": f"{P2B}/reason_base2b.jsonl", "v2b": f"{P2B}/reason_v2b.jsonl", | |
| "v2b_ro": f"{P2B}/reason_v2b_ro.jsonl"} | |
| # routed pairs: name -> (readout, reasoner) | |
| PAIRS = {"base": ("base", "base"), "v1": ("v1", "v1"), "v2": ("v2", "v2"), "rl60": ("rl60", "rl60"), "rl40": ("rl40", "rl40"), | |
| "v2b": ("v2b", "v2b"), "v2b_ro": ("v2b_ro", "v2b_ro"), "hybrid2b": ("v2b_ro", "base2b"), "base2b": ("base2b", "base2b")} | |
| SETS = ["eval_proc", "eval_ts", "opendecisions", "localllama"] | |
| GRID = [round(0.01 * i, 2) for i in range(101)] | |
| def bins15(conf, ok, bins=15): | |
| b = np.minimum((conf * bins).astype(int), bins - 1) | |
| rows = [] | |
| for k in range(bins): | |
| s = b == k | |
| if s.any(): | |
| rows.append({"bin": k, "lo": k / bins, "hi": (k + 1) / bins, "n": int(s.sum()), "conf": float(conf[s].mean()), | |
| "acc": float(ok[s].mean())}) | |
| return rows | |
| def route(r_ok, t_ok, toks, score, fracs): | |
| order = np.argsort(score, kind="mergesort") | |
| n = len(r_ok) | |
| out = [] | |
| for f in fracs: | |
| m = int(round(f * n)) | |
| routed = np.zeros(n, bool) | |
| routed[order[:m]] = True | |
| out.append({"f": f, "acc": float(np.where(routed, t_ok, r_ok).mean()), "tokens": float(np.where(routed, toks, 0).mean())}) | |
| return out, order | |
| def main() -> None: | |
| ev = read_jsonl(f"{V2}/data/eval.jsonl") | |
| cal = read_jsonl(f"{V2}/data/calib.jsonl") | |
| subset = set(open(f"{V2}/data/eval_subset_ids.txt").read().split()) | |
| st = json.load(open(f"{ROOT}/results/final_stats.json")) | |
| gold = np.array([it["gold"] for it in ev]) | |
| sets = np.array([it["set"] for it in ev]) | |
| pos = {it["id"]: i for i, it in enumerate(ev)} | |
| ev_r = [it for it in ev if it["id"] in subset] | |
| sub_idx = np.array([pos[it["id"]] for it in ev_r]) | |
| assert len(ev_r) == 1498 | |
| gold_r = gold[sub_idx] | |
| out = {"n_eval": len(ev), "n_subset": len(ev_r), "set_n_eval": {s: int((sets == s).sum()) for s in SETS}, | |
| "set_n_subset": {s: int((sets[sub_idx] == s).sum()) for s in SETS}, "readout": {}, "thinking": {}, "routing": {}, | |
| "routing_table2b": {}} | |
| # ------------------------------------------------------------ fast readout | |
| R = {} | |
| for name, path in READOUT.items(): | |
| ro = fs.load_readout(path) | |
| temps = fit_temps(cal, [ro[it["id"]] for it in cal]) | |
| probs = calibrated(ev, [ro[it["id"]] for it in ev], temps) | |
| ok = np.array([int(np.argmax(p)) == g for p, g in zip(probs, gold)]) | |
| conf = np.array([p.max() for p in probs]) | |
| lmg = np.array([margin(p) for p in probs]) | |
| pgap = np.array([np.sort(p)[-1] - np.sort(p)[-2] for p in probs]) | |
| R[name] = {"ok": ok, "conf": conf, "lmg": lmg, "pgap": pgap} | |
| out["readout"][name] = {"temps": temps, "acc_all": 100 * float(ok.mean()), "acc_subset": 100 * float(ok[sub_idx].mean()), | |
| "ece15": ece(conf, ok), "acc_by_set": {s: 100 * float(ok[sets == s].mean()) for s in SETS}} | |
| if name in fs.READOUT_FILES: | |
| assert abs(out["readout"][name]["acc_all"] - st["fast_readout"][f"acc_{name}"]["point"]) < 1e-9, name | |
| assert abs(out["readout"][name]["ece15"] - st["fast_readout"][f"ece_{name}"]["point"]) < 1e-12, name | |
| out["reliability_fast_rl60"] = bins15(R["rl60"]["conf"], R["rl60"]["ok"]) | |
| # ------------------------------------------------------------ thinking mode (1,498) | |
| TH = {} | |
| for name, path in REASON.items(): | |
| rr = fs.load_reason(path) | |
| rows = [rr[it["id"]] for it in ev_r] | |
| has = np.array([r["answer"] is not None for r in rows]) | |
| t_ok = np.array([r["answer"] == g for r, g in zip(rows, gold_r)]) | |
| looped = np.array([bool(r["looped"]) for r in rows]) | |
| toks = np.array([r["tokens"] for r in rows], float) | |
| cal_rows = [(it, rr[it["id"]]) for it in cal if it["id"] in rr and rr[it["id"]]["post_lp"] is not None] | |
| pt = fit_temps([it for it, _ in cal_rows], [r["post_lp"] for _, r in cal_rows]) | |
| pconf, pok, hp = [], [], [] | |
| for i, r in enumerate(rows): | |
| if r["post_lp"] is not None: | |
| p = softmax(np.asarray(r["post_lp"]), pt[ev_r[i]["qtype"]]) | |
| pconf.append(p.max()); pok.append(int(np.argmax(p)) == gold_r[i]); hp.append(i) | |
| pconf, pok = np.array(pconf), np.array(pok) | |
| TH[name] = {"has": has, "t_ok": t_ok, "looped": looped, "toks": toks} | |
| sset = sets[sub_idx] | |
| out["thinking"][name] = {"acc": 100 * float(t_ok.mean()), "loop_rate": 100 * float(looped.mean()), "n_looped": int(looped.sum()), | |
| "no_answer": int((~has).sum()), "tokens_median": float(np.median(toks)), "tokens_mean": float(toks.mean()), | |
| "tokens_p90": float(np.percentile(toks, 90)), "post_ece15": ece(pconf, pok), "post_n": int(len(pconf)), | |
| "acc_by_set": {s: 100 * float(t_ok[sset == s].mean()) for s in SETS}} | |
| if name in fs.REASON_FILES: | |
| ts = st["thinking_subset"] | |
| assert abs(out["thinking"][name]["acc"] - ts[f"think_{name}"]["point"]) < 1e-9, name | |
| assert abs(out["thinking"][name]["loop_rate"] - ts[f"loop_{name}"]["point"]) < 1e-9, name | |
| assert abs(out["thinking"][name]["post_ece15"] - ts[f"postece_{name}"]["point"]) < 1e-12, name | |
| if name in ("v2", "rl60", "base"): | |
| out["thinking"][name]["tokens"] = [int(x) for x in toks] | |
| out["thinking"][name]["looped_flags"] = [int(x) for x in looped] | |
| if name == "rl60": | |
| out["reliability_post_rl60"] = bins15(pconf, pok) | |
| # ------------------------------------------------------------ routing | |
| for pname, (rn, tn) in PAIRS.items(): | |
| r_ok = R[rn]["ok"][sub_idx] | |
| x = TH[tn] | |
| t_route = np.where(x["has"], x["t_ok"], r_ok) | |
| curve, order = route(r_ok, t_route, x["toks"], R[rn]["lmg"][sub_idx], GRID) | |
| out["routing"][pname] = curve | |
| tb, _ = route(r_ok, x["t_ok"], x["toks"], R[rn]["pgap"][sub_idx], [0.0, 0.3, 0.5, 1.0]) | |
| out["routing_table2b"][pname] = tb | |
| if pname in fs.REASON_FILES: | |
| f3 = curve[30] | |
| assert abs(100 * f3["acc"] - st["thinking_subset"][f"routed30_{pname}"]["point"]) < 1e-9, pname | |
| assert abs(f3["tokens"] - st["thinking_subset"][f"tokens30_{pname}"]["point"]) < 1e-9, pname | |
| if pname == "rl60": # fix / break cells (readout x thinking), all and inside the routed 30% | |
| routed = np.zeros(len(r_ok), bool) | |
| routed[order[:int(round(0.3 * len(r_ok)))]] = True | |
| sset = sets[sub_idx] | |
| fb = {} | |
| for s in SETS + ["all"]: | |
| sel = np.ones(len(r_ok), bool) if s == "all" else (sset == s) | |
| c = lambda a, b, extra=True: int((sel & (r_ok == a) & (x["t_ok"] == b) & extra).sum()) | |
| fb[s] = {"n": int(sel.sum()), "routed": int((sel & routed).sum()), | |
| "fix": c(False, True), "break": c(True, False), "fix_routed": c(False, True, routed), "break_routed": c(True, False, routed)} | |
| out["fix_break_rl60"] = fb | |
| # ------------------------------------------------------------ S1MB yes/no target mass | |
| yes_t = [] | |
| for line in open(f"{ROOT}/data/s1mb/items.jsonl", encoding="utf-8"): | |
| r = json.loads(line) | |
| if r["task"] == "noul": | |
| yes_t.append(dict(zip([o["id"] for o in r["options"]], r["target"]))["true"]) | |
| out["s1mb_noul_judgments"] = len(yes_t) | |
| out["s1mb_target_yes_mass"] = float(np.mean(yes_t)) | |
| os.makedirs(os.path.dirname(OUT), exist_ok=True) | |
| json.dump(out, open(OUT, "w"), indent=1) | |
| rt = out["routing_table2b"] | |
| print("fast", {k: round(v["acc_all"], 2) for k, v in out["readout"].items()}) | |
| print("think", {k: (round(v["acc"], 2), round(v["loop_rate"], 2)) for k, v in out["thinking"].items()}) | |
| print("routed30 metrics", {k: (round(100 * v[30]["acc"], 2), round(v[30]["tokens"], 1)) for k, v in out["routing"].items()}) | |
| print("routed table2b 30/50/100", {k: [(round(100 * r["acc"], 2), round(r["tokens"], 1)) for r in v[1:]] for k, v in rt.items()}) | |
| print("fix_break", out["fix_break_rl60"]) | |
| print("s1mb yes mass", out["s1mb_noul_judgments"], round(out["s1mb_target_yes_mass"], 4)) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 10.2 kB
- Xet hash:
- e2f609dc10d631ff0be68b8f02ff9639f3e597cca03c7233c7ee74c7ca4fef9d
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.