Buckets:
| """Quick V2 readout look: accuracy per set (argmax, temperature-free), ECE/NLL after per-qtype temperatures fitted on calib, procedural with/without pool duplicates.""" | |
| import json, sys, collections | |
| import numpy as np | |
| sys.path.insert(0, "/workspace/decision-reasoner") | |
| from common import ece, fit_temperature, softmax | |
| R = "/workspace/decision-reasoner" | |
| ev = [json.loads(l) for l in open(f"{R}/data/eval.jsonl")]; cal = [json.loads(l) for l in open(f"{R}/data/calib.jsonl")] | |
| key = lambda x: (x.get("evidence") or "")[:300] + "||" + (x.get("criterion") or "")[:200] + "||" + "|".join(map(str, x["options"])) | |
| pool2 = collections.Counter(key(json.loads(l)) for l in open(f"{R}/runs/v2/data/train_pool.jsonl")) | |
| dup = {x["id"] for x in ev if pool2[key(x)]} | |
| runs = {"base": f"{R}/results/predictions/readout_base.jsonl", "v1_joint": f"{R}/results/predictions/readout_joint.jsonl", | |
| "v2_abl_gold": f"{R}/runs/v2/results/predictions/readout_abl_gold.jsonl", "v2_abl_mix": f"{R}/runs/v2/results/predictions/readout_abl_mix.jsonl", | |
| "v2_joint": f"{R}/runs/v2/results/predictions/readout_v2_joint.jsonl", "rl60": f"{R}/runs/v2/results/predictions/readout_rl60.jsonl", "rl40": f"{R}/runs/v2/results/predictions/readout_rl40.jsonl", "base_2b": f"{R}/runs/v2_2b/results/predictions/readout_base2b.jsonl", "v2_2b": f"{R}/runs/v2_2b/results/predictions/readout_v2b.jsonl", "v2_2b_ro": f"{R}/runs/v2_2b/results/predictions/readout_v2b_ro.jsonl"} | |
| import os | |
| out = {} | |
| for name, p in runs.items(): | |
| if not os.path.exists(p) or os.path.getsize(p) == 0: continue | |
| lp = {r["id"]: np.array(r["lp"]) for r in map(json.loads, open(p))} | |
| T = {qt: fit_temperature([lp[x["id"]] for x in cal if x["qtype"] == qt], [x["gold"] for x in cal if x["qtype"] == qt]) for qt in ("choice", "noul", "score")} | |
| row = {} | |
| for s in ("eval_proc", "eval_ts", "opendecisions", "localllama", "all", "proc_nodup"): | |
| xs = [x for x in ev if (s == "all" or x["set"] == s or (s == "proc_nodup" and x["set"] == "eval_proc" and x["id"] not in dup))] | |
| if s == "proc_nodup": xs = [x for x in xs if x["set"] == "eval_proc"] | |
| pr = [softmax(lp[x["id"]], T[x["qtype"]]) for x in xs] | |
| ok = np.array([int(np.argmax(p) == x["gold"]) for p, x in zip(pr, xs)]) | |
| row[s] = (len(xs), ok.mean(), ece(np.array([p.max() for p in pr]), ok), np.mean([-np.log(max(p[x["gold"]], 1e-12)) for p, x in zip(pr, xs)])) | |
| out[name] = row | |
| print(f"{name:12s} " + " ".join(f"{s}={r[1]*100:.1f}%" for s, r in row.items()) + f" all: ECE {row['all'][2]:.3f} NLL {row['all'][3]:.3f}", flush=True) | |
| print("eval_proc duplicates of a V2 pool item:", len(dup)) | |
Xet Storage Details
- Size:
- 2.64 kB
- Xet hash:
- ed918495a485b4dc52e0b341629b8562b88a8329a335aa5fa09d227589880915
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.