Buckets:
| """Collect everything the dashboard shows from the run's files into one JSON blob (works on partial runs).""" | |
| from __future__ import annotations | |
| import collections | |
| import datetime as dt | |
| import glob | |
| import json | |
| import math | |
| import os | |
| import random | |
| import re | |
| import sys | |
| import numpy as np | |
| R = "/workspace/decision-reasoner" | |
| sys.path.insert(0, R) | |
| import od_score # noqa: E402 | |
| from common import auroc, ece, fit_temperature, softmax # noqa: E402 | |
| SETS = ["eval_proc", "eval_ts", "opendecisions", "localllama"] | |
| SET_LABEL = {"eval_proc": "Procedural", "eval_ts": "tasksource", "opendecisions": "OpenDecisions", "localllama": "LocalLLaMA"} | |
| def jl(path): | |
| p = os.path.join(R, path) | |
| out = [] | |
| if os.path.exists(p): | |
| with open(p, encoding="utf-8") as f: | |
| for line in f: | |
| try: | |
| out.append(json.loads(line)) | |
| except Exception: | |
| pass | |
| return out | |
| def jload(path): | |
| p = os.path.join(R, path) | |
| return json.load(open(p)) if os.path.exists(p) else None | |
| def rd(x, n=4): | |
| if x is None or (isinstance(x, float) and (math.isnan(x) or math.isinf(x))): | |
| return None | |
| return round(float(x), n) | |
| # ------------------------------------------------------------------ run state and timeline | |
| def timeline(): | |
| ev = jl("logs/timeline.jsonl") | |
| stages, open_ = [], {} | |
| for e in ev: | |
| if e["event"] == "start": | |
| open_[e["stage"]] = e["time"] | |
| elif e["event"] in ("done", "manual_stop") and e["stage"] in open_: | |
| stages.append({"stage": e["stage"], "start": open_.pop(e["stage"]), "end": e["time"], "how": e["event"], | |
| "note": e.get("note")}) | |
| for s, t in open_.items(): | |
| stages.append({"stage": s, "start": t, "end": None, "how": "running"}) | |
| stages.sort(key=lambda s: s["start"]) | |
| return ev, stages | |
| # ------------------------------------------------------------------ throughput series | |
| STAT_RE = re.compile(r"INFO (\d\d-\d\d \d\d:\d\d:\d\d) \[loggers\.py.*?Avg prompt throughput: ([\d.]+) tokens/s, " | |
| r"Avg generation throughput: ([\d.]+) tokens/s, Running: (\d+) reqs, Waiting: (\d+) reqs, " | |
| r"GPU KV cache usage: ([\d.]+)%(?:, Prefix cache hit rate: ([\d.]+)%)?") | |
| def engine_stats(): | |
| out = {} | |
| for stage in ["probe", "baseline", "generate", "eval_joint", "eval_gold", "eval_votes", "v2_generate", "v2_eval_v2_joint"]: | |
| p = os.path.join(R, f"logs/{stage}.log") | |
| if not os.path.exists(p): | |
| continue | |
| rows = [] | |
| with open(p, errors="ignore") as f: | |
| for line in f: | |
| m = STAT_RE.search(line) | |
| if m: | |
| t = "2026-" + m.group(1).replace(" ", "T") + "Z" | |
| rows.append({"t": t, "prompt": float(m.group(2)), "gen": float(m.group(3)), "running": int(m.group(4)), | |
| "waiting": int(m.group(5)), "kv": float(m.group(6)), | |
| "hit": float(m.group(7)) if m.group(7) else None}) | |
| if rows: | |
| out[stage] = rows | |
| return out | |
| def progress_series(): | |
| out = {} | |
| for name, path in [("generation", "data/gen.progress.jsonl"), ("baseline reasoning", "results/predictions/reason_base.raw.progress.jsonl"), | |
| ("joint reasoning", "results/predictions/reason_joint.raw.progress.jsonl"), | |
| ("V2 generation", "runs/v2/data/gen.progress.jsonl"), | |
| ("V2 joint reasoning", "runs/v2/results/predictions/reason_v2_joint.raw.progress.jsonl")]: | |
| rows = jl(path) | |
| if rows: | |
| out[name] = [{k: r.get(k) for k in ("time", "elapsed_min", "submitted", "finished", "in_flight", "gen_tokens", "tok_per_s", | |
| "chain_tokens_median", "chain_tokens_p90", "parse_rate", "looped")} for r in rows] | |
| return out | |
| # ------------------------------------------------------------------ training | |
| def training(): | |
| out = {} | |
| for name, base in [("joint", "results/adapters/joint"), ("gold_only", "results/adapters/gold_only"), ("votes_only", "results/adapters/votes_only"), | |
| ("v2_joint", "runs/v2/results/adapters/v2_joint"), ("v2_abl_gold", "runs/v2/results/adapters/abl_gold"), | |
| ("v2_abl_mix", "runs/v2/results/adapters/abl_mix"), ("sweep_1e-4", "runs/v2/results/adapters/sweep_1e-4"), | |
| ("sweep_2e-4", "runs/v2/results/adapters/sweep_2e-4"), ("sweep_3e-4", "runs/v2/results/adapters/sweep_3e-4"), | |
| ("v2b_joint", "runs/v2_2b/results/adapters/v2b_joint"), ("v2b_ro", "runs/v2_2b/results/adapters/v2b_ro")]: | |
| log = jl(f"{base}/train_log.jsonl") | |
| meta = jload(f"{base}/train_meta.json") | |
| if log or meta: | |
| out[name] = {"meta": meta, "steps": [{k: (rd(v, 5) if isinstance(v, float) else v) for k, v in r.items()} for r in log]} | |
| return out | |
| # ------------------------------------------------------------------ evaluation from raw predictions | |
| def temps_for(cal, lps): | |
| return {qt: fit_temperature([np.array(lps[x["id"]]) for x in cal if x["qtype"] == qt], | |
| [x["gold"] for x in cal if x["qtype"] == qt]) for qt in ("choice", "noul", "score")} | |
| def evaluation(ev, cal): | |
| tasks = od_score.read_jsonl(os.path.join(R, "data/raw/jgeuter__OpenDecisions/opendecisions.jsonl")) | |
| out = {"readout": {}, "reasoning": {}, "pairs": {}} | |
| readouts = {} | |
| for name in ("base", "gold_only", "votes_only", "joint"): | |
| rows = {r["id"]: r["lp"] for r in jl(f"results/predictions/readout_{name}.jsonl")} | |
| if not rows or any(x["id"] not in rows for x in ev): | |
| continue | |
| T = temps_for(cal, rows) | |
| probs = {x["id"]: softmax(np.array(rows[x["id"]]), T[x["qtype"]]) for x in ev} | |
| readouts[name] = probs | |
| acc = {s: rd(np.mean([int(probs[x["id"]].argmax()) == x["gold"] for x in ev if x["set"] == s])) for s in SETS} | |
| acc["all"] = rd(np.mean([int(probs[x["id"]].argmax()) == x["gold"] for x in ev])) | |
| od = [x for x in ev if x["set"] == "opendecisions"] | |
| s = od_score.score(tasks, [{"id": x["od_id"], "probs": {l: float(p) for l, p in zip(x["od_labels"], probs[x["id"]])}} for x in od]) | |
| tier = {t: rd(np.mean([int(probs[x["id"]].argmax()) == x["gold"] for x in od if x["tier"] == t])) for t in ("easy", "standard", "judge", "hard")} | |
| lvl = {str(l): rd(np.mean([int(probs[x["id"]].argmax()) == x["gold"] for x in ev if x["set"] == "eval_proc" and x["level"] == l])) for l in range(5)} | |
| out["readout"][name] = {"temps": {k: rd(v, 3) for k, v in T.items()}, "acc": acc, "od": {k: rd(s[k], 2) for k in ("capability", "intelligence", "calibration")}, | |
| "od_tier": tier, "proc_level": lvl} | |
| for name in ("base", "joint"): | |
| rr = {r["id"]: r for r in jl(f"results/predictions/reason_{name}.jsonl")} | |
| if not rr: # partial: the raw stream | |
| for r in jl(f"results/predictions/reason_{name}.raw.jsonl"): | |
| s = r["samples"][0] | |
| rr[r["id"]] = {"answer": s["answer"], "tokens": s["tokens"], "looped": s["looped"], "closed": s["closed"], | |
| "post_lp": s.get("post_lp"), "text": s["text"]} | |
| done = [x for x in ev if x["id"] in rr] | |
| if not done: | |
| continue | |
| blk = {"n_done": len(done), "n_total": len(ev), "by_set": {}} | |
| for s in SETS + ["all"]: | |
| xs = [x for x in done if s == "all" or x["set"] == s] | |
| if not xs: | |
| continue | |
| toks = [rr[x["id"]]["tokens"] for x in xs] | |
| b = {"n": len(xs), "acc": rd(np.mean([rr[x["id"]]["answer"] == x["gold"] for x in xs])), | |
| "tok_median": int(np.median(toks)), "tok_p90": int(np.percentile(toks, 90)), | |
| "loops": int(sum(rr[x["id"]]["looped"] for x in xs)), | |
| "loops_per_1k": rd(1000 * np.mean([rr[x["id"]]["looped"] for x in xs]), 2)} | |
| if name in readouts: | |
| pr = readouts[name] | |
| has = [x for x in xs if rr[x["id"]]["answer"] is not None] | |
| agree = np.array([int(pr[x["id"]].argmax()) == rr[x["id"]]["answer"] for x in has]) | |
| b["agree"] = rd(agree.mean()) if len(has) else None | |
| b["dis_auroc"] = rd(auroc(np.array([1 - pr[x["id"]].max() for x in has]), ~agree)) if len(has) > 10 else None | |
| blk["by_set"][s] = b | |
| lv = {} | |
| for l in range(5): | |
| xs = [x for x in done if x["set"] == "eval_proc" and x["level"] == l] | |
| if xs: | |
| lv[str(l)] = {"acc": rd(np.mean([rr[x["id"]]["answer"] == x["gold"] for x in xs])), "tok_median": int(np.median([rr[x["id"]]["tokens"] for x in xs]))} | |
| blk["proc_level"] = lv | |
| tiers = {} | |
| for t in ("easy", "standard", "judge", "hard"): | |
| xs = [x for x in done if x["set"] == "opendecisions" and x["tier"] == t] | |
| if xs: | |
| tiers[t] = {"acc": rd(np.mean([rr[x["id"]]["answer"] == x["gold"] for x in xs])), "tok_median": int(np.median([rr[x["id"]]["tokens"] for x in xs]))} | |
| blk["od_tier"] = tiers | |
| blk["tokens_all"] = [int(rr[x["id"]]["tokens"]) for x in done] | |
| out["reasoning"][name] = blk | |
| out["_rr_" + name] = rr | |
| # same-item comparison while the joint eval runs | |
| if "_rr_base" in out and "_rr_joint" in out: | |
| b, j = out["_rr_base"], out["_rr_joint"] | |
| common = [x for x in ev if x["id"] in j] | |
| pairs = {} | |
| for s in SETS + ["all"]: | |
| xs = [x for x in common if s == "all" or x["set"] == s] | |
| if not xs: | |
| continue | |
| pairs[s] = {"n": len(xs), | |
| "reason_base": rd(np.mean([b[x["id"]]["answer"] == x["gold"] for x in xs])), | |
| "reason_joint": rd(np.mean([j[x["id"]]["answer"] == x["gold"] for x in xs])), | |
| "tok_base": int(np.median([b[x["id"]]["tokens"] for x in xs])), | |
| "tok_joint": int(np.median([j[x["id"]]["tokens"] for x in xs])), | |
| "loops_base": int(sum(b[x["id"]]["looped"] for x in xs)), | |
| "loops_joint": int(sum(j[x["id"]]["looped"] for x in xs))} | |
| for nm, rr in (("base", b), ("joint", j)): | |
| if nm in readouts: | |
| has = [x for x in xs if rr[x["id"]]["answer"] is not None] | |
| pairs[s][f"agree_{nm}"] = rd(np.mean([int(readouts[nm][x["id"]].argmax()) == rr[x["id"]]["answer"] for x in has])) | |
| out["pairs"] = pairs | |
| return out, readouts | |
| # ------------------------------------------------------------------ samples | |
| def clip(t, n): | |
| return t if len(t) <= n else t[: n // 2] + f"\n\n[... {len(t) - n:,} characters not shown ...]\n\n" + t[-n // 2:] | |
| def item_view(x): | |
| return {"id": x["id"], "set": SET_LABEL.get(x["set"], x["set"]), "family": x["family"], "level": x.get("level"), "tier": x.get("tier"), | |
| "qtype": x["qtype"], "criterion": x["criterion"], "evidence": clip(x["evidence"], 2400), | |
| "options": [{"letter": chr(65 + i), "text": (o["id"] + (": " + o["desc"] if o.get("desc") else "")) if o.get("id") else (o.get("desc") or "")} | |
| for i, o in enumerate(x["options"])], "gold": x["gold"], "target": [rd(v, 3) for v in x["target"]]} | |
| def samples(ev, evo, readouts): | |
| rr_b, rr_j = evo.get("_rr_base", {}), evo.get("_rr_joint", {}) | |
| by_id = {x["id"]: x for x in ev} | |
| picks = [] | |
| def add(kind, title, why, x, extra=None): | |
| v = item_view(x) | |
| v.update({"kind": kind, "title": title, "why": why}) | |
| for nm in ("base", "joint"): | |
| if nm in readouts: | |
| v[f"readout_{nm}"] = [rd(p, 3) for p in readouts[nm][x["id"]]] | |
| rr = rr_b if nm == "base" else rr_j | |
| if x["id"] in rr: | |
| r = rr[x["id"]] | |
| v[f"reason_{nm}"] = {"answer": r["answer"], "tokens": r["tokens"], "looped": r["looped"], "text": clip(r["text"], 9000)} | |
| if extra: | |
| v.update(extra) | |
| picks.append(v) | |
| rng = random.Random(7) | |
| # 1. readout confidently wrong, reasoning right (base), from procedural level 4 | |
| cand = [x for x in ev if x["set"] == "eval_proc" and x.get("level") == 4 and x["id"] in rr_b and "base" in readouts | |
| and int(readouts["base"][x["id"]].argmax()) != x["gold"] and rr_b[x["id"]]["answer"] == x["gold"] and readouts["base"][x["id"]].max() > 0.55] | |
| if cand: | |
| add("readout_vs_reasoning", "The fast readout is confidently wrong; reasoning works it out", | |
| "Base model, procedural level 4. The readout's top option has more than 55% probability and is wrong; one thinking sample gets it right.", rng.choice(cand)) | |
| # 2. base looped, joint closed | |
| cand = [x for x in ev if x["id"] in rr_b and x["id"] in rr_j and rr_b[x["id"]]["looped"] and rr_j[x["id"]]["closed"] and rr_j[x["id"]]["answer"] == x["gold"]] | |
| if cand: | |
| add("loop_fixed", "Base loops to the context limit; the trained reasoner answers", "Same item, same prompt. Base ran to ~32k tokens repeating itself; joint closed its chain and was correct.", cand[0]) | |
| # 3. joint readout fixed what base readout got wrong, agreeing with reasoning | |
| cand = [x for x in ev if x["id"] in rr_j and "joint" in readouts and "base" in readouts and x["set"] == "eval_proc" | |
| and int(readouts["base"][x["id"]].argmax()) != x["gold"] and int(readouts["joint"][x["id"]].argmax()) == x["gold"] == rr_j[x["id"]]["answer"]] | |
| if cand: | |
| add("readout_fixed", "Training moved the fast answer to where reasoning lands", | |
| "Base readout wrong; after joint training the one-pass readout picks the option the reasoning chain concludes, and it is correct.", rng.choice(cand)) | |
| # 4. tasksource label convention: reasoning wrong vs gold, readout right | |
| cand = [x for x in ev if x["set"] == "eval_ts" and x["id"] in rr_b and "base" in readouts and rr_b[x["id"]]["answer"] not in (None, x["gold"]) | |
| and int(readouts["base"][x["id"]].argmax()) == x["gold"]] | |
| if cand: | |
| add("convention", "Reasoning argues against the dataset's labelling convention", | |
| "Natural-language item where the readout matches gold and the reasoning does not: typical of sentiment, humour and word-sense tasks.", rng.choice(cand)) | |
| # 5. LocalLLaMA split soft gold | |
| cand = [x for x in ev if x["set"] == "localllama" and x["id"] in rr_b and rr_b[x["id"]]["answer"] not in (None, x["gold"]) and sorted(x["target"])[-1] < 0.5] | |
| if cand: | |
| add("soft_gold", "LocalLLaMA: the gold label is a split teacher vote", "Gold is the mean of three ~4B teacher samples; here no option has half the probability.", rng.choice(cand)) | |
| # 6. a training chain (shortest correct) | |
| pool = {x["id"]: x for x in jl("data/train_pool.jsonl")} | |
| gen = jl("data/gen.jsonl") | |
| tr = [] | |
| for r in gen: | |
| it = pool.get(r["id"]) | |
| if not it: | |
| continue | |
| ok = [s for s in r["samples"] if s["closed"] and s["answer"] is not None and r["order"][s["answer"]] == it["gold"]] | |
| if ok and it["set"] == "train_proc" and 500 < min(s["tokens"] for s in ok) < 1200: | |
| tr.append((it, r, ok)) | |
| if tr: | |
| it, r, ok = tr[len(tr) // 3] | |
| best = min(ok, key=lambda s: s["tokens"]) | |
| votes = collections.Counter(r["order"][s["answer"]] for s in r["samples"] if s["answer"] is not None) | |
| v = item_view(it) | |
| v.update({"kind": "training", "title": "What the reasoner is trained on: its own shortest correct chain", | |
| "why": f"4 sampled chains ({', '.join(str(s['tokens']) for s in r['samples'])} tokens); the shortest correct one is the training target. Votes over options: " | |
| + ", ".join(f"{chr(65 + r['order'].index(k))}={n}" for k, n in votes.items()) + ".", | |
| "train_chain": {"tokens": best["tokens"], "text": clip(best["text"], 9000)}, "train_order": r["order"]}) | |
| picks.append(v) | |
| # browsable sample: stratified, items with both reasonings when available | |
| browse = [] | |
| for s in SETS: | |
| xs = [x for x in ev if x["set"] == s and x["id"] in rr_b] | |
| rng.shuffle(xs) | |
| xs.sort(key=lambda x: x["id"] not in rr_j) | |
| for x in xs[:30]: | |
| v = item_view(x) | |
| v["evidence"] = clip(x["evidence"], 1500) | |
| for nm in ("base", "joint"): | |
| if nm in readouts: | |
| v[f"readout_{nm}"] = [rd(p, 3) for p in readouts[nm][x["id"]]] | |
| rr = rr_b if nm == "base" else rr_j | |
| if x["id"] in rr: | |
| r = rr[x["id"]] | |
| v[f"reason_{nm}"] = {"answer": r["answer"], "tokens": r["tokens"], "looped": r["looped"], "text": clip(r["text"], 3500)} | |
| browse.append(v) | |
| return picks, browse | |
| # ------------------------------------------------------------------ generation data | |
| def generation_view(): | |
| pool = {x["id"]: x for x in jl("data/train_pool.jsonl")} | |
| gen = jl("data/gen.jsonl") | |
| fam = collections.defaultdict(lambda: [0, 0.0]) | |
| share = collections.defaultdict(collections.Counter) | |
| toks, short = [], [] | |
| for r in gen: | |
| it = pool.get(r["id"]) | |
| if not it: | |
| continue | |
| ans = [r["order"][s["answer"]] for s in r["samples"] if s["answer"] is not None and s["closed"]] | |
| if it["set"] == "train_proc": | |
| f = fam[it["family"]] | |
| f[0] += 1 | |
| f[1] += sum(a == it["gold"] for a in ans) / max(len(r["samples"]), 1) | |
| if len(ans) >= 2: | |
| top = collections.Counter(ans).most_common(1)[0] | |
| key = "unanimous, = gold" if top[1] == len(ans) and top[0] == it["gold"] else ( | |
| "unanimous, against gold" if top[1] == len(ans) else "split votes") | |
| share[it["set"]][key] += 1 | |
| else: | |
| share[it["set"]]["fewer than 2 parsed"] += 1 | |
| for s in r["samples"]: | |
| toks.append(s["tokens"]) | |
| ok = [s["tokens"] for s in r["samples"] if s["closed"] and s["answer"] is not None and r["order"][s["answer"]] == it["gold"]] | |
| if ok: | |
| short.append(min(ok)) | |
| return {"items": len(gen), "family_acc": {k: rd(v[1] / v[0], 3) for k, v in fam.items() if v[0]}, | |
| "family_n": {k: v[0] for k, v in fam.items()}, | |
| "vote_share": {k: dict(v) for k, v in share.items()}, "sample_tokens": toks, "shortest_correct_tokens": short} | |
| # ------------------------------------------------------------------ text logs | |
| def text_tail(path, n=60, drop=("WARNING", "Capturing CUDA", "it/s]")): | |
| p = os.path.join(R, path) | |
| if not os.path.exists(p): | |
| return [] | |
| with open(p, errors="ignore") as f: | |
| lines = [l.rstrip("\n") for l in f.read().replace("\r", "\n").split("\n")] | |
| lines = [l for l in lines if l.strip() and not any(d in l for d in drop)] | |
| return [l[:400] for l in lines[-n:]] | |
| def train_v2_log(): | |
| """What the V2 joint log says beyond train_log.jsonl: the data line, the timing probe, out-of-memory redos.""" | |
| p = os.path.join(R, "logs/v2_train_v2_joint.log") | |
| if not os.path.exists(p): | |
| return None | |
| out = {"oom_retries": 0} | |
| for l in open(p, errors="replace"): | |
| if l.startswith('{"oom_retry"'): | |
| out["oom_retries"] += 1 | |
| elif l.startswith('{"examples"'): | |
| out["data"], out["oom_retries"] = json.loads(l), 0 # a restarted run starts its count again | |
| elif l.startswith('{"probe_tok_per_s"'): | |
| out["probe"] = json.loads(l) | |
| return out | |
| def v2_stages(): | |
| """V2 stages from run_v2.sh's timeline, plus the two generation runs reconstructed from their per-minute progress.""" | |
| out = [] | |
| day = "2026-10-31" | |
| for name, path, prefix in [("v2_generate", "logs/v2_generate.log", "progress "), ("v2_teacher", "logs/v2_teacher.log", "progress ")]: | |
| p = os.path.join(R, path) | |
| if not os.path.exists(p): | |
| continue | |
| rows, done = [], False | |
| with open(p, errors="ignore") as f: | |
| for line in f: | |
| if line.startswith(prefix): | |
| try: | |
| rows.append(json.loads(line[len(prefix):])) | |
| except json.JSONDecodeError: | |
| pass | |
| elif line.startswith("EXIT") or line.startswith("{\"items\""): | |
| done = True | |
| if rows: | |
| t_last = dt.datetime.fromisoformat(f"{day}T{rows[-1]['time']}+00:00") | |
| t0 = t_last - dt.timedelta(minutes=rows[-1]["elapsed_min"]) | |
| fmt = lambda t: t.strftime("%Y-%m-%dT%H:%M:%SZ") | |
| out.append({"stage": name, "start": fmt(t0), "end": fmt(t_last) if done else None, "how": "done" if done else "running"}) | |
| ev = jl("logs/v2_timeline.jsonl") | |
| open_ = {} | |
| for e in ev: | |
| if e["event"] == "start": | |
| open_[e["stage"]] = e["time"] | |
| elif e["event"] == "done" and e["stage"] in open_: | |
| out.append({"stage": e["stage"], "start": open_.pop(e["stage"]), "end": e["time"], "how": "done"}) | |
| for st, t in open_.items(): | |
| out.append({"stage": st, "start": t, "end": None, "how": "running"}) | |
| out.sort(key=lambda x: x["start"]) | |
| return out, [e for e in ev if e["event"] in ("chosen", "failed", "finished")] | |
| def v2_sources(): | |
| """Local 4B chains vs the API teacher's, per question type: correct-of-4 spread, chain length, closure.""" | |
| pool = {x["id"]: x for x in jl("runs/v2/data/train_pool.jsonl")} | |
| v1 = {r["id"] for r in jl("runs/v2/data/gen_v1.jsonl")} | |
| recs = [(r, "local") for r in jl("runs/v2/data/gen.jsonl") if r.get("source") != "teacher"] + \ | |
| [(r, "teacher") for r in jl("runs/v2/data/gen_teacher.jsonl")] | |
| agg = {} | |
| for r, src in recs: | |
| it = pool.get(r["id"]) | |
| if it is None: | |
| continue | |
| key = f"{src}:{r['id'].split(':')[0]}" + (":v1" if r["id"] in v1 else "") | |
| a = agg.setdefault(key, {"items": 0, "c": [0] * 5, "toks": [], "closed": 0, "n": 0}) | |
| c = sum(1 for x in r["samples"] if x["closed"] and x["answer"] is not None and r["order"][x["answer"]] == it["gold"]) | |
| a["items"] += 1 | |
| a["c"][min(c, 4)] += 1 | |
| for x in r["samples"]: | |
| a["toks"].append(x["tokens"]) | |
| a["closed"] += x["closed"] | |
| a["n"] += 1 | |
| out = {} | |
| for k, a in sorted(agg.items()): | |
| t = sorted(a["toks"]) | |
| out[k] = {"items": a["items"], "correct_of_4": [c / a["items"] for c in a["c"]], "closed": a["closed"] / max(a["n"], 1), | |
| "tokens_median": t[len(t) // 2] if t else None, "tokens_p90": t[int(0.9 * (len(t) - 1))] if t else None} | |
| return out | |
| def teacher_progress(): | |
| rows = [] | |
| p = os.path.join(R, "logs/v2_teacher.log") | |
| if os.path.exists(p): | |
| for line in open(p, errors="ignore"): | |
| if line.startswith("progress "): | |
| try: | |
| r = json.loads(line[9:]) | |
| rows.append({k: r.get(k) for k in ("time", "elapsed_min", "cost", "items_done", "samples", "correct", "out_tok_per_s")}) | |
| except json.JSONDecodeError: | |
| pass | |
| return rows | |
| def rl_progress(): | |
| """Per optimizer step from the RL logs (TRL prints one dict per step; rewards on generation steps only). Phase a: | |
| logs/rl_grpo.log at lr 1e-5, stopped at step 40; phase b: logs/rl_grpo_b.log at 2.5e-5 from its step-40 adapter, | |
| numbered on from 41.""" | |
| import ast | |
| phases = [("logs/rl_grpo.log", "runs/v2/checkpoints/rl_grpo/completions", 0, 40), | |
| ("logs/rl_grpo_b.log", "runs/v2/checkpoints/rl_grpo_b/completions", 40, None)] | |
| steps, rounds, err, done = [], [], None, False | |
| for log, comp, offset, keep in phases: | |
| p = os.path.join(R, log) | |
| if not os.path.exists(p): | |
| continue | |
| text = open(p, errors="replace").read().replace("\r", "\n") | |
| n = 0 | |
| for m in re.finditer(r"\{'loss'[^{}]*\}", text): | |
| try: | |
| d = ast.literal_eval(m.group(0)) | |
| except Exception: | |
| continue | |
| n += 1 | |
| if keep and n > keep: | |
| break | |
| row = {"step": offset + n, "phase": "b" if offset else "a"} | |
| for k, v in d.items(): | |
| try: | |
| row[k.replace("completions/", "c_").replace("rewards/reward/", "r_")] = float(v) | |
| except (TypeError, ValueError): | |
| pass | |
| steps.append(row) | |
| for l in text.splitlines(): | |
| if "Traceback" in l or "OutOfMemoryError" in l: | |
| err = l[:300] | |
| done = "RL_EXIT 0" in text if offset else done | |
| try: # one file per optimizer step; the 4 steps of a round share their chains | |
| import pandas as pd | |
| for f in sorted(glob.glob(os.path.join(R, comp, "completions_*.parquet"))): | |
| df = pd.read_parquet(f, columns=["step", "completion", "reward"]) | |
| st = int(df["step"].iloc[0]) | |
| if (st - 1) % 4 or (keep and st > keep): | |
| continue | |
| fin = df[df["reward"] > -1.4] | |
| rounds.append({"step": offset + st, "fin_chars_median": float(fin["completion"].str.len().median()) if len(fin) else None, | |
| "waits_per_fin": float(fin["completion"].str.count(r"\bWait\b").mean()) if len(fin) else None, | |
| "right_among_fin": float((fin["reward"] > 0).mean()) if len(fin) else None}) | |
| except Exception as e: # the dashboard still builds without the chain files | |
| rounds.append({"error": repr(e)[:200]}) | |
| if not steps and not os.path.exists(os.path.join(R, "logs/rl_grpo.log")): | |
| return None | |
| ev = jl("logs/v2_timeline.jsonl") | |
| started = ([e["time"] for e in ev if e.get("stage") in ("rl_grpo", "rl_grpo_b") and e["event"] in ("start", "restart")] or [None])[-1] | |
| return {"steps": steps, "rounds": rounds, "planned": 60, "started": started, "exit": done, "error": err, | |
| "phase_b_from": 41, "lr": {"a": 1e-5, "b": 2.5e-5}} | |
| def reasoning_compare(ev): | |
| """Thinking mode, one sample per item, on the items every listed reasoner answered (the 1,498-item V2 subset).""" | |
| files = [("base", "results/predictions/reason_base.jsonl"), ("v1_joint", "results/predictions/reason_joint.jsonl"), | |
| ("v2_joint", "runs/v2/results/predictions/reason_v2_joint.jsonl"), ("rl60", "runs/v2/results/predictions/reason_rl60.jsonl"), | |
| ("rl40", "runs/v2/results/predictions/reason_rl40.jsonl"), ("base_2b", "runs/v2_2b/results/predictions/reason_base2b.jsonl"), | |
| ("v2_2b", "runs/v2_2b/results/predictions/reason_v2b.jsonl"), ("v2_2b_ro", "runs/v2_2b/results/predictions/reason_v2b_ro.jsonl")] | |
| evd = {x["id"]: x for x in ev} | |
| runs = {n: {r["id"]: r for r in jl(f)} for n, f in files} | |
| runs = {n: r for n, r in runs.items() if r} | |
| if len(runs) < 2: | |
| return None | |
| ids = [i for i in evd if all(i in r for r in runs.values())] | |
| out = {"n": len(ids), "rows": {}} | |
| for n, r in runs.items(): | |
| row = {} | |
| for st in SETS + ["all"]: | |
| xs = [i for i in ids if st == "all" or evd[i]["set"] == st] | |
| if not xs: | |
| continue | |
| row[st] = {"acc": rd(np.mean([r[i]["answer"] is not None and r[i]["answer"] == evd[i]["gold"] for i in xs])), | |
| "looped": rd(np.mean([bool(r[i]["looped"]) for i in xs])), | |
| "tok_median": float(np.median([r[i]["tokens"] for i in xs])), "tok_mean": float(np.mean([r[i]["tokens"] for i in xs]))} | |
| out["rows"][n] = row | |
| return out | |
| def s1mb_view(): | |
| """S1MB table for the page: the top 25, every 0.8B-4B published model, and ours.""" | |
| d = jload("results/s1mb.json") | |
| if not d: | |
| return None | |
| import re as _re | |
| keep = [] | |
| for i, r in enumerate(d["leaderboard"]): | |
| small = _re.search(r"(?i)(^|[-_])(0\.8b|2b|3b|4b|e2b|e4b)([-_.]|$)", r["model"].split("__")[-1]) | |
| if i < 25 or r["model"].startswith("OURS") or small: | |
| keep.append({"rank": i + 1, **r}) | |
| return {"cohort": d["cohort"], "benchmarks": d["benchmarks"], "rescore_check": d["rescore_check"], "rows": keep, | |
| "ours": {k: {m: {t: v[m].get(t) for t in ("noul", "choice", "score", "task_avg")} for m in ("raw", "calibrated") if v.get(m)} for k, v in d["ours"].items()}} | |
| def twob(): | |
| """The ~2B run: candidate probes (with the 4B's probe for reference), the pick, the sweep, training and eval files.""" | |
| probes = {} | |
| for f in sorted(glob.glob(os.path.join(R, "runs/v2_2b/results/probe/*.json"))) + [os.path.join(R, "results/probe/Qwen3.5-4B.json")]: | |
| r = jload(os.path.relpath(f, R)) | |
| if not r or "readout_acc" not in r: | |
| continue | |
| pa = r["pool_acc"]; n = sum(v["n"] for v in pa.values()) | |
| probes[r["model"]] = {"readout": r["readout_acc"], "readout_mean": rd(sum(r["readout_acc"].values()) / len(r["readout_acc"])), | |
| "think1": rd(sum(v["think1"] * v["n"] for v in pa.values()) / n), "maj4": rd(sum(v["maj4"] * v["n"] for v in pa.values()) / n), | |
| "parse": r["think"]["parse_rate"], "looped": r["think"]["looped"], "samples": r["think"]["samples"], | |
| "tokens_median": r["think"]["tokens_median"], "od": r.get("od_raw")} | |
| return {"probes": probes, "pick": jload("runs/v2_2b/results/pick_2b.json"), "sweep": jload("runs/v2_2b/results/lr_sweep.json"), | |
| "metrics": jload("runs/v2_2b/results/metrics_2b.json"), | |
| "train": jload("runs/v2_2b/results/adapters/v2b_joint/train_meta.json")} | |
| def v2_readout_first_look(ev, cal): | |
| """Readout accuracy per eval set as soon as each V2 prediction file exists (metrics_v2.py adds intervals later). | |
| Accuracy is the argmax (temperature-free); ECE and NLL use per-type temperatures fitted on the calibration items. | |
| proc_nodup leaves out the eval_proc items that exactly repeat a V2 training-pool item.""" | |
| key = lambda x: (x.get("evidence") or "")[:300] + "||" + (x.get("criterion") or "")[:200] + "||" + "|".join(map(str, x["options"])) | |
| pool = collections.Counter(key(x) for x in jl("runs/v2/data/train_pool.jsonl")) | |
| dup = {x["id"] for x in ev if pool[key(x)]} | |
| files = [("base", "results/predictions/readout_base.jsonl"), ("v1_joint", "results/predictions/readout_joint.jsonl"), | |
| ("v2_abl_gold", "runs/v2/results/predictions/readout_abl_gold.jsonl"), ("v2_abl_mix", "runs/v2/results/predictions/readout_abl_mix.jsonl"), | |
| ("v2_joint", "runs/v2/results/predictions/readout_v2_joint.jsonl"), ("rl60", "runs/v2/results/predictions/readout_rl60.jsonl"), | |
| ("rl40", "runs/v2/results/predictions/readout_rl40.jsonl"), ("base_2b", "runs/v2_2b/results/predictions/readout_base2b.jsonl"), | |
| ("v2_2b", "runs/v2_2b/results/predictions/readout_v2b.jsonl"), ("v2_2b_ro", "runs/v2_2b/results/predictions/readout_v2b_ro.jsonl")] | |
| out = {"n_dup": len(dup), "rows": {}} | |
| for name, path in files: | |
| lps = {r["id"]: np.array(r["lp"]) for r in jl(path)} | |
| if not lps or any(x["id"] not in lps for x in ev + cal): | |
| continue | |
| T = temps_for(cal, lps) | |
| pr = {x["id"]: softmax(lps[x["id"]], T[x["qtype"]]) for x in ev} | |
| row = {} | |
| for s, xs in [(s, [x for x in ev if x["set"] == s]) for s in SETS] + [("proc_nodup", [x for x in ev if x["set"] == "eval_proc" and x["id"] not in dup]), ("all", ev)]: | |
| ok = np.array([int(pr[x["id"]].argmax() == x["gold"]) for x in xs]) | |
| row[s] = {"n": len(xs), "acc": rd(ok.mean()), "ece": rd(ece(np.array([pr[x["id"]].max() for x in xs]), ok)), | |
| "nll": rd(float(np.mean([-np.log(max(pr[x["id"]][x["gold"]], 1e-12)) for x in xs])))} | |
| out["rows"][name] = row | |
| return out | |
| def main(out_path): | |
| ev = jl("data/eval.jsonl") | |
| cal = jl("data/calib.jsonl") | |
| events, stages = timeline() | |
| v2s, v2d = v2_stages() | |
| evo, readouts = evaluation(ev, cal) | |
| picks, browse = samples(ev, evo, readouts) | |
| for k in list(evo): | |
| if k.startswith("_rr_"): | |
| del evo[k] | |
| data = { | |
| "generated_at": dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), | |
| "run_start": events[0]["time"] if events else None, | |
| "events": events, "stages": stages, | |
| "summary": jload("data/summary.json"), "probe": jload("results/probe.json"), "generation": jload("results/generation.json"), | |
| "probe_models": {m: jload(f"results/probe/{m}.json") for m in ("Qwen3-4B", "Qwen3.5-4B")}, | |
| "engine": engine_stats(), "progress": progress_series(), "training": training(), | |
| "eval": evo, "metrics": jload("results/metrics.json"), "metrics_base": jload("results/metrics_baseline_only.json"), | |
| "latency": {n: jload(f"results/predictions/latency_{n}.json") for n in ("base", "joint")}, | |
| "gen_view": generation_view(), "samples": picks, "browse": browse, | |
| "deviations": open(os.path.join(R, "results/DEVIATIONS.md")).read(), | |
| "notebook": open(os.path.join(R, "results/NOTEBOOK.md")).read(), | |
| "todo": open(os.path.join(R, "TODO.md")).read(), | |
| "v2": {"stages": v2s, "decisions": v2d, "summary": jload("runs/v2/data/summary_v2.json"), "lr_sweep": jload("runs/v2/results/lr_sweep.json"), | |
| "metrics": jload("runs/v2/results/metrics_v2.json"), "sources": v2_sources(), "teacher": jload("runs/v2/results/teacher_generation.json"), | |
| "teacher_progress": teacher_progress(), "bench": jload("runs/v2/results/speed_bench.json"), | |
| "train_log": train_v2_log(), "first_look": v2_readout_first_look(ev, cal), "rl": rl_progress(), "upcoming": jload("runs/v2/results/upcoming.json"), "reason_cmp": reasoning_compare(ev), "jevals": jload("results/jevals.json"), "twob": twob(), "s1mb": s1mb_view()}, | |
| "logs": {s: text_tail(f"logs/{s}.log", 40) for s in ["probe", "baseline", "generate", "train_joint", "train_gold", "train_votes", "eval_joint", "eval_gold", "eval_votes", "metrics", "report", | |
| "v2_generate", "v2_teacher", "v2_merge_teacher", "v2_speed_bench", "v2_sweep_1e-4", "v2_sweep_2e-4", "v2_sweep_3e-4", | |
| "v2_train_v2_joint", "v2_merge_v2_joint", "v2_train_abl_gold", "v2_train_abl_mix", "v2_eval_v2_joint", | |
| "v2_eval_abl_gold", "v2_eval_abl_mix", "v2_metrics_v2"] | |
| if os.path.exists(os.path.join(R, f"logs/{s}.log"))}, | |
| } | |
| for k in ("probe", "generation"): | |
| if data[k] and "engine" in data[k]: | |
| data[k].pop("engine", None) | |
| s = json.dumps(data, ensure_ascii=False, separators=(",", ":"), default=lambda o: None) | |
| with open(out_path, "w", encoding="utf-8") as f: | |
| f.write(s) | |
| print(f"wrote {out_path}: {len(s) / 1e6:.2f} MB; samples {len(picks)}, browse {len(browse)}") | |
| if __name__ == "__main__": | |
| main(sys.argv[1]) | |
Xet Storage Details
- Size:
- 35.4 kB
- Xet hash:
- 33590352175a951e5dc96ff8aba931cdfd8e1527b5b7d8b71668b44afc5f686b
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.