Buckets:
| """Compact reading views of thinking chains for the rl60 error analysis. | |
| python analysis/examples_view.py --items ITEMS.jsonl --ids ID [ID ...] [--head 1200 --tail 1800 --ev 900] [--model rl60|v2|base] | |
| python analysis/examples_view.py --items ITEMS.jsonl --idfile ids.txt --grep "Wait" # also lists lines matching a pattern | |
| Prints the question (evidence truncated), options with gold-target shares, the readout pick and probability, the | |
| thinking answers of rl60 / V2 / base, and the head and tail of the chosen model's chain (whitespace collapsed). | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import os | |
| import re | |
| import sys | |
| ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| sys.path.insert(0, os.path.join(ROOT, "runs", "v2")) | |
| from common import option_text, read_jsonl, split_chain # noqa: E402 | |
| V2 = os.path.join(ROOT, "runs", "v2") | |
| FILES = {"rl60": os.path.join(V2, "results", "predictions", "reason_rl60.jsonl"), | |
| "v2": os.path.join(V2, "results", "predictions", "reason_v2_joint.jsonl"), | |
| "base": os.path.join(ROOT, "results", "predictions", "reason_base.jsonl")} | |
| L = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" | |
| def letter(i): | |
| return "-" if i is None else L[i] | |
| def squash(s): | |
| return re.sub(r"\s+", " ", s).strip() | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--items", required=True) | |
| ap.add_argument("--ids", nargs="*", default=[]) | |
| ap.add_argument("--idfile") | |
| ap.add_argument("--model", default="rl60") | |
| ap.add_argument("--head", type=int, default=1200) | |
| ap.add_argument("--tail", type=int, default=1800) | |
| ap.add_argument("--ev", type=int, default=900) | |
| ap.add_argument("--grep", default=None, help="regex; print chain sentences that match") | |
| a = ap.parse_args() | |
| ids = list(a.ids) + ([l.split()[0] for l in open(a.idfile) if l.strip()] if a.idfile else []) | |
| want = set(ids) | |
| ev = {it["id"]: it for it in read_jsonl(os.path.join(V2, "data", "eval.jsonl")) if it["id"] in want} | |
| rows = {r["id"]: r for r in map(json.loads, open(a.items)) if r["id"] in want} | |
| chains = {r["id"]: r["text"] for r in read_jsonl(FILES[a.model]) if r["id"] in want} | |
| for i in ids: | |
| it, r = ev[i], rows[i] | |
| print("=" * 100) | |
| print(f"{i} | {it['set']} | {it.get('family')} | {it['qtype']} | routed={r['routed']} margin={r['margin']:.2f}") | |
| evd = it["evidence"] | |
| print("EVIDENCE:", squash(evd[:a.ev]) + (" [...]" if len(evd) > a.ev else "")) | |
| print("CRITERION:", squash(it["criterion"])) | |
| for k, o in enumerate(it["options"]): | |
| print(f" {L[k]}. {squash(option_text(o))[:160]} [t={it['target'][k]:.2f}]") | |
| print(f"GOLD {letter(r['gold'])} | readout {letter(r['ro_pick'])} p={r['ro_p']:.2f} (p_gold={r['ro_p_gold']:.2f}) | " | |
| f"rl60 {letter(r['rl60']['answer'])} post_p={r['rl60']['post_p'] or 0:.2f} tok={r['rl60']['tokens']} waits={r['rl60']['waits']} | " | |
| f"v2 {letter(r['v2']['answer'])} tok={r['v2']['tokens']} loop={r['v2']['looped']} | base {letter(r['base']['answer'])}") | |
| chain, tail = split_chain(chains[i]) | |
| c = squash(chain if chain is not None else chains[i]) | |
| if len(c) <= a.head + a.tail + 200: | |
| print(f"CHAIN[{a.model}] ({len(c)} chars):", c) | |
| else: | |
| print(f"CHAIN[{a.model}] ({len(c)} chars) HEAD:", c[:a.head]) | |
| print(" ... TAIL:", c[-a.tail:]) | |
| if a.grep: | |
| for m in re.finditer(r"[^.!?\n]*(?:" + a.grep + r")[^.!?\n]*[.!?]?", c): | |
| print(" >>", m.group(0).strip()[:220]) | |
| print("AFTER </think>:", squash(tail)[:80]) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 3.69 kB
- Xet hash:
- 6edb716c8ecb3e3f415e00591ac5636bdb65f392bc9e80e8978204d837501f13
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.