Buckets:
| """Count tables for the rl60 error analysis, from the per-item table written by examples_build.py. | |
| python analysis/examples_counts.py --items /path/items.jsonl --out /path/counts.json | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import collections | |
| import json | |
| SETS = ["eval_proc", "eval_ts", "opendecisions", "localllama"] | |
| NAMES = {"eval_proc": "procedural", "eval_ts": "tasksource", "opendecisions": "OpenDecisions", "localllama": "LocalLLaMA"} | |
| CELLS = ["RR", "RW", "WR", "WW"] # readout right/wrong x thinking right/wrong | |
| def cell(r, think="rl60"): | |
| return ("R" if r["ro_ok"] else "W") + ("R" if r[think]["ok"] else "W") | |
| def table(rows): | |
| c = collections.Counter(cell(r) for r in rows) | |
| cr = collections.Counter(cell(r) for r in rows if r["routed"]) | |
| return {"n": len(rows), "routed": sum(r["routed"] for r in rows), | |
| "cells": {k: c[k] for k in CELLS}, "cells_routed": {k: cr[k] for k in CELLS}, | |
| "readout_acc": sum(r["ro_ok"] for r in rows) / len(rows), | |
| "thinking_acc": sum(r["rl60"]["ok"] for r in rows) / len(rows), | |
| "routed_system_acc": sum((r["rl60"]["ok"] if (r["routed"] and r["rl60"]["answer"] is not None) else r["ro_ok"]) | |
| for r in rows) / len(rows)} | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--items", required=True) | |
| ap.add_argument("--out", required=True) | |
| a = ap.parse_args() | |
| rows = [json.loads(l) for l in open(a.items)] | |
| out = {"all": table(rows)} | {s: table([r for r in rows if r["set"] == s]) for s in SETS} | |
| # RL vs V2 thinking transitions | |
| tr = collections.Counter(("R" if r["v2"]["ok"] else "W") + ("R" if r["rl60"]["ok"] else "W") for r in rows) | |
| out["v2_to_rl60_thinking"] = {k: tr[k] for k in CELLS} | { | |
| "by_set": {s: dict(collections.Counter(("R" if r["v2"]["ok"] else "W") + ("R" if r["rl60"]["ok"] else "W") | |
| for r in rows if r["set"] == s)) for s in SETS}, | |
| "v2_looped_among_WR": sum(r["v2"]["looped"] for r in rows if not r["v2"]["ok"] and r["rl60"]["ok"]), | |
| "v2_no_answer_among_WR": sum(r["v2"]["answer"] is None for r in rows if not r["v2"]["ok"] and r["rl60"]["ok"])} | |
| tb = collections.Counter(("R" if r["base"]["ok"] else "W") + ("R" if r["rl60"]["ok"] else "W") for r in rows) | |
| out["base_to_rl60_thinking"] = {k: tb[k] for k in CELLS} | |
| # parsed letter vs post-chain readout argmax | |
| out["answer_vs_post_lp"] = {"disagree": sum(r["rl60"]["answer"] != r["rl60"]["post_pick"] for r in rows), | |
| "no_answer": sum(r["rl60"]["answer"] is None for r in rows), | |
| "post_acc": sum(r["rl60"]["post_ok"] for r in rows) / len(rows)} | |
| # routing: readout right, sent to thinking, thinking wrong (the routing "harm") | |
| rt = [r for r in rows if r["routed"]] | |
| out["routing"] = {"routed": len(rt), "harm_RW": sum(cell(r) == "RW" for r in rt), "help_WR": sum(cell(r) == "WR" for r in rt), | |
| "routed_readout_acc": sum(r["ro_ok"] for r in rt) / len(rt), | |
| "routed_thinking_acc": sum(r["rl60"]["ok"] for r in rt) / len(rt), | |
| "unrouted_readout_acc": sum(r["ro_ok"] for r in rows if not r["routed"]) / (len(rows) - len(rt)), | |
| "unrouted_thinking_acc": sum(r["rl60"]["ok"] for r in rows if not r["routed"]) / (len(rows) - len(rt)), | |
| "margin_cutoff": max(r["margin"] for r in rt), | |
| "missed_WR_outside_route": sum(cell(r) == "WR" for r in rows if not r["routed"]), | |
| "RW_outside_route": sum(cell(r) == "RW" for r in rows if not r["routed"])} | |
| # confidently wrong fast readout: wrong with calibrated top prob >= 0.8 / 0.9 | |
| out["confident_wrong_readout"] = {f">={t}": {"n": sum((not r["ro_ok"]) and r["ro_p"] >= t for r in rows), | |
| "thinking_fixes": sum((not r["ro_ok"]) and r["ro_p"] >= t and r["rl60"]["ok"] for r in rows), | |
| "by_set": dict(collections.Counter(r["set"] for r in rows if (not r["ro_ok"]) and r["ro_p"] >= t))} | |
| for t in (0.7, 0.8, 0.9)} | |
| # oracle: route optimally at the same 30% budget | |
| out["oracle_any_right"] = sum(r["ro_ok"] or r["rl60"]["ok"] for r in rows) / len(rows) | |
| # tokens / waits by cell | |
| def med(v): | |
| v = sorted(v) | |
| return v[len(v) // 2] if v else None | |
| out["by_cell_chain"] = {k: {"n": sum(cell(r) == k for r in rows), | |
| "tokens_median": med([r["rl60"]["tokens"] for r in rows if cell(r) == k]), | |
| "waits_median": med([r["rl60"]["waits"] for r in rows if cell(r) == k]), | |
| "post_p_median": med([r["rl60"]["post_p"] or 0 for r in rows if cell(r) == k]), | |
| "post_p_ge_0.9": sum((r["rl60"]["post_p"] or 0) >= 0.9 for r in rows if cell(r) == k), | |
| "readout_p_median": med([r["ro_p"] for r in rows if cell(r) == k])} for k in CELLS} | |
| out["by_qtype"] = {q: dict(collections.Counter(cell(r) for r in rows if r["qtype"] == q)) for q in ("choice", "noul", "score")} | |
| json.dump(out, open(a.out, "w"), indent=1) | |
| print(json.dumps(out, indent=1)) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 5.4 kB
- Xet hash:
- 015baa2e66f33bff7eb68005490e98accb7c3643ba4cfd75c050ae723ca8ff82
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.