Buckets:
| """Quantitative checks behind the patterns named in results/examples_analysis.md (rl60 on the 1,498-item thinking subset). | |
| python analysis/examples_patterns.py --items ITEMS.jsonl --out patterns.json | |
| - chains that appeal to an imagined dataset / online source / "provided solution" (regex), accuracy with and without | |
| - "Wait" count buckets vs accuracy | |
| - LocalLLaMA: direction of errors on ordinal (score) items and on true/false escalation questions; soft score | |
| (annotator share of the picked option) for readout vs thinking | |
| - chain length of rl60 vs V2 on the items where they disagree | |
| - per-set routing: net gain of the routed 30% by set, and an in-sample upper bound for a per-set margin threshold | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import collections | |
| import json | |
| import os | |
| import re | |
| import sys | |
| import numpy as np | |
| ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| sys.path.insert(0, os.path.join(ROOT, "runs", "v2")) | |
| from common import read_jsonl, split_chain # noqa: E402 | |
| V2 = os.path.join(ROOT, "runs", "v2") | |
| FILES = {"rl60": os.path.join(V2, "results", "predictions", "reason_rl60.jsonl"), | |
| "v2": os.path.join(V2, "results", "predictions", "reason_v2_joint.jsonl"), | |
| "base": os.path.join(ROOT, "results", "predictions", "reason_base.jsonl")} | |
| SETS = ["eval_proc", "eval_ts", "opendecisions", "localllama"] | |
| IMAGINED = re.compile(r"(found a (similar|specific)|similar (examples?|questions?|tasks?|cases?|problems?|pattern)|" | |
| r"(provided|known) solution|from a (specific )?(dataset|benchmark)|in (many|some|these|such) " | |
| r"(datasets|benchmarks)|search(ed)? (my internal knowledge|online)|(similar|found|search\w*|look\w*)[^.]{0,40}\bonline\b|" | |
| r"I recall)", re.I) | |
| ESCALATE_TRUE = {"needs_review", "needs_human", "credential_compromise", "true_positive"} | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--items", required=True) | |
| ap.add_argument("--out", required=True) | |
| a = ap.parse_args() | |
| rows = [json.loads(l) for l in open(a.items)] | |
| ids = {r["id"] for r in rows} | |
| ev = {it["id"]: it for it in read_jsonl(os.path.join(V2, "data", "eval.jsonl")) if it["id"] in ids} | |
| out = {} | |
| # 1. imagined-source appeals | |
| imag = {} | |
| for name, path in FILES.items(): | |
| hit = {} | |
| for r in read_jsonl(path): | |
| if r["id"] in ids: | |
| ch, _ = split_chain(r["text"]) | |
| hit[r["id"]] = bool(IMAGINED.search(ch if ch is not None else r["text"])) | |
| res = {} | |
| for s in SETS + ["all"]: | |
| sub = [x for x in rows if s == "all" or x["set"] == s] | |
| h = [x[name]["ok"] for x in sub if hit[x["id"]]] | |
| n = [x[name]["ok"] for x in sub if not hit[x["id"]]] | |
| res[s] = {"share_with_appeal": len(h) / len(sub), "acc_with": float(np.mean(h)) if h else None, | |
| "acc_without": float(np.mean(n)) if n else None, "n_with": len(h)} | |
| imag[name] = res | |
| for x in rows: | |
| x.setdefault("imag", {})[name] = hit[x["id"]] | |
| out["imagined_source_appeal"] = imag | |
| cell = lambda x: ("R" if x["ro_ok"] else "W") + ("R" if x["rl60"]["ok"] else "W") | |
| out["imagined_by_cell_rl60"] = {c: float(np.mean([x["imag"]["rl60"] for x in rows if cell(x) == c])) for c in ("RR", "RW", "WR", "WW")} | |
| # 2. waits buckets | |
| def bucket(w): | |
| return "0" if w == 0 else "1-2" if w <= 2 else "3-9" if w <= 9 else "10+" | |
| wb = collections.defaultdict(list) | |
| for x in rows: | |
| wb[bucket(x["rl60"]["waits"])].append(x) | |
| out["waits_buckets_rl60"] = {b: {"n": len(v), "thinking_acc": float(np.mean([x["rl60"]["ok"] for x in v])), | |
| "readout_acc": float(np.mean([x["ro_ok"] for x in v])), | |
| "share_localllama": float(np.mean([x["set"] == "localllama" for x in v]))} | |
| for b, v in sorted(wb.items())} | |
| # 3. LocalLLaMA direction of errors and soft score | |
| ll = [x for x in rows if x["set"] == "localllama"] | |
| sc = [x for x in ll if x["qtype"] == "score"] | |
| def signed(x, pick): | |
| return pick - x["gold"] | |
| out["localllama"] = { | |
| "n": len(ll), | |
| "score_items": len(sc), | |
| "score_mean_signed_error_readout": float(np.mean([signed(x, x["ro_pick"]) for x in sc])), | |
| "score_mean_signed_error_thinking": float(np.mean([signed(x, x["rl60"]["answer"]) for x in sc if x["rl60"]["answer"] is not None])), | |
| "score_wrong_above_gold_share_readout": float(np.mean([x["ro_pick"] > x["gold"] for x in sc if not x["ro_ok"]])), | |
| "score_wrong_above_gold_share_thinking": float(np.mean([x["rl60"]["answer"] > x["gold"] for x in sc if not x["rl60"]["ok"] and x["rl60"]["answer"] is not None])), | |
| "soft_score_readout": float(np.mean([x["target"][x["ro_pick"]] for x in ll])), | |
| "soft_score_thinking": float(np.mean([x["target"][x["rl60"]["answer"]] if x["rl60"]["answer"] is not None else 0 for x in ll])), | |
| "soft_score_v2": float(np.mean([x["target"][x["v2"]["answer"]] if x["v2"]["answer"] is not None else 0 for x in ll])), | |
| "soft_score_base": float(np.mean([x["target"][x["base"]["answer"]] if x["base"]["answer"] is not None else 0 for x in ll])), | |
| "gold_share_median": float(np.median([max(x["target"]) for x in ll])), | |
| "acc_by_panel_agrees": {str(k): {"n": sum(ev[x["id"]]["panel_agrees"] == k for x in ll), | |
| "readout": float(np.mean([x["ro_ok"] for x in ll if ev[x["id"]]["panel_agrees"] == k])), | |
| "thinking": float(np.mean([x["rl60"]["ok"] for x in ll if ev[x["id"]]["panel_agrees"] == k]))} | |
| for k in (True, False)}, | |
| "thinking_wrong_pick_share_ge_0.25": float(np.mean([x["target"][x["rl60"]["answer"]] >= 0.25 for x in ll | |
| if not x["rl60"]["ok"] and x["rl60"]["answer"] is not None])), | |
| "RW_thinking_pick_share_ge_0.25": float(np.mean([x["target"][x["rl60"]["answer"]] >= 0.25 for x in ll | |
| if x["ro_ok"] and not x["rl60"]["ok"] and x["rl60"]["answer"] is not None])), | |
| "RW_gold_share_lt_0.5": float(np.mean([max(x["target"]) < 0.5 for x in ll if x["ro_ok"] and not x["rl60"]["ok"]])), | |
| } | |
| esc = [x for x in ll if ev[x["id"]]["question_name"] in ESCALATE_TRUE] | |
| tr = lambda x, i: ev[x["id"]]["options"][i]["id"] == "true" | |
| out["localllama"]["escalation_tf"] = { | |
| "n": len(esc), "gold_true_share": float(np.mean([tr(x, x["gold"]) for x in esc])), | |
| "readout_true_share": float(np.mean([tr(x, x["ro_pick"]) for x in esc])), | |
| "thinking_true_share": float(np.mean([tr(x, x["rl60"]["answer"]) for x in esc if x["rl60"]["answer"] is not None])), | |
| "v2_true_share": float(np.mean([tr(x, x["v2"]["answer"]) for x in esc if x["v2"]["answer"] is not None])), | |
| "base_true_share": float(np.mean([tr(x, x["base"]["answer"]) for x in esc if x["base"]["answer"] is not None])), | |
| "readout_acc": float(np.mean([x["ro_ok"] for x in esc])), "thinking_acc": float(np.mean([x["rl60"]["ok"] for x in esc]))} | |
| # severity-ordered choice questions: security disposition, invoice disposition, cs action | |
| order = {"disposition": {"close_benign": 0, "monitor": 1, "investigate": 2, "contain": 3, | |
| "approve": 0, "hold": 1, "manual_review": 2, "reject": 3}} | |
| disp = [x for x in ll if ev[x["id"]]["question_name"] == "disposition"] | |
| lvl = lambda x, i: order["disposition"][ev[x["id"]]["options"][i]["id"]] | |
| out["localllama"]["disposition"] = { | |
| "n": len(disp), | |
| "readout_mean_level_minus_gold": float(np.mean([lvl(x, x["ro_pick"]) - lvl(x, x["gold"]) for x in disp])), | |
| "thinking_mean_level_minus_gold": float(np.mean([lvl(x, x["rl60"]["answer"]) - lvl(x, x["gold"]) for x in disp if x["rl60"]["answer"] is not None]))} | |
| # 4. chain length on rl60 vs V2 disagreements | |
| def med(v): | |
| return float(np.median(v)) if v else None | |
| fl = {"V2R_RLW": [x for x in rows if x["v2"]["ok"] and not x["rl60"]["ok"]], | |
| "V2W_RLR": [x for x in rows if not x["v2"]["ok"] and x["rl60"]["ok"]]} | |
| out["flip_lengths"] = {k: {"n": len(v), "rl60_tokens_median": med([x["rl60"]["tokens"] for x in v]), | |
| "v2_tokens_median": med([x["v2"]["tokens"] for x in v]), | |
| "rl60_shorter_share": float(np.mean([x["rl60"]["tokens"] < x["v2"]["tokens"] for x in v])), | |
| "v2_looped": sum(x["v2"]["looped"] for x in v), "rl60_looped": sum(x["rl60"]["looped"] for x in v), | |
| "by_set": dict(collections.Counter(x["set"] for x in v))} for k, v in fl.items()} | |
| out["base_thinking_by_set"] = {s: float(np.mean([x["base"]["ok"] for x in rows if x["set"] == s])) for s in SETS} | |
| out["v2_thinking_by_set"] = {s: float(np.mean([x["v2"]["ok"] for x in rows if x["set"] == s])) for s in SETS} | |
| # 5. routing by set | |
| n_route = sum(x["routed"] for x in rows) | |
| gain = lambda x: int(x["rl60"]["ok"] if x["rl60"]["answer"] is not None else x["ro_ok"]) - int(x["ro_ok"]) | |
| out["routing_by_set"] = {s: {"routed": sum(x["routed"] for x in rows if x["set"] == s), | |
| "net_gain_routed": sum(gain(x) for x in rows if x["set"] == s and x["routed"]), | |
| "net_gain_if_all_routed": sum(gain(x) for x in rows if x["set"] == s)} for s in SETS} | |
| # in-sample upper bound: best split of the same budget across sets, each set routed in its own margin order | |
| curves = {} | |
| for s in SETS: | |
| sub = sorted([x for x in rows if x["set"] == s], key=lambda x: x["margin"]) | |
| curves[s] = np.concatenate([[0], np.cumsum([gain(x) for x in sub])]) | |
| # max-plus combination of the per-set curves under a total budget of exactly n_route items | |
| acc_curve = {0: (0, {})} | |
| for s in SETS: | |
| nxt = {} | |
| for used, (g0, alloc) in acc_curve.items(): | |
| for k in range(0, min(len(curves[s]) - 1, n_route - used) + 1): | |
| g = g0 + int(curves[s][k]) | |
| if used + k not in nxt or g > nxt[used + k][0]: | |
| nxt[used + k] = (g, alloc | {s: k}) | |
| acc_curve = nxt | |
| best = {"gain": acc_curve[n_route][0], "alloc": acc_curve[n_route][1]} | |
| base_acc = np.mean([x["ro_ok"] for x in rows]) | |
| best["accuracy"] = float(base_acc + best["gain"] / len(rows)) | |
| best["note"] = "in-sample (allocation chosen on these same items): an upper bound, not a deployable estimate" | |
| out["per_set_routing_upper_bound"] = best | |
| out["global_routing_accuracy"] = float(base_acc + sum(gain(x) for x in rows if x["routed"]) / len(rows)) | |
| json.dump(out, open(a.out, "w"), indent=1) | |
| print(json.dumps(out, indent=1)) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 11 kB
- Xet hash:
- 238adf9ebc5fe5855bd3813960bb721588a421e7af882ee16c5bd693639e5230
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.