Buckets:
| """Which questions RL may use, decided before RL starts and written to data/rl_prompts.jsonl with the reason. | |
| python rl_select.py --model checkpoints/v2_joint_merged --fresh data/rl_fresh_pool.jsonl # the RL run | |
| python rl_select.py # rules only (counts, no GPU) | |
| In-domain (training-pool questions with 4 chains from the 4B): | |
| 1. Mixed for the 4B: 1-3 of its own 4 chains correct (all correct or all wrong gives GRPO nothing to compare). | |
| Teacher records are left out: their mix says nothing about the 4B. | |
| 2. Label noise. Procedural labels are computed, so they are exact: kept at 1-3 of 4. Soft-label items (annotator vote | |
| shares) are kept, and RL rewards them by the share of annotators who chose the answer. Hard-label tasksource items | |
| need at least 2 of 4 chains backing gold: at 1 of 4, three chains reject the label, the pattern of a wrong label | |
| (the small-loss rule of noisy-label learning, e.g. Co-teaching). | |
| 3. Still useful after V2 training (with --model): the V2 fast readout, one forward pass per item. Dropped when it | |
| already gives the gold answer >= 0.95 (thinking mode would get 8 of 8 and teach nothing) and, for hard-label | |
| tasksource items, when it gives gold < 0.05 (the trained model is sure the label is wrong). | |
| Fresh (--fresh, rl_fresh.py: tasksource questions never trained on, mostly reasoning-heavy sources): | |
| 4. No chains to check, so the V2 readout decides: kept when it gives the gold answer between 0.15 and 0.85 (unsure, so | |
| thinking mode should split; this also drops labels the model is sure are wrong). Not memorised: the readout never | |
| saw these questions. | |
| The final list mixes --n questions: --fresh-share of them fresh (--reasoning-share of those from reasoning sources), | |
| the rest in-domain. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import collections | |
| import json | |
| import random | |
| import numpy as np | |
| from common import read_jsonl, softmax | |
| SATURATED, NOISE = 0.95, 0.05 | |
| FRESH_BAND = (0.15, 0.85) | |
| def kind(item: dict) -> str: | |
| if item["id"].startswith("proc:"): | |
| return "exact" | |
| return "soft" if max(item["target"]) < 1.0 else "hard" | |
| def main() -> None: | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--pool", default="data/train_pool.jsonl") | |
| ap.add_argument("--gen", default="data/gen.jsonl") | |
| ap.add_argument("--fresh", default=None, help="rl_fresh.py output") | |
| ap.add_argument("--model", default=None, help="merged V2 checkpoint for the readout check") | |
| ap.add_argument("--n", type=int, default=480, help="questions in the final list (120 steps x 4)") | |
| ap.add_argument("--fresh-share", type=float, default=0.6) | |
| ap.add_argument("--reasoning-share", type=float, default=2 / 3, help="of the fresh questions") | |
| ap.add_argument("--seed", type=int, default=43) | |
| ap.add_argument("--out", default="data/rl_prompts.jsonl") | |
| args = ap.parse_args() | |
| pool = {x["id"]: x for x in read_jsonl(args.pool)} | |
| fresh = {x["id"]: x for x in read_jsonl(args.fresh)} if args.fresh else {} | |
| counts = collections.Counter() | |
| rows = [] | |
| for rec in read_jsonl(args.gen): | |
| it = pool.get(rec["id"]) | |
| if it is None or rec.get("source") == "teacher": | |
| continue | |
| correct = sum(1 for s in rec["samples"] if s["closed"] and s["answer"] is not None and rec["order"][s["answer"]] == it["gold"]) | |
| if not 1 <= correct <= 3: | |
| continue | |
| k = kind(it) | |
| counts[f"mixed_{k}"] += 1 | |
| if k == "hard" and correct < 2: | |
| counts["dropped_hard_label_1_of_4"] += 1 | |
| continue | |
| rows.append({"id": it["id"], "kind": k, "fresh": False, "base_correct_of_4": correct}) | |
| rows += [{"id": x["id"], "kind": kind(x), "fresh": True, "reasoning_source": x["reasoning_source"], "source": x["family"]} | |
| for x in fresh.values()] | |
| if args.model: | |
| from transformers import AutoTokenizer | |
| from vllm_utils import make_llm, readout | |
| tok = AutoTokenizer.from_pretrained(args.model) | |
| llm = make_llm(args.model) | |
| items = [fresh[r["id"]] if r["fresh"] else pool[r["id"]] for r in rows] | |
| lps = readout(llm, tok, items, [list(range(len(x["options"]))) for x in items]) | |
| keep = [] | |
| for r, it, lp in zip(rows, items, lps): | |
| p = float(softmax(np.asarray(lp), 1.0)[it["gold"]]) | |
| r["v2_readout_p_gold"] = p | |
| if r["fresh"]: | |
| if FRESH_BAND[0] <= p <= FRESH_BAND[1]: | |
| keep.append(r) | |
| else: | |
| counts["fresh_dropped_" + ("sure_right" if p > FRESH_BAND[1] else "sure_wrong_or_noisy")] += 1 | |
| elif p >= SATURATED: | |
| counts["dropped_v2_already_sure_and_right"] += 1 | |
| elif r["kind"] == "hard" and p < NOISE: | |
| counts["dropped_v2_sure_label_is_wrong"] += 1 | |
| else: | |
| keep.append(r) | |
| rows = keep | |
| counts.update(("kept_fresh_" + ("reasoning" if r["reasoning_source"] else "general")) if r["fresh"] else f"kept_{r['kind']}" for r in rows) | |
| rng = random.Random(args.seed) | |
| rng.shuffle(rows) | |
| if fresh: | |
| n_fresh = round(args.n * args.fresh_share) | |
| n_reason = round(n_fresh * args.reasoning_share) | |
| fr = [r for r in rows if r["fresh"] and r["reasoning_source"]][:n_reason] | |
| fg = [r for r in rows if r["fresh"] and not r["reasoning_source"]][:n_fresh - len(fr)] | |
| ind = [r for r in rows if not r["fresh"]][:args.n - len(fr) - len(fg)] | |
| rows = fr + fg + ind | |
| rng.shuffle(rows) | |
| counts.update(("final_fresh_" + ("reasoning" if r["reasoning_source"] else "general")) if r["fresh"] else "final_in_domain" for r in rows) | |
| counts["final"] = len(rows) | |
| with open(args.out, "w") as f: | |
| for r in rows: | |
| f.write(json.dumps(r) + "\n") | |
| print(json.dumps(dict(counts), indent=1)) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 5.97 kB
- Xet hash:
- c28962b9e6b5358e7167151bba2a90309ed1a15e5e88198e2bf19a562ecc4f7b
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.