#!/usr/bin/env python """Assign a real intrinsic-complexity proxy to ReCode pairs. The ReCode pipeline (``scripts/recode_pipeline.py``) currently writes a constant ``complexity_score=0.5`` / ``complexity="medium"`` for every pair (no per-problem C is computed from the HumanEval problem). The paper-2 noise-response criterion (``Paper/paper2_noise_response.tex``) reads its confounding probe as ``|rho_C| = |Corr(predictor_output, C_proxy)|`` (``scripts/noise_study.py::_run_one_real``), so a constant proxy makes the probe undefined (NaN). This script back-fills a genuine per-problem proxy from the code itself and rewrites the JSONL **in place**. Proxy choice (see main text of the paper for the criterion's reliance on a proxy for intrinsic complexity C): the log-token length of the problem's *canonical reference solution* (``canonical_solution`` in HumanEval), min-max normalised to [0,1]. This follows the paper's surface-length baseline philosophy (verbosity of the reference implementation as a proxy for how much the model/implementation must do) and is independent of the sourcedness of any single transform, since it is a property of the base problem only. ``complexity`` is bucketed to easy/medium/hard by tertile for compatibility with the schema's categorical field. Usage ----- python scripts/fix_recode_complexity.py [--pairs data/recode_labeled_pairs.jsonl] [--dry-run] Run after ``scripts/recode_pipeline.py`` (full 164-problem regeneration), on the exact output file the noise study will read. Idempotent: re-running on an already-fixed file recomputes the same values. """ from __future__ import annotations import argparse import json from collections import defaultdict from pathlib import Path import pyarrow.parquet as pq REPO_ROOT = Path(__file__).resolve().parents[1] DEFAULT_PAIRS = REPO_ROOT / "data" / "recode_labeled_pairs.jsonl" HE_PARQUET = REPO_ROOT / "cache" / "recode" / "humaneval_test.parquet" # base_id -> (task_id) map derived by parsing "recode-HumanEval-N" ids. # Pipeline writes base_id = f"recode-{task_id.replace('/', '-')}" i.e. # "recode-HumanEval-"; HumanEval task_id is "HumanEval/". import re _BASE_ID_RE = re.compile(r"^recode-HumanEval-(\d+)$") def canonical_len(task_id: str, table_rows: dict[str, int]) -> float: """Return log character-length of the canonical solution for a task.""" return table_rows[task_id] def build_sol_len_by_task(parquet: Path) -> dict[str, float]: t = pq.read_table(str(parquet)) rows = {} for i in range(len(t)): task_id = t["task_id"][i].as_py() # "HumanEval/" sol = t["canonical_solution"][i].as_py() or "" rows[task_id] = max(0.0, len(sol)) return rows def normalize(values: list[float]) -> list[float]: lo, hi = min(values), max(values) span = hi - lo if span <= 0: return [0.5] * len(values) return [(v - lo) / span for v in values] def bucket(x: float) -> str: if x < 1 / 3: return "easy" if x < 2 / 3: return "medium" return "hard" def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--pairs", default=str(DEFAULT_PAIRS)) parser.add_argument("--parquet", default=str(HE_PARQUET)) parser.add_argument("--dry-run", action="store_true") args = parser.parse_args() pairs_path = Path(args.pairs) parquet_path = Path(args.parquet) if not pairs_path.exists(): raise SystemExit(f"pairs file not found: {pairs_path}") if not parquet_path.exists(): raise SystemExit(f"parquet not found: {parquet_path}") sol_len = build_sol_len_by_task(parquet_path) with open(pairs_path) as fh: pairs = [json.loads(line) for line in fh if line.strip()] if not pairs: raise SystemExit("no pairs loaded — is the dataset empty?") # Per base_id, the natural log of the canonical solution length. raw: dict[str, float] = {} unmatched = set() for p in pairs: bid = p["base_id"] if bid in raw: continue m = _BASE_ID_RE.match(bid) if not m: unmatched.add(bid) raw[bid] = 0.0 continue task_id = f"HumanEval/{m.group(1)}" raw[bid] = sol_len.get(task_id, 0.0) # Assign normalized log-length per base_id, then propagate to every twin. vals_by_bid = {bid: v for bid, v in raw.items()} base_ids = list(vals_by_bid.keys()) norm = {bid: x for bid, x in zip(base_ids, normalize([vals_by_bid[b] for b in base_ids]))} changed = 0 for p in pairs: bid = p["base_id"] score = norm[bid] if p.get("complexity_score") != score or p.get("complexity") != bucket(score): p["complexity_score"] = score p["complexity"] = bucket(score) changed += 1 n_problems = len(base_ids) n_pairs = len(pairs) distinct = len({raw[b] for b in base_ids}) print(f"pairs : {n_pairs}") print(f"unique problems : {n_problems}") print(f"distinct sol-lens: {distinct}") print(f"problems to fix : {changed} row updates") if unmatched: print(f"WARNING: {len(unmatched)} base_ids did not match HumanEval pattern " f"(assigned proxy 0, check): {sorted(unmatched)[:5]} ...") # Sanity: the probe needs non-degenerate C. Report rank spread. import statistics import math lens = [vals_by_bid[b] for b in base_ids] print(f"log-len stats : min={min(lens):.1f} med={statistics.median(lens):.1f} " f"max={max(lens):.1f}") if args.dry_run: print("dry-run: not writing") return with open(pairs_path, "w") as fh: for p in pairs: fh.write(json.dumps(p, ensure_ascii=False) + "\n") print(f"wrote → {pairs_path}") if __name__ == "__main__": main()