Conditional_Causal_Difficulty_for_LLMs / scripts /fix_recode_complexity.py
reza2ei's picture
Duplicate from reza2ei/Conditional_Causal_Difficulty_for_LLMs
a0d1de2
Raw History Blame Contribute Delete
5.89 kB
#!/usr/bin/env python
"""Assign a real intrinsic-complexity proxy to ReCode pairs.
The ReCode pipeline (``scripts/recode_pipeline.py``) currently writes a
constant ``complexity_score=0.5`` / ``complexity="medium"`` for every pair
(no per-problem C is computed from the HumanEval problem). The paper-2
noise-response criterion (``Paper/paper2_noise_response.tex``) reads its
confounding probe as ``|rho_C| = |Corr(predictor_output, C_proxy)|``
(``scripts/noise_study.py::_run_one_real``), so a constant proxy makes the
probe undefined (NaN). This script back-fills a genuine per-problem proxy
from the code itself and rewrites the JSONL **in place**.
Proxy choice (see main text of the paper for the criterion's reliance on a
proxy for intrinsic complexity C): the log-token length of the problem's
*canonical reference solution* (``canonical_solution`` in HumanEval),
min-max normalised to [0,1]. This follows the paper's surface-length
baseline philosophy (verbosity of the reference implementation as a proxy
for how much the model/implementation must do) and is independent of the
sourcedness of any single transform, since it is a property of the base
problem only. ``complexity`` is bucketed to easy/medium/hard by tertile for
compatibility with the schema's categorical field.
Usage
-----
python scripts/fix_recode_complexity.py [--pairs data/recode_labeled_pairs.jsonl] [--dry-run]
Run after ``scripts/recode_pipeline.py`` (full 164-problem regeneration), on
the exact output file the noise study will read. Idempotent: re-running on
an already-fixed file recomputes the same values.
"""
from __future__ import annotations
import argparse
import json
from collections import defaultdict
from pathlib import Path
import pyarrow.parquet as pq
REPO_ROOT = Path(__file__).resolve().parents[1]
DEFAULT_PAIRS = REPO_ROOT / "data" / "recode_labeled_pairs.jsonl"
HE_PARQUET = REPO_ROOT / "cache" / "recode" / "humaneval_test.parquet"
# base_id -> (task_id) map derived by parsing "recode-HumanEval-N" ids.
# Pipeline writes base_id = f"recode-{task_id.replace('/', '-')}" i.e.
# "recode-HumanEval-<n>"; HumanEval task_id is "HumanEval/<n>".
import re
_BASE_ID_RE = re.compile(r"^recode-HumanEval-(\d+)$")
def canonical_len(task_id: str, table_rows: dict[str, int]) -> float:
"""Return log character-length of the canonical solution for a task."""
return table_rows[task_id]
def build_sol_len_by_task(parquet: Path) -> dict[str, float]:
t = pq.read_table(str(parquet))
rows = {}
for i in range(len(t)):
task_id = t["task_id"][i].as_py() # "HumanEval/<n>"
sol = t["canonical_solution"][i].as_py() or ""
rows[task_id] = max(0.0, len(sol))
return rows
def normalize(values: list[float]) -> list[float]:
lo, hi = min(values), max(values)
span = hi - lo
if span <= 0:
return [0.5] * len(values)
return [(v - lo) / span for v in values]
def bucket(x: float) -> str:
if x < 1 / 3:
return "easy"
if x < 2 / 3:
return "medium"
return "hard"
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--pairs", default=str(DEFAULT_PAIRS))
parser.add_argument("--parquet", default=str(HE_PARQUET))
parser.add_argument("--dry-run", action="store_true")
args = parser.parse_args()
pairs_path = Path(args.pairs)
parquet_path = Path(args.parquet)
if not pairs_path.exists():
raise SystemExit(f"pairs file not found: {pairs_path}")
if not parquet_path.exists():
raise SystemExit(f"parquet not found: {parquet_path}")
sol_len = build_sol_len_by_task(parquet_path)
with open(pairs_path) as fh:
pairs = [json.loads(line) for line in fh if line.strip()]
if not pairs:
raise SystemExit("no pairs loaded — is the dataset empty?")
# Per base_id, the natural log of the canonical solution length.
raw: dict[str, float] = {}
unmatched = set()
for p in pairs:
bid = p["base_id"]
if bid in raw:
continue
m = _BASE_ID_RE.match(bid)
if not m:
unmatched.add(bid)
raw[bid] = 0.0
continue
task_id = f"HumanEval/{m.group(1)}"
raw[bid] = sol_len.get(task_id, 0.0)
# Assign normalized log-length per base_id, then propagate to every twin.
vals_by_bid = {bid: v for bid, v in raw.items()}
base_ids = list(vals_by_bid.keys())
norm = {bid: x for bid, x in
zip(base_ids, normalize([vals_by_bid[b] for b in base_ids]))}
changed = 0
for p in pairs:
bid = p["base_id"]
score = norm[bid]
if p.get("complexity_score") != score or p.get("complexity") != bucket(score):
p["complexity_score"] = score
p["complexity"] = bucket(score)
changed += 1
n_problems = len(base_ids)
n_pairs = len(pairs)
distinct = len({raw[b] for b in base_ids})
print(f"pairs : {n_pairs}")
print(f"unique problems : {n_problems}")
print(f"distinct sol-lens: {distinct}")
print(f"problems to fix : {changed} row updates")
if unmatched:
print(f"WARNING: {len(unmatched)} base_ids did not match HumanEval pattern "
f"(assigned proxy 0, check): {sorted(unmatched)[:5]} ...")
# Sanity: the probe needs non-degenerate C. Report rank spread.
import statistics
import math
lens = [vals_by_bid[b] for b in base_ids]
print(f"log-len stats : min={min(lens):.1f} med={statistics.median(lens):.1f} "
f"max={max(lens):.1f}")
if args.dry_run:
print("dry-run: not writing")
return
with open(pairs_path, "w") as fh:
for p in pairs:
fh.write(json.dumps(p, ensure_ascii=False) + "\n")
print(f"wrote → {pairs_path}")
if __name__ == "__main__":
main()