Download scripts/fix_recode_complexity.py from Upwork-AIML/Conditional_Causal_Difficulty_for_LLMs: direct link, hf CLI and curl.
- Browser
- Download file 5.89 kB
-
https://huggingface.co/Upwork-AIML/Conditional_Causal_Difficulty_for_LLMs/resolve/main/scripts/fix_recode_complexity.py
- Command line
-
hf download hf://Upwork-AIML/Conditional_Causal_Difficulty_for_LLMs/scripts/fix_recode_complexity.py
-
curl -L -o fix_recode_complexity.py https://huggingface.co/Upwork-AIML/Conditional_Causal_Difficulty_for_LLMs/resolve/main/scripts/fix_recode_complexity.py
5.89 kB
| #!/usr/bin/env python | |
| """Assign a real intrinsic-complexity proxy to ReCode pairs. | |
| The ReCode pipeline (``scripts/recode_pipeline.py``) currently writes a | |
| constant ``complexity_score=0.5`` / ``complexity="medium"`` for every pair | |
| (no per-problem C is computed from the HumanEval problem). The paper-2 | |
| noise-response criterion (``Paper/paper2_noise_response.tex``) reads its | |
| confounding probe as ``|rho_C| = |Corr(predictor_output, C_proxy)|`` | |
| (``scripts/noise_study.py::_run_one_real``), so a constant proxy makes the | |
| probe undefined (NaN). This script back-fills a genuine per-problem proxy | |
| from the code itself and rewrites the JSONL **in place**. | |
| Proxy choice (see main text of the paper for the criterion's reliance on a | |
| proxy for intrinsic complexity C): the log-token length of the problem's | |
| *canonical reference solution* (``canonical_solution`` in HumanEval), | |
| min-max normalised to [0,1]. This follows the paper's surface-length | |
| baseline philosophy (verbosity of the reference implementation as a proxy | |
| for how much the model/implementation must do) and is independent of the | |
| sourcedness of any single transform, since it is a property of the base | |
| problem only. ``complexity`` is bucketed to easy/medium/hard by tertile for | |
| compatibility with the schema's categorical field. | |
| Usage | |
| ----- | |
| python scripts/fix_recode_complexity.py [--pairs data/recode_labeled_pairs.jsonl] [--dry-run] | |
| Run after ``scripts/recode_pipeline.py`` (full 164-problem regeneration), on | |
| the exact output file the noise study will read. Idempotent: re-running on | |
| an already-fixed file recomputes the same values. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| from collections import defaultdict | |
| from pathlib import Path | |
| import pyarrow.parquet as pq | |
| REPO_ROOT = Path(__file__).resolve().parents[1] | |
| DEFAULT_PAIRS = REPO_ROOT / "data" / "recode_labeled_pairs.jsonl" | |
| HE_PARQUET = REPO_ROOT / "cache" / "recode" / "humaneval_test.parquet" | |
| # base_id -> (task_id) map derived by parsing "recode-HumanEval-N" ids. | |
| # Pipeline writes base_id = f"recode-{task_id.replace('/', '-')}" i.e. | |
| # "recode-HumanEval-<n>"; HumanEval task_id is "HumanEval/<n>". | |
| import re | |
| _BASE_ID_RE = re.compile(r"^recode-HumanEval-(\d+)$") | |
| def canonical_len(task_id: str, table_rows: dict[str, int]) -> float: | |
| """Return log character-length of the canonical solution for a task.""" | |
| return table_rows[task_id] | |
| def build_sol_len_by_task(parquet: Path) -> dict[str, float]: | |
| t = pq.read_table(str(parquet)) | |
| rows = {} | |
| for i in range(len(t)): | |
| task_id = t["task_id"][i].as_py() # "HumanEval/<n>" | |
| sol = t["canonical_solution"][i].as_py() or "" | |
| rows[task_id] = max(0.0, len(sol)) | |
| return rows | |
| def normalize(values: list[float]) -> list[float]: | |
| lo, hi = min(values), max(values) | |
| span = hi - lo | |
| if span <= 0: | |
| return [0.5] * len(values) | |
| return [(v - lo) / span for v in values] | |
| def bucket(x: float) -> str: | |
| if x < 1 / 3: | |
| return "easy" | |
| if x < 2 / 3: | |
| return "medium" | |
| return "hard" | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--pairs", default=str(DEFAULT_PAIRS)) | |
| parser.add_argument("--parquet", default=str(HE_PARQUET)) | |
| parser.add_argument("--dry-run", action="store_true") | |
| args = parser.parse_args() | |
| pairs_path = Path(args.pairs) | |
| parquet_path = Path(args.parquet) | |
| if not pairs_path.exists(): | |
| raise SystemExit(f"pairs file not found: {pairs_path}") | |
| if not parquet_path.exists(): | |
| raise SystemExit(f"parquet not found: {parquet_path}") | |
| sol_len = build_sol_len_by_task(parquet_path) | |
| with open(pairs_path) as fh: | |
| pairs = [json.loads(line) for line in fh if line.strip()] | |
| if not pairs: | |
| raise SystemExit("no pairs loaded — is the dataset empty?") | |
| # Per base_id, the natural log of the canonical solution length. | |
| raw: dict[str, float] = {} | |
| unmatched = set() | |
| for p in pairs: | |
| bid = p["base_id"] | |
| if bid in raw: | |
| continue | |
| m = _BASE_ID_RE.match(bid) | |
| if not m: | |
| unmatched.add(bid) | |
| raw[bid] = 0.0 | |
| continue | |
| task_id = f"HumanEval/{m.group(1)}" | |
| raw[bid] = sol_len.get(task_id, 0.0) | |
| # Assign normalized log-length per base_id, then propagate to every twin. | |
| vals_by_bid = {bid: v for bid, v in raw.items()} | |
| base_ids = list(vals_by_bid.keys()) | |
| norm = {bid: x for bid, x in | |
| zip(base_ids, normalize([vals_by_bid[b] for b in base_ids]))} | |
| changed = 0 | |
| for p in pairs: | |
| bid = p["base_id"] | |
| score = norm[bid] | |
| if p.get("complexity_score") != score or p.get("complexity") != bucket(score): | |
| p["complexity_score"] = score | |
| p["complexity"] = bucket(score) | |
| changed += 1 | |
| n_problems = len(base_ids) | |
| n_pairs = len(pairs) | |
| distinct = len({raw[b] for b in base_ids}) | |
| print(f"pairs : {n_pairs}") | |
| print(f"unique problems : {n_problems}") | |
| print(f"distinct sol-lens: {distinct}") | |
| print(f"problems to fix : {changed} row updates") | |
| if unmatched: | |
| print(f"WARNING: {len(unmatched)} base_ids did not match HumanEval pattern " | |
| f"(assigned proxy 0, check): {sorted(unmatched)[:5]} ...") | |
| # Sanity: the probe needs non-degenerate C. Report rank spread. | |
| import statistics | |
| import math | |
| lens = [vals_by_bid[b] for b in base_ids] | |
| print(f"log-len stats : min={min(lens):.1f} med={statistics.median(lens):.1f} " | |
| f"max={max(lens):.1f}") | |
| if args.dry_run: | |
| print("dry-run: not writing") | |
| return | |
| with open(pairs_path, "w") as fh: | |
| for p in pairs: | |
| fh.write(json.dumps(p, ensure_ascii=False) + "\n") | |
| print(f"wrote → {pairs_path}") | |
| if __name__ == "__main__": | |
| main() | |