Buckets:

Mercity
/

Desicion-Model-Custom

Files

xet

Mercity/Desicion-Model-Custom / code /report.py

Pranav2748

4 days ago

download

raw

8.98 kB

"""results/report.md from results/metrics.json (plus probe, generation and training metadata)."""
from __future__ import annotations

import json
import os


def pct(x):
    return "–" if x is None else f"{100 * x:.1f}"


def num(x, d=3):
    return "–" if x is None else f"{x:.{d}f}"


def load(path):
    return json.load(open(path)) if os.path.exists(path) else None


def main() -> None:
    m = load("results/metrics.json")
    probe, gen = load("results/probe.json"), load("results/generation.json")
    L = ["# Reasoning decision model: results", ""]
    H = m["hypotheses"]
    L += ["## Hypotheses", "", "| # | Result | Numbers | Confirmed |", "|---|---|---|---|"]
    if "H1" in H:
        h = H["H1"]
        L.append(f"| H1 | joint readout vs gold-only, in-distribution accuracy | {100 * h['joint_minus_gold_only_acc']:+.1f} pts "
                 f"[{100 * h['ci95'][0]:+.1f}, {100 * h['ci95'][1]:+.1f}] | {'yes' if h['confirmed'] else 'no'} |")
    if "H2" in H:
        h = H["H2"]
        parts = [f"vs {k.split('_minus_')[1]} {v['delta']:+.3f} [{v['ci95'][0]:+.3f}, {v['ci95'][1]:+.3f}]"
                 for k, v in h.items() if k != "confirmed"]
        L.append(f"| H2 | disagreement AUROC, joint vs others (need +0.03) | {'; '.join(parts)} | {'yes' if h['confirmed'] else 'no'} |")
    if "H3" in H:
        h = H["H3"]
        L.append(f"| H3 | joint reasoner vs base reasoner, all eval | {100 * h['joint_minus_base_reasoning_acc']:+.1f} pts "
                 f"[{100 * h['ci95'][0]:+.1f}, {100 * h['ci95'][1]:+.1f}] | {'yes' if h['confirmed'] else 'no'} |")
    if "H4" in H:
        h = H["H4"]
        extra = f"; learned router {pct(h.get('gap_recovered_f0.3_learned_router'))}%" if "gap_recovered_f0.3_learned_router" in h else ""
        L.append(f"| H4 | gap recovered at f = 0.3 (need 80%) | {pct(h['gap_recovered_f0.3'])}% of "
                 f"{pct(h['readout_acc'])} → {pct(h['reasoning_acc'])}{extra} | {'yes' if h['confirmed'] else 'no'} |")
    L += [""]

    L += ["## Readout", "", "Accuracy (%) after per-type temperatures fitted on the calibration set; ECE uses 15 bins.", "",
          "| Readout | Procedural | tasksource | OpenDecisions | LocalLLaMA | LocalLLaMA (panel agrees) | OD Capability | OD Capability (raw) | In-dist ECE | In-dist NLL |",
          "|---|---|---|---|---|---|---|---|---|---|"]
    for name, b in m["readout"].items():
        s = b["by_set"]
        L.append(f"| {name} | {pct(s['eval_proc']['accuracy'])} | {pct(s['eval_ts']['accuracy'])} | {pct(s['opendecisions']['accuracy'])} | "
                 f"{pct(s['localllama']['accuracy'])} | {pct(b['localllama']['accuracy_panel_agrees'])} | "
                 f"{num(b['od_capability']['capability'], 1)} | {num(b['od_capability_raw']['capability'], 1)} | "
                 f"{num(b['in_distribution']['ece15'])} | {num(b['in_distribution']['nll'])} |")
    L += ["", "Temperatures (choice / yes-no / score): " + "; ".join(
        f"{n} {b['temperatures']['choice']:.2f} / {b['temperatures']['noul']:.2f} / {b['temperatures']['score']:.2f}"
        for n, b in m["readout"].items()), ""]

    if m["reasoning"]:
        L += ["## Reasoning (one sample per item)", "",
              "| Reasoner | Procedural | tasksource | OpenDecisions | LocalLLaMA | All | Median tokens | p90 tokens | Looped | Letter = readout after chain |",
              "|---|---|---|---|---|---|---|---|---|---|"]
        for name, b in m["reasoning"].items():
            L.append(f"| {name} | {pct(b['eval_proc']['accuracy'])} | {pct(b['eval_ts']['accuracy'])} | {pct(b['opendecisions']['accuracy'])} | "
                     f"{pct(b['localllama']['accuracy'])} | {pct(b['all']['accuracy'])} | {b['all']['tokens_median']:.0f} | "
                     f"{b['all']['tokens_p90']:.0f} | {pct(b['all']['looped_share'])}% | {pct(b['all']['letter_matches_post_readout'])}% |")
        L += [""]

    if m["agreement"]:
        L += ["## Does the readout agree with its reasoning?", "",
              "Agreement = the readout's answer equals the same model's thinking-mode answer (the base reasoner for the readout-only ablations). "
              "The thinking call never sees the readout's answer, so this is also the faithfulness check of explanation on demand.", "",
              "| Readout | Agreement | Procedural | tasksource | OpenDecisions | LocalLLaMA | Disagreement AUROC | Fixable AUROC |",
              "|---|---|---|---|---|---|---|---|"]
        for name, b in m["agreement"].items():
            s = b["agreement_by_set"]
            L.append(f"| {name} | {pct(b['agreement'])} | {pct(s['eval_proc'])} | {pct(s['eval_ts'])} | {pct(s['opendecisions'])} | "
                     f"{pct(s['localllama'])} | {num(b['disagreement_auroc'])} | {num(b['fixable_auroc'])} |")
        L += [""]

    if m["routing"]:
        L += ["## Routing", "", "Route the lowest-margin share f of items to thinking mode (all eval sets pooled).", "",
              "| Readout | Router | f=0 | 0.1 | 0.2 | 0.3 | 0.5 | 1.0 | tokens at 0.3 | random at 0.3 |", "|---|---|---|---|---|---|---|---|---|---|"]
        for name, b in m["routing"].items():
            for kind in ("margin", "learned"):
                if kind not in b:
                    continue
                c = {r["f"]: r for r in b[kind]}
                L.append(f"| {name} | {kind} | {pct(c[0.0]['accuracy'])} | {pct(c[0.1]['accuracy'])} | {pct(c[0.2]['accuracy'])} | "
                         f"{pct(c[0.3]['accuracy'])} | {pct(c[0.5]['accuracy'])} | {pct(c[1.0]['accuracy'])} | "
                         f"{c[0.3]['mean_tokens']:.0f} | {pct(c[0.3]['random_accuracy'])} |")
        L += [""]

    if m.get("system"):
        L += ["## The deployable system", "",
              "Readout by default; items with the lowest readout margin go to thinking mode, whose probabilities are the readout "
              "after the chain (temperatures fitted on calibration chains only). All eval sets; OpenDecisions Capability with the official scorer.", "",
              "| Model | Mode | Routed | Procedural | tasksource | OpenDecisions | LocalLLaMA | All | OD Capability | Mean generated tokens |",
              "|---|---|---|---|---|---|---|---|---|---|"]
        for name, b in m["system"].items():
            for key, label in [("readout_only", "readout only"), ("routed_margin_f0.1", "routed"), ("routed_margin_f0.2", "routed"),
                               ("routed_margin_f0.3", "routed"), ("routed_margin_f0.5", "routed"), ("thinking_always", "thinking always")]:
                if key not in b:
                    continue
                r = b[key]
                L.append(f"| {name} | {label} | {pct(r['routed_share'])}% | {pct(r['acc_eval_proc'])} | {pct(r['acc_eval_ts'])} | "
                         f"{pct(r['acc_opendecisions'])} | {pct(r['acc_localllama'])} | {pct(r['acc_all'])} | "
                         f"{num(r['od_capability']['capability'], 1)} | {r['mean_tokens']:.0f} |")
        L += ["", "Post-chain temperatures (choice / yes-no / score): " + "; ".join(
            f"{n} {b['post_temperatures']['choice']:.2f} / {b['post_temperatures']['noul']:.2f} / {b['post_temperatures']['score']:.2f}"
            for n, b in m["system"].items()), ""]

    if m["latency"]:
        L += ["## Latency (batch size 1)", "", "| Model | Readout median | Readout p90 | Reasoning median | Reasoning p90 |", "|---|---|---|---|---|"]
        for name, b in m["latency"].items():
            L.append(f"| {name} | {b['readout_ms']['median']:.0f} ms | {b['readout_ms']['p90']:.0f} ms | "
                     f"{b['reasoning_ms']['median'] / 1000:.1f} s | {b['reasoning_ms']['p90'] / 1000:.1f} s |")
        L += [""]

    L += ["## Run", ""]
    if probe:
        L.append(f"- Throughput probe: {probe['gen_tok_per_s']:.0f} generated tokens/s; chain tokens median {probe['tokens_median']:.0f}, "
                 f"p90 {probe['tokens_p90']:.0f}, max {probe['tokens_max']}; parse rate {pct(probe['parse_rate'])}%.")
    if gen:
        L.append(f"- Generation: {gen['items']} items, {gen['samples']} chains, {gen['tokens_total'] / 1e6:.1f}M tokens; "
                 f"{gen['items_with_correct_chain']} items with a correct chain; {gen['looped']} chains looped to the context limit.")
    for name in ("joint", "gold_only", "votes_only"):
        meta = load(f"results/adapters/{name}/train_meta.json")
        if meta:
            L.append(f"- Training {name}: {meta['reasoner_examples']} reasoner + {meta['readout_examples']} readout examples, "
                     f"{meta['train_tokens'] / 1e6:.1f}M tokens, {meta['steps_done']}/{meta['steps_planned']} steps, "
                     f"{meta['train_seconds'] / 60:.0f} min ({meta['tok_per_s']:.0f} tokens/s).")
    L.append("- Single seed (17); see results/DEVIATIONS.md for every change to the plan.")
    with open("results/report.md", "w") as f:
        f.write("\n".join(L) + "\n")
    print("\n".join(L))


if __name__ == "__main__":
    main()

Xet Storage Details

Size:: 8.98 kB
Xet hash:: 84e549db8f49a0ab6e35b5a369e095d5d4dd0d378ae47e2d5ad72289eca08dd2

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.

	"""results/report.md from results/metrics.json (plus probe, generation and training metadata)."""
	from __future__ import annotations

	import json
	import os


	def pct(x):
	return "–" if x is None else f"{100 * x:.1f}"


	def num(x, d=3):
	return "–" if x is None else f"{x:.{d}f}"


	def load(path):
	return json.load(open(path)) if os.path.exists(path) else None


	def main() -> None:
	m = load("results/metrics.json")
	probe, gen = load("results/probe.json"), load("results/generation.json")
	L = ["# Reasoning decision model: results", ""]
	H = m["hypotheses"]
	L += ["## Hypotheses", "", "\| # \| Result \| Numbers \| Confirmed \|", "\|---\|---\|---\|---\|"]
	if "H1" in H:
	h = H["H1"]
	L.append(f"\| H1 \| joint readout vs gold-only, in-distribution accuracy \| {100 * h['joint_minus_gold_only_acc']:+.1f} pts "
	f"[{100 * h['ci95'][0]:+.1f}, {100 * h['ci95'][1]:+.1f}] \| {'yes' if h['confirmed'] else 'no'} \|")
	if "H2" in H:
	h = H["H2"]
	parts = [f"vs {k.split('_minus_')[1]} {v['delta']:+.3f} [{v['ci95'][0]:+.3f}, {v['ci95'][1]:+.3f}]"
	for k, v in h.items() if k != "confirmed"]
	L.append(f"\| H2 \| disagreement AUROC, joint vs others (need +0.03) \| {'; '.join(parts)} \| {'yes' if h['confirmed'] else 'no'} \|")
	if "H3" in H:
	h = H["H3"]
	L.append(f"\| H3 \| joint reasoner vs base reasoner, all eval \| {100 * h['joint_minus_base_reasoning_acc']:+.1f} pts "
	f"[{100 * h['ci95'][0]:+.1f}, {100 * h['ci95'][1]:+.1f}] \| {'yes' if h['confirmed'] else 'no'} \|")
	if "H4" in H:
	h = H["H4"]
	extra = f"; learned router {pct(h.get('gap_recovered_f0.3_learned_router'))}%" if "gap_recovered_f0.3_learned_router" in h else ""
	L.append(f"\| H4 \| gap recovered at f = 0.3 (need 80%) \| {pct(h['gap_recovered_f0.3'])}% of "
	f"{pct(h['readout_acc'])} → {pct(h['reasoning_acc'])}{extra} \| {'yes' if h['confirmed'] else 'no'} \|")
	L += [""]

	L += ["## Readout", "", "Accuracy (%) after per-type temperatures fitted on the calibration set; ECE uses 15 bins.", "",
	"\| Readout \| Procedural \| tasksource \| OpenDecisions \| LocalLLaMA \| LocalLLaMA (panel agrees) \| OD Capability \| OD Capability (raw) \| In-dist ECE \| In-dist NLL \|",
	"\|---\|---\|---\|---\|---\|---\|---\|---\|---\|---\|"]
	for name, b in m["readout"].items():
	s = b["by_set"]
	L.append(f"\| {name} \| {pct(s['eval_proc']['accuracy'])} \| {pct(s['eval_ts']['accuracy'])} \| {pct(s['opendecisions']['accuracy'])} \| "
	f"{pct(s['localllama']['accuracy'])} \| {pct(b['localllama']['accuracy_panel_agrees'])} \| "
	f"{num(b['od_capability']['capability'], 1)} \| {num(b['od_capability_raw']['capability'], 1)} \| "
	f"{num(b['in_distribution']['ece15'])} \| {num(b['in_distribution']['nll'])} \|")
	L += ["", "Temperatures (choice / yes-no / score): " + "; ".join(
	f"{n} {b['temperatures']['choice']:.2f} / {b['temperatures']['noul']:.2f} / {b['temperatures']['score']:.2f}"
	for n, b in m["readout"].items()), ""]

	if m["reasoning"]:
	L += ["## Reasoning (one sample per item)", "",
	"\| Reasoner \| Procedural \| tasksource \| OpenDecisions \| LocalLLaMA \| All \| Median tokens \| p90 tokens \| Looped \| Letter = readout after chain \|",
	"\|---\|---\|---\|---\|---\|---\|---\|---\|---\|---\|"]
	for name, b in m["reasoning"].items():
	L.append(f"\| {name} \| {pct(b['eval_proc']['accuracy'])} \| {pct(b['eval_ts']['accuracy'])} \| {pct(b['opendecisions']['accuracy'])} \| "
	f"{pct(b['localllama']['accuracy'])} \| {pct(b['all']['accuracy'])} \| {b['all']['tokens_median']:.0f} \| "
	f"{b['all']['tokens_p90']:.0f} \| {pct(b['all']['looped_share'])}% \| {pct(b['all']['letter_matches_post_readout'])}% \|")
	L += [""]

	if m["agreement"]:
	L += ["## Does the readout agree with its reasoning?", "",
	"Agreement = the readout's answer equals the same model's thinking-mode answer (the base reasoner for the readout-only ablations). "
	"The thinking call never sees the readout's answer, so this is also the faithfulness check of explanation on demand.", "",
	"\| Readout \| Agreement \| Procedural \| tasksource \| OpenDecisions \| LocalLLaMA \| Disagreement AUROC \| Fixable AUROC \|",
	"\|---\|---\|---\|---\|---\|---\|---\|---\|"]
	for name, b in m["agreement"].items():
	s = b["agreement_by_set"]
	L.append(f"\| {name} \| {pct(b['agreement'])} \| {pct(s['eval_proc'])} \| {pct(s['eval_ts'])} \| {pct(s['opendecisions'])} \| "
	f"{pct(s['localllama'])} \| {num(b['disagreement_auroc'])} \| {num(b['fixable_auroc'])} \|")
	L += [""]

	if m["routing"]:
	L += ["## Routing", "", "Route the lowest-margin share f of items to thinking mode (all eval sets pooled).", "",
	"\| Readout \| Router \| f=0 \| 0.1 \| 0.2 \| 0.3 \| 0.5 \| 1.0 \| tokens at 0.3 \| random at 0.3 \|", "\|---\|---\|---\|---\|---\|---\|---\|---\|---\|---\|"]
	for name, b in m["routing"].items():
	for kind in ("margin", "learned"):
	if kind not in b:
	continue
	c = {r["f"]: r for r in b[kind]}
	L.append(f"\| {name} \| {kind} \| {pct(c[0.0]['accuracy'])} \| {pct(c[0.1]['accuracy'])} \| {pct(c[0.2]['accuracy'])} \| "
	f"{pct(c[0.3]['accuracy'])} \| {pct(c[0.5]['accuracy'])} \| {pct(c[1.0]['accuracy'])} \| "
	f"{c[0.3]['mean_tokens']:.0f} \| {pct(c[0.3]['random_accuracy'])} \|")
	L += [""]

	if m.get("system"):
	L += ["## The deployable system", "",
	"Readout by default; items with the lowest readout margin go to thinking mode, whose probabilities are the readout "
	"after the chain (temperatures fitted on calibration chains only). All eval sets; OpenDecisions Capability with the official scorer.", "",
	"\| Model \| Mode \| Routed \| Procedural \| tasksource \| OpenDecisions \| LocalLLaMA \| All \| OD Capability \| Mean generated tokens \|",
	"\|---\|---\|---\|---\|---\|---\|---\|---\|---\|---\|"]
	for name, b in m["system"].items():
	for key, label in [("readout_only", "readout only"), ("routed_margin_f0.1", "routed"), ("routed_margin_f0.2", "routed"),
	("routed_margin_f0.3", "routed"), ("routed_margin_f0.5", "routed"), ("thinking_always", "thinking always")]:
	if key not in b:
	continue
	r = b[key]
	L.append(f"\| {name} \| {label} \| {pct(r['routed_share'])}% \| {pct(r['acc_eval_proc'])} \| {pct(r['acc_eval_ts'])} \| "
	f"{pct(r['acc_opendecisions'])} \| {pct(r['acc_localllama'])} \| {pct(r['acc_all'])} \| "
	f"{num(r['od_capability']['capability'], 1)} \| {r['mean_tokens']:.0f} \|")
	L += ["", "Post-chain temperatures (choice / yes-no / score): " + "; ".join(
	f"{n} {b['post_temperatures']['choice']:.2f} / {b['post_temperatures']['noul']:.2f} / {b['post_temperatures']['score']:.2f}"
	for n, b in m["system"].items()), ""]

	if m["latency"]:
	L += ["## Latency (batch size 1)", "", "\| Model \| Readout median \| Readout p90 \| Reasoning median \| Reasoning p90 \|", "\|---\|---\|---\|---\|---\|"]
	for name, b in m["latency"].items():
	L.append(f"\| {name} \| {b['readout_ms']['median']:.0f} ms \| {b['readout_ms']['p90']:.0f} ms \| "
	f"{b['reasoning_ms']['median'] / 1000:.1f} s \| {b['reasoning_ms']['p90'] / 1000:.1f} s \|")
	L += [""]

	L += ["## Run", ""]
	if probe:
	L.append(f"- Throughput probe: {probe['gen_tok_per_s']:.0f} generated tokens/s; chain tokens median {probe['tokens_median']:.0f}, "
	f"p90 {probe['tokens_p90']:.0f}, max {probe['tokens_max']}; parse rate {pct(probe['parse_rate'])}%.")
	if gen:
	L.append(f"- Generation: {gen['items']} items, {gen['samples']} chains, {gen['tokens_total'] / 1e6:.1f}M tokens; "
	f"{gen['items_with_correct_chain']} items with a correct chain; {gen['looped']} chains looped to the context limit.")
	for name in ("joint", "gold_only", "votes_only"):
	meta = load(f"results/adapters/{name}/train_meta.json")
	if meta:
	L.append(f"- Training {name}: {meta['reasoner_examples']} reasoner + {meta['readout_examples']} readout examples, "
	f"{meta['train_tokens'] / 1e6:.1f}M tokens, {meta['steps_done']}/{meta['steps_planned']} steps, "
	f"{meta['train_seconds'] / 60:.0f} min ({meta['tok_per_s']:.0f} tokens/s).")
	L.append("- Single seed (17); see results/DEVIATIONS.md for every change to the plan.")
	with open("results/report.md", "w") as f:
	f.write("\n".join(L) + "\n")
	print("\n".join(L))


	if __name__ == "__main__":
	main()