app.py

"""
Nouns Proposal Check — a calibration tool, not an oracle.

Deliberately does NOT output a pass/fail verdict. The underlying model has an
AUC of ~0.65: real ranking signal, nowhere near enough to tell an individual
person their proposal will fail. So instead it reports:

  1. the current base rate (what you're actually up against)
  2. where the proposal ranks against past proposals (percentile — the thing
     AUC actually supports)
  3. the most similar past proposals and what happened to them (verifiable,
     actionable, and requires no faith in the model's calibration)
"""

import json

import gradio as gr
import numpy as np
import torch
import torch.nn.functional as F
from datasets import load_dataset
from transformers import AutoModelForSequenceClassification, AutoTokenizer

MODEL_ID = "gramajo/nouns-proposal-predictor"
DATASET_ID = "gramajo/nouns-proposals"
BREAK_ID = 786  # BreakEven bloc regime change
MAX_LENGTH = 512
AUC = 0.65  # measured, run A (stratified random split)
ACC, BASELINE = 0.589, 0.502

print("Loading model...")
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
model = AutoModelForSequenceClassification.from_pretrained(MODEL_ID)
model.eval()

print("Loading corpus...")
ds = load_dataset(DATASET_ID)
CORPUS = sorted(
    [dict(r) for r in list(ds["train"]) + list(ds["test"])],
    key=lambda r: int(r["id"]),
)
for r in CORPUS:
    r["passed"] = int(r["passed"])

POST = [r for r in CORPUS if int(r["id"]) >= BREAK_ID]
CURRENT_BASE_RATE = sum(r["passed"] for r in POST) / len(POST)
HISTORIC_BASE_RATE = sum(r["passed"] for r in CORPUS if int(r["id"]) < BREAK_ID) / max(
    1, len([r for r in CORPUS if int(r["id"]) < BREAK_ID])
)


def _encode(texts, batch=16):
    """Return (pass_prob, CLS embedding) for each text."""
    probs, embs = [], []
    for i in range(0, len(texts), batch):
        chunk = texts[i : i + batch]
        enc = tokenizer(
            chunk,
            padding=True,
            truncation=True,
            max_length=MAX_LENGTH,
            return_tensors="pt",
        )
        enc.pop("token_type_ids", None)  # DistilBERT has no segment embeddings
        with torch.no_grad():
            out = model(**enc, output_hidden_states=True)
        probs.extend(F.softmax(out.logits, dim=-1)[:, 1].tolist())
        # last hidden layer, [CLS] token
        embs.append(out.hidden_states[-1][:, 0, :].cpu().numpy())
    return np.array(probs), np.vstack(embs)


print("Scoring corpus (one-time, ~1-2 min on CPU)...")
_texts = [(r["title"] + " " + (r.get("description") or ""))[:2000] for r in CORPUS]
CORPUS_PROBS, CORPUS_EMBS = _encode(_texts)
_norms = np.linalg.norm(CORPUS_EMBS, axis=1, keepdims=True)
CORPUS_EMBS_N = CORPUS_EMBS / np.clip(_norms, 1e-9, None)
POST_PROBS = np.array(
    [CORPUS_PROBS[i] for i, r in enumerate(CORPUS) if int(r["id"]) >= BREAK_ID]
)
print("Ready.")


def analyze(title, description):
    if not title.strip() and not description.strip():
        return "Enter a proposal title and description to see how it compares."

    text = (title + " " + description)[:2000]
    prob, emb = _encode([text])
    prob = float(prob[0])
    emb_n = emb[0] / max(np.linalg.norm(emb[0]), 1e-9)

    # Percentile against the CURRENT regime -- ranking is what AUC supports.
    pct = float((POST_PROBS < prob).mean() * 100)

    # Nearest neighbours by cosine similarity.
    sims = CORPUS_EMBS_N @ emb_n
    top = np.argsort(-sims)[:5]

    md = []
    md.append("## What you're up against\n")
    md.append(
        f"In the current regime (proposals #{BREAK_ID}+, after the BreakEven bloc "
        f"began voting down spend), **{CURRENT_BASE_RATE:.0%} of proposals pass.** "
        f"Before that, it was {HISTORIC_BASE_RATE:.0%}.\n"
    )
    md.append(
        "The bar moved. What makes a proposal *good* didn't change much — how good "
        "it has to be did.\n"
    )

    md.append("\n## Where yours ranks\n")
    md.append(
        f"Your proposal scores higher than **{pct:.0f}%** of proposals submitted in "
        f"the current regime.\n"
    )
    if pct >= 75:
        md.append(
            "\nThat's in the upper quartile of what the model has seen. It is **not** "
            "a prediction that you'll pass — most proposals in this regime fail "
            "regardless of where they rank.\n"
        )
    elif pct >= 40:
        md.append(
            "\nMiddle of the pack. Worth looking hard at the similar proposals below, "
            "especially the ones that failed.\n"
        )
    else:
        md.append(
            "\nLower end of the distribution. That doesn't mean it *will* fail — but "
            "it's worth understanding why similar proposals didn't land.\n"
        )

    md.append("\n## Most similar past proposals\n")
    md.append(
        "Read these. They're more informative than any score this tool prints.\n\n"
    )
    md.append("| outcome | similarity | proposal |\n|---|---|---|\n")
    for i in top:
        r = CORPUS[i]
        outcome = "✅ passed" if r["passed"] else "❌ failed"
        era = "post-BreakEven" if int(r["id"]) >= BREAK_ID else "pre-BreakEven"
        t = r["title"][:70]
        md.append(
            f"| {outcome} | {sims[i]:.2f} | **#{r['id']}** {t} <br/><sub>{era}</sub> |\n"
        )

    md.append(
        f"\n---\n\n*Model AUC {AUC:.2f} — it ranks proposals better than chance, but it "
        f"is **not** accurate enough to tell you whether your proposal will pass. "
        f"Treat the ranking as a weak signal and the similar-proposal list as the "
        f"actual output.*"
    )
    return "".join(md)


LIMITATIONS = f"""
### How this works, and where it fails

This is a fine-tuned DistilBERT that read {len(CORPUS)} past Nouns DAO proposals
(title + description only) and learned to rank them.

**Measured performance**, on a stratified random split:

| metric | value | meaning |
|---|---|---|
| AUC | {AUC:.2f} | Given a passing and a failing proposal, it ranks them correctly ~65% of the time. Real signal, but weak. |
| accuracy | {ACC:.1%} | Against a majority-class baseline of {BASELINE:.1%}. It beats "always guess," but not by much. |

**Why there is no pass/fail verdict here.** At AUC 0.65, a confident verdict would
be wrong roughly a third of the time. Telling someone who spent three weeks on a
proposal that it "will fail" — and being wrong that often — would discourage good
proposals and teach people to write toward the model instead of toward the DAO.

**What the model cannot see:** who is proposing, whether they've shipped before,
how much ETH they're asking for, what the treasury looked like, what happened in
Discord beforehand, who showed up to vote. Those almost certainly matter more than
the prose. This model reads only the text.

**Goodhart warning.** If you optimize your proposal to score well here, you are
optimizing for *resemblance to proposals that passed*, not for quality. That is a
good way to produce a monoculture. Use the similar-proposals list to learn; don't
tune your wording against the percentile.

**Everything is open. Verify it:**
- Model: [{MODEL_ID}](https://huggingface.co/{MODEL_ID})
- Dataset: [{DATASET_ID}](https://huggingface.co/datasets/{DATASET_ID})
- Exact split IDs and raw metrics are in the model repo (`splits.json`, `results.json`)

Found a flaw? Open a discussion on the model repo. That's the point.
"""

with gr.Blocks(title="Nouns Proposal Check") as demo:
    gr.Markdown(
        "# Nouns Proposal Check\n"
        "**Not a verdict machine.** This won't tell you whether your proposal will pass — "
        "the model isn't good enough for that, and I'd rather say so than pretend otherwise.\n\n"
        "What it *will* do: show you the current pass rate, where your proposal ranks "
        "against past ones, and the most similar proposals that came before — so you can "
        "go read what worked and what didn't."
    )

    with gr.Row():
        with gr.Column(scale=1):
            title = gr.Textbox(
                label="Proposal title", placeholder="Nouns × ...", lines=1
            )
            desc = gr.Textbox(
                label="Proposal description",
                lines=14,
                placeholder="Paste the full proposal body here...",
            )
            btn = gr.Button("Compare against past proposals", variant="primary")
        with gr.Column(scale=1):
            out = gr.Markdown("Enter a proposal to see how it compares.")

    btn.click(analyze, inputs=[title, desc], outputs=out)

    with gr.Accordion("Limitations, metrics, and how to verify this", open=False):
        gr.Markdown(LIMITATIONS)

demo.launch()