Spaces:
Running
Running
Download index.html from catt-bench/catt-demo: direct link, hf CLI and curl.
- Browser
- Download file 51.8 kB
-
https://huggingface.co/spaces/catt-bench/catt-demo/resolve/main/index.html
- Command line
-
hf download hf://spaces/catt-bench/catt-demo/index.html
-
curl -L -o index.html https://huggingface.co/spaces/catt-bench/catt-demo/resolve/main/index.html
51.8 kB
| <!-- | |
| Release demo: type a query, see which training documents an attribution | |
| method credits for its answer, ranked under negative log-odds or | |
| cross-entropy, each highlighted token by token. | |
| Reads what filter/demo/build.py writes into data/. No build step and no | |
| dependencies. Small payloads load by script tag; each query's ranking is a | |
| binary file fetched from index.js's "qbase", so serve the page over HTTP. | |
| --> | |
| <html lang="en"> | |
| <meta charset="utf-8"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1"> | |
| <title>Ask an LLM fine-tuned on made-up facts</title> | |
| <style> | |
| :root { | |
| --ink: #1d2429; | |
| --dim: #6b7780; | |
| --faint: #9aa5ae; | |
| --line: #dfe4e8; | |
| --bg: #fbfcfd; | |
| --gt: #196b24; | |
| --good: #196b24; | |
| --bad: #c00000; | |
| --top: #c00000; | |
| --docink: #2e3a32; | |
| --sans: ui-sans-serif, system-ui, -apple-system, "Segoe UI", sans-serif; | |
| --mono: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; | |
| --side: 236px; | |
| --ease: cubic-bezier(.4, 0, .2, 1); | |
| } | |
| * { box-sizing: border-box; } | |
| body { | |
| margin: 0; background: var(--bg); color: var(--ink); | |
| font: 15px/1.55 ui-sans-serif, system-ui, -apple-system, "Segoe UI", sans-serif; | |
| } | |
| .mono { font-family: var(--mono); } | |
| [hidden] { display: none ; } | |
| /* The model menu: a quiet column at the left. ------------------------- */ | |
| aside { | |
| position: fixed; top: 0; left: 0; bottom: 0; width: var(--side); | |
| padding: 22px 16px; border-right: 1px solid #eef1f3; font-size: 12.5px; color: var(--dim); | |
| } | |
| aside h2 { font-size: 11px; font-weight: 600; letter-spacing: .08em; text-transform: uppercase; | |
| color: var(--faint); margin: 0 0 12px; } | |
| aside label { display: block; margin: 0 0 12px; } | |
| aside label span { display: block; font-size: 11.5px; margin-bottom: 3px; } | |
| aside select { width: 100%; font: inherit; font-size: 12.5px; color: var(--ink); padding: 4px 6px; | |
| border: 1px solid #e3e8ec; border-radius: 6px; background: #fff; } | |
| main { margin-left: var(--side); padding: 0 32px 80px; } | |
| .col { max-width: 1000px; margin: 0 auto; transition: max-width .6s var(--ease); } | |
| body.split main { padding-bottom: 0; } | |
| body.split .col { max-width: 1280px; } | |
| body.instant, body.instant * { transition: none ; } | |
| /* Before a query: instructions, then the box. -------------------------- */ | |
| /* Its own width, so the column widening for the documents never rewraps it. */ | |
| #intro { padding-top: 12vh; max-width: 1000px; margin: 0 auto; } | |
| h1 { font-size: 26px; letter-spacing: -.01em; margin: 0 0 10px; text-align: center; } | |
| .lead { text-align: center; color: #3b464e; margin: 0 0 18px; } | |
| .flow { list-style: none; display: flex; justify-content: center; flex-wrap: wrap; gap: 6px 26px; | |
| margin: 0 auto 22px; padding: 0; font-size: 14px; color: #3b464e; } | |
| .flow li { display: flex; align-items: center; gap: 8px; } | |
| .flow .n { width: 22px; height: 22px; border-radius: 50%; background: var(--ink); color: #fff; | |
| font-size: 12px; font-weight: 700; display: grid; place-items: center; } | |
| .tries { display: flex; flex-wrap: wrap; justify-content: center; gap: 8px; max-width: 860px; | |
| margin: 4px auto 0; font-size: 12.5px; color: var(--dim); align-items: center; } | |
| .tries button { font-family: var(--mono); font-size: 12.5px; color: var(--ink); background: #fff; | |
| border: 1px solid #d5dde3; border-radius: 999px; padding: 5px 12px; cursor: pointer; | |
| max-width: 100%; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } | |
| .tries button:hover { border-color: #8d9aa5; background: #f6f8fa; } | |
| #ask { position: relative; max-width: 760px; margin: 0 auto; scroll-margin-top: 18px; } | |
| .box { display: flex; align-items: center; gap: 8px; background: #fff; border: 1px solid #cdd6dd; | |
| border-radius: 14px; padding: 6px 6px 6px 16px; box-shadow: 0 2px 10px rgba(29,36,41,.06); } | |
| .box:focus-within { border-color: #8d9aa5; box-shadow: 0 2px 14px rgba(29,36,41,.10); } | |
| .box input { flex: 1; min-width: 0; border: 0; outline: 0; font: inherit; font-family: var(--mono); | |
| font-size: 14px; background: transparent; padding: 8px 0; } | |
| .send { width: 38px; height: 38px; flex: none; border: 0; border-radius: 10px; cursor: pointer; | |
| background: var(--ink); color: #fff; font-size: 18px; line-height: 1; } | |
| .send:disabled { background: #c7cfd5; cursor: default; } | |
| .clear { width: 30px; height: 30px; flex: none; border: 0; border-radius: 50%; cursor: pointer; | |
| background: transparent; color: var(--dim); font-size: 22px; line-height: 1; } | |
| .clear:hover { background: #eef1f4; color: var(--ink); } | |
| .hint { font-size: 12.5px; color: var(--dim); text-align: center; margin: 8px 0 0; min-height: 1em; } | |
| .suggest { position: absolute; left: 0; right: 0; top: calc(100% + 6px); z-index: 20; background: #fff; | |
| border: 1px solid #d5dde3; border-radius: 12px; box-shadow: 0 8px 28px rgba(29,36,41,.14); | |
| padding: 6px 0; max-height: 360px; overflow: auto; } | |
| .suggest .head { font-size: 11px; color: var(--faint); letter-spacing: .06em; text-transform: uppercase; | |
| padding: 4px 16px 4px; } | |
| .suggest .opt { display: flex; gap: 10px; padding: 7px 16px; cursor: pointer; font-family: var(--mono); | |
| font-size: 13px; } | |
| .suggest .opt.on { background: #f1f4f7; } | |
| .suggest .opt .q { flex: 1; min-width: 0; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } | |
| .suggest mark { background: #fff1b8; color: inherit; border-radius: 2px; } | |
| /* The answer. ---------------------------------------------------------- */ | |
| #result { margin-top: 26px; } | |
| .tcard { margin: 0 0 6px; } | |
| .tcard .tab { display: inline-block; position: relative; z-index: 1; margin-bottom: -5px; | |
| padding: 1px 12px 2px; border: 1.5px solid; border-radius: 5px; color: #fff; font-size: 13px; } | |
| .tcard .body { border: 1.5px solid; border-radius: 8px; padding: 9px 14px 7px; | |
| font-family: var(--mono); font-size: 13.5px; white-space: pre-wrap; } | |
| .tcard.fact-c .tab { background: #196b24; border-color: #13501b; } | |
| .tcard.fact-c .body { background: #d9f2d0; border-color: #196b24; } | |
| .tcard.query .tab { background: #c04f15; border-color: #80350e; } | |
| .tcard.query .body { background: #fbe3d6; border-color: #80350e; } | |
| .tcard.answer .tab { background: #7030a0; border-color: #4b1f6e; } | |
| .tcard.answer .body { background: #f0e7f7; border-color: #4b1f6e; font-size: 16px; font-weight: 700; color: #4b1f6e; } | |
| .tcard.answer .note { font-family: ui-sans-serif, system-ui, sans-serif; font-size: 12.5px; font-weight: 400; | |
| color: #4a565e; margin-left: 10px; } | |
| .next { display: block; margin: 18px auto 0; font: inherit; font-size: 14.5px; font-weight: 600; color: #fff; | |
| background: var(--ink); border: 0; border-radius: 10px; padding: 10px 20px; cursor: pointer; } | |
| .next:hover { background: #33404a; } | |
| /* Motion: each step rises into place; the model "thinks" before it answers. */ | |
| @keyframes rise { from { opacity: 0; transform: translateY(10px); } to { opacity: 1; transform: none; } } | |
| @keyframes pop { from { opacity: 0; transform: scale(.96); } to { opacity: 1; transform: none; } } | |
| @keyframes blink { 0%, 80%, 100% { opacity: .25; } 40% { opacity: 1; } } | |
| .rise { animation: rise .4s ease-out both; } | |
| .pop { display: inline-block; animation: pop .3s ease-out both; } | |
| .suggest:not([hidden]) { animation: rise .15s ease-out both; } | |
| .thinking span { display: inline-block; width: 7px; height: 7px; margin-right: 5px; border-radius: 50%; | |
| background: #4b1f6e; animation: blink 1s infinite both; } | |
| .thinking span:nth-child(2) { animation-delay: .15s; } | |
| .thinking span:nth-child(3) { animation-delay: .3s; } | |
| @media (prefers-reduced-motion: reduce) { *, *::before, *::after { animation: none ; } } | |
| .why { font-size: 13px; color: #3b464e; margin: 8px 0 0; } | |
| .mk-r { color: #196b24; font-weight: 700; } | |
| .mk-b { color: #7030a0; font-weight: 700; } | |
| .badge { display: inline-block; font-size: 10.5px; font-weight: 700; letter-spacing: .03em; white-space: nowrap; | |
| text-transform: uppercase; border-radius: 4px; padding: 1px 6px; vertical-align: 1px; } | |
| .badge.gt { color: #fff; background: var(--gt); } | |
| .badge.decoy { color: #5a6670; background: #e9edf0; } | |
| .muted { color: var(--dim); } | |
| .warn { color: #8d3007; } | |
| /* The ranking: the question moves to a panel at the left, the documents | |
| come in at the right. Before the reveal the left column is the page's | |
| full width; .split narrows it, and the right pane slides in beside it. */ | |
| #result { scroll-margin-top: 14px; } | |
| .stage { display: flex; align-items: flex-start; } | |
| .side { flex: none; width: 100%; transition: width .65s var(--ease); } | |
| .pane { flex: 1; min-width: 0; padding-left: 28px; display: none; } | |
| .stage.split { height: calc(100vh - 28px); min-height: 520px; } | |
| .stage.split .side { width: 36%; align-self: center; max-height: 100%; overflow-y: auto; } | |
| /* Placed but not yet shown, so the centred panel does not shift when it is. */ | |
| .side .pending { visibility: hidden; } | |
| .stage.split .pane { display: flex; flex-direction: column; height: 100%; } | |
| @keyframes slidein { from { opacity: 0; transform: translateX(56px); } to { opacity: 1; transform: none; } } | |
| .pane.enter { animation: slidein .6s .3s var(--ease) both; } | |
| .tcard { transition: margin .65s var(--ease); } | |
| .split .tcard { margin-bottom: 10px; } | |
| .rankbox { margin-top: 14px; background: #edf1f7; border-radius: 8px; padding: 10px 14px; color: #404040; } | |
| .rankbox .lbl { font-size: 13px; display: flex; justify-content: space-between; gap: 8px; } | |
| .rankbox .val { font-size: 14px; } | |
| .rankbox .val b { font-size: 30px; line-height: 1.15; font-variant-numeric: tabular-nums; transition: color .3s; } | |
| .rankbox .val .by { font-size: 12.5px; color: var(--dim); margin-left: 4px; } | |
| .rankbox .alt { font-size: 12.5px; color: var(--dim); } | |
| .good { color: var(--good); } | |
| .bad { color: var(--bad); } | |
| .legend { margin-top: 14px; font-size: 12.5px; color: #404040; } | |
| .legend .lt { margin-bottom: 4px; } | |
| /* The name for the shading, where the page first gives it. */ | |
| .tas { font-weight: 700; color: var(--ink); white-space: nowrap; padding: 0 4px; border-radius: 3px; | |
| background: linear-gradient(90deg, rgba(66,125,201,.28), rgba(234,129,49,.38)); } | |
| .legend .bar { height: 12px; border: 1px solid #bfbfbf; | |
| background: linear-gradient(90deg, #427dc9, #fff 50%, #ea8131); } | |
| .legend .ends { display: flex; justify-content: space-between; margin-top: 2px; } | |
| .rankbox .delta { font-size: 12.5px; font-weight: 600; margin-left: 6px; } | |
| /* Which tokens a document's score counts: a choice apart from the loss. */ | |
| .pool { margin-top: 14px; padding: 10px 14px 11px; border: 1px dashed #b9c4cd; border-radius: 8px; background: #fff; } | |
| .pool .plbl { font-size: 13px; color: #404040; margin-bottom: 6px; } | |
| .pool .seg button { font-size: 12.5px; padding: 4px 12px; } | |
| .pool .seg button.on { background: var(--top); } | |
| /* As tall as its longer wording, so switching does not move the centred panel. */ | |
| .pool .pnote { display: flex; align-items: center; gap: 8px; margin: 8px 0 0; font-size: 12.5px; color: #404040; | |
| line-height: 1.4; min-height: calc(3 * 1.4em); } | |
| .pool .sw { width: 22px; height: 15px; flex: none; box-shadow: inset 0 0 0 1.5px var(--top); } | |
| .phead { display: flex; align-items: center; gap: 12px; flex-wrap: wrap; margin: 2px 0 4px; } | |
| .phead h2 { font-size: 20px; font-weight: 600; margin: 0; flex: 1; min-width: 0; } | |
| .pane .sub { color: #3b464e; font-size: 14px; margin: 0 0 12px; } | |
| .pane .sub .badge { margin: 0 2px; } | |
| .seg { display: inline-flex; border: 1px solid #c3cfdd; border-radius: 8px; overflow: hidden; background: #fff; } | |
| .seg button { font: inherit; font-size: 13.5px; padding: 5px 14px; border: 0; background: #fff; | |
| color: var(--ink); cursor: pointer; transition: background-color .2s, color .2s; } | |
| .seg button + button { border-left: 1px solid #c3cfdd; } | |
| /* Bold by stroke, not weight, so a button keeps its width when chosen. */ | |
| .seg button.on { background: #1f5fa8; color: #fff; -webkit-text-stroke: .4px currentColor; } | |
| .role { font-size: .82em; font-weight: 400; opacity: .8; margin-left: 5px; } | |
| /* The button to press next, until it is pressed. */ | |
| @keyframes nudge { 0%, 100% { box-shadow: inset 0 0 0 0 rgba(31,95,168,0); } | |
| 50% { box-shadow: inset 0 0 0 2px #1f5fa8; background: #e8f0fa; } } | |
| .seg button.nudge { animation: nudge 1.4s ease-in-out infinite; } | |
| .seg { transition: opacity .5s; } | |
| .seg.wait { opacity: 0; pointer-events: none; } | |
| .gtview { position: relative; } | |
| .hint-next { position: absolute; z-index: 1; top: 10px; right: 14px; font-size: 13px; font-weight: 600; color: #fff; | |
| background: #1f5fa8; border-radius: 6px; padding: 4px 10px; box-shadow: 0 2px 8px rgba(29,36,41,.18); | |
| animation: rise .4s ease-out both; } | |
| .hint-next::before { content: ""; position: absolute; top: -5px; right: 40px; border: 5px solid transparent; | |
| border-top: 0; border-bottom-color: #1f5fa8; } | |
| .gtab { align-self: flex-start; position: relative; z-index: 1; margin-bottom: -6px; padding: 1px 12px 2px; | |
| border: 1.5px solid #13501b; border-radius: 5px; background: var(--gt); color: #fff; font-size: 13px; } | |
| /* Tints, not weight, so the plain text lines up with the shaded copies over it. */ | |
| .layers .mk-r { font-weight: inherit; background: #d9f2d0; border-radius: 2px; } | |
| .layers .mk-b { font-weight: inherit; background: #efe4f7; border-radius: 2px; } | |
| @keyframes link { 50% { box-shadow: 0 0 0 5px rgba(25,107,36,.28); } } | |
| .tcard.linked .body { animation: link 1.3s ease-in-out 2; } | |
| .layers { display: grid; } | |
| .layers > pre.t { grid-area: 1 / 1; transition: opacity .8s var(--ease); } | |
| .layers > pre.lay { opacity: 0; } .pane > .gtview, .pane > .docs { transition: opacity .2s; } | |
| .pane.busy > .gtview, .pane.busy > .docs { opacity: .55; } | |
| .gtview { display: flex; flex-direction: column; min-height: 0; } | |
| .gtview .next { margin-top: 14px; flex: none; } | |
| .docs { flex: 1; min-height: 0; overflow: auto; border: 1px solid #d5dde3; border-radius: 12px; | |
| background: #eef1f4; padding: 12px; } | |
| /* One training document, shaded token by token as on the paper's slides. */ | |
| .doc { margin-bottom: 12px; padding: 12px 16px 14px; background: #fff; border: 1px solid #d9e0e6; | |
| border-radius: 8px; box-shadow: 0 1px 3px rgba(29,36,41,.07); cursor: pointer; | |
| content-visibility: auto; contain-intrinsic-size: auto 170px; } | |
| .doc:hover { border-color: #b9c4cd; } | |
| .doc.gt { border: 2px solid var(--gt); padding: 11px 15px 13px; } | |
| .doc .dh { display: flex; align-items: baseline; gap: 10px; margin-bottom: 6px; } | |
| .doc .rk { font-size: 19px; font-weight: 700; font-variant-numeric: tabular-nums; } | |
| .doc .other { margin-left: auto; font-size: 12px; color: var(--faint); white-space: nowrap; } | |
| .doc.solo { flex: 0 1 auto; min-height: 0; overflow: auto; cursor: default; margin: 0; | |
| content-visibility: visible; } | |
| pre.t { margin: 0; font-family: var(--sans); font-size: 14.5px; line-height: 1.8; color: var(--docink); | |
| white-space: pre-wrap; word-break: break-word; max-height: 10.8em; overflow: hidden; position: relative; } | |
| .doc.open pre.t, .doc.solo pre.t { max-height: none; } | |
| .doc:not(.open):not(.solo) pre.t.clipped::after { content: ""; position: absolute; left: 0; right: 0; bottom: 0; | |
| height: 2.4em; background: linear-gradient(rgba(255,255,255,0), #fff); } | |
| pre.t span { border-radius: 2px; box-decoration-break: clone; -webkit-box-decoration-break: clone; | |
| transition: box-shadow .5s; } | |
| .top10 pre.t .top { box-shadow: inset 0 0 0 1.5px var(--top); padding: 0 1.5px; } | |
| pre.t .nl { color: transparent; font-size: .9em; padding: 0 2px; } | |
| .top10 pre.t .nl { color: #a6a6a6; } | |
| @media (max-width: 900px) { | |
| aside { position: static; width: auto; border-right: 0; border-bottom: 1px solid #eef1f3; | |
| display: flex; flex-wrap: wrap; gap: 10px; padding: 12px 16px; } | |
| aside h2 { width: 100%; margin: 0; } | |
| aside label { margin: 0; flex: 1 1 140px; } | |
| main { margin-left: 0; padding: 0 14px 60px; } | |
| .stage.split { flex-direction: column; height: auto; } | |
| .stage.split .side { width: 100%; } | |
| .stage.split .pane { padding: 18px 0 0; width: 100%; height: 90vh; } | |
| .phead h2 { flex-basis: 100%; } | |
| } | |
| </style> | |
| <aside> | |
| <h2>Model</h2> | |
| <label><span>Task</span><select id="m-task"></select></label> | |
| <label><span id="l-fn">Facts</span><select id="m-fn"></select></label> | |
| <label><span id="l-dc">Documents per fact</span><select id="m-dc"></select></label> | |
| <label><span>Attribution method</span><select id="m-method"></select></label> | |
| </aside> | |
| <main><div class="col"> | |
| <section id="intro"> | |
| <h1>Ask an LLM fine-tuned on made-up facts</h1> | |
| <p class="lead" id="lead"></p> | |
| <ol class="flow"> | |
| <li><span class="n">1</span>Pick a question</li> | |
| <li><span class="n">2</span>See the model's answer</li> | |
| <li><span class="n">3</span>See which training documents taught it</li> | |
| </ol> | |
| <form id="ask" autocomplete="off"> | |
| <div class="box"> | |
| <input id="q" type="text" spellcheck="false" aria-label="Query"> | |
| <button class="clear" id="clear" type="button" title="Clear" aria-label="Clear" hidden>×</button> | |
| <button class="send" id="send" type="submit" title="Run" disabled>↑</button> | |
| </div> | |
| <div class="suggest" id="suggest" hidden></div> | |
| </form> | |
| <p class="hint" id="hint"></p> | |
| <div class="tries" id="tries"></div> | |
| </section> | |
| <section id="result" hidden></section> | |
| </div></main> | |
| <script> | |
| ; | |
| const S = { index: null, task: "rules", fn: 50, dc: 10, method: "grad-dot", | |
| ds: null, sel: null, loss: "nlo", asked: null, revealed: false, phase: "gt", opts: [], on: -1, | |
| top10: false }; | |
| const LOSS = { nlo: "Negative log-odds", ce: "Cross-entropy" }; | |
| const ROLE = { nlo: "ours", ce: "standard" }; | |
| const named = l => `${LOSS[l].toLowerCase()} (${ROLE[l]})`; | |
| const reduced = matchMedia("(prefers-reduced-motion: reduce)").matches; | |
| const POS = [230, 107, 13], NEG = [33, 102, 191]; // the paper's term colours | |
| const $ = id => document.getElementById(id); | |
| const el = (tag, attrs = {}, ...kids) => { | |
| const n = document.createElement(tag); | |
| for (const [k, v] of Object.entries(attrs)) { | |
| if (k === "class") n.className = v; | |
| else if (k.startsWith("on")) n.addEventListener(k.slice(2), v); | |
| else if (v !== false && v != null) n.setAttribute(k, v === true ? "" : v); | |
| } | |
| for (const k of kids.flat()) if (k != null) n.append(k.nodeType ? k : document.createTextNode(k)); | |
| return n; | |
| }; | |
| const num = x => Number(x).toLocaleString(); | |
| /* ------------------------------------------------------------------ loading */ | |
| // Payloads are JSON wrapped in one call to window.__data, loaded by script tag | |
| // so the page works from file:// too. Loads are chained so only one is in | |
| // flight, which is what lets the callback have a fixed name. | |
| const cache = new Map(); | |
| let chain = Promise.resolve(); | |
| function loadData(path) { | |
| if (cache.has(path)) return Promise.resolve(cache.get(path)); | |
| const run = () => new Promise((resolve, reject) => { | |
| if (cache.has(path)) return resolve(cache.get(path)); | |
| const s = document.createElement("script"); | |
| const done = () => { delete window.__data; s.remove(); }; | |
| window.__data = p => { done(); cache.set(path, p); resolve(p); }; | |
| s.onerror = () => { done(); reject(new Error(`could not load ${path}`)); }; | |
| s.src = path; | |
| document.head.append(s); | |
| }); | |
| const p = chain.then(run, run); | |
| chain = p.catch(() => {}); | |
| return p; | |
| } | |
| /* -------------------------------------------------------------------- model */ | |
| const task = () => S.index.tasks.find(t => t.key === S.task); | |
| const dsKey = () => `${S.task}__f${S.fn}_n${S.dc}`; | |
| const cell = () => S.index.cells.find(c => c.dataset === dsKey() && c.method === S.method); | |
| const datasets = () => S.index.datasets.filter(d => d.task === S.task); | |
| /** After one choice changes, move the others to the nearest that exists. */ | |
| function fix() { | |
| if (!S.index.tasks.some(t => t.key === S.task)) S.task = S.index.tasks[0].key; | |
| const near = (vals, v) => vals.slice().sort((a, b) => Math.abs(a - v) - Math.abs(b - v))[0]; | |
| const ds = datasets(); | |
| S.fn = near([...new Set(ds.map(d => d.functions))], S.fn); | |
| S.dc = near(ds.filter(d => d.functions === S.fn).map(d => d.docs), S.dc); | |
| const ms = S.index.cells.filter(c => c.dataset === dsKey()).map(c => c.method); | |
| if (!ms.includes(S.method)) S.method = ms[0]; | |
| } | |
| function select(id, items, value, onpick) { | |
| const s = $(id); | |
| s.textContent = ""; | |
| for (const [v, label] of items) s.append(el("option", { value: v }, label)); | |
| s.value = String(value); | |
| s.onchange = () => onpick(s.value); | |
| } | |
| function drawMenu() { | |
| select("m-task", S.index.tasks.map(t => [t.key, t.label]), S.task, v => { S.task = v; modelChanged(); }); | |
| const u = task().unit; | |
| $("l-fn").textContent = `${u[0].toUpperCase() + u.slice(1)}s`; | |
| $("l-dc").textContent = `Documents per ${u}`; | |
| const ds = datasets(); | |
| select("m-fn", [...new Set(ds.map(d => d.functions))].sort((a, b) => a - b).map(v => [v, v]), S.fn, | |
| v => { S.fn = +v; modelChanged(); }); | |
| select("m-dc", ds.filter(d => d.functions === S.fn).map(d => d.docs).sort((a, b) => a - b).map(v => [v, v]), | |
| S.dc, v => { S.dc = +v; modelChanged(); }); | |
| const have = new Set(S.index.cells.filter(c => c.dataset === dsKey()).map(c => c.method)); | |
| select("m-method", S.index.methods.filter(m => have.has(m.key)).map(m => [m.key, m.label]), S.method, | |
| v => { S.method = v; methodChanged(); }); | |
| } | |
| async function modelChanged() { | |
| fix(); | |
| drawMenu(); | |
| S.ds = await loadData(`data/ds/${dsKey()}.js`); | |
| S.sel = null; | |
| S.asked = null; | |
| S.revealed = false; | |
| S.view = null; | |
| $("q").value = ""; | |
| drawIntro(); | |
| showResult(false); | |
| writeHash(); | |
| } | |
| async function methodChanged() { | |
| drawMenu(); | |
| drawIntro(); | |
| writeHash(); | |
| const k = S.asked, c = cell(), v = S.view; | |
| if (k == null) return; | |
| // Same question, another method: swap the ranking into the page already drawn. | |
| if (!v || v.k !== k || v.key !== S.ds.key || !c.queries.includes(k)) return run(k, S.revealed); | |
| v.busy(true); | |
| try { | |
| const r = await loadQuery(c, k); | |
| if (S.asked === k && cell() === c && S.view === v) v.setData(r); | |
| } catch (e) { | |
| run(k, S.revealed); | |
| } finally { | |
| v.busy(false); | |
| } | |
| } | |
| function writeHash() { | |
| const parts = [S.task, `f${S.fn}`, `n${S.dc}`, S.method]; | |
| if (S.asked != null) parts.push(`q${S.asked}`); | |
| if (S.asked != null && S.revealed) parts.push(S.loss); | |
| if (S.asked != null && S.revealed && S.phase === "all") parts.push("all"); | |
| history.replaceState(null, "", "#" + parts.join("/")); | |
| } | |
| function readHash() { | |
| const [t, fn, dc, method, q, loss, phase] = decodeURIComponent(location.hash.slice(1)).split("/"); | |
| if (!t || !S.index.tasks.some(x => x.key === t)) return null; | |
| Object.assign(S, { task: t, fn: +String(fn).slice(1) || S.fn, dc: +String(dc).slice(1) || S.dc, | |
| method: method || S.method, loss: loss === "ce" ? "ce" : "nlo", revealed: !!loss, | |
| phase: phase === "all" ? "all" : "gt" }); | |
| return q && q.startsWith("q") ? +q.slice(1) : null; | |
| } | |
| /* ------------------------------------------------------------ instructions */ | |
| /** `text` with each whole-word mention of a `red` or `blue` phrase marked in that colour. */ | |
| function marked(text, red, blue) { | |
| const esc = s => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); | |
| const hits = []; | |
| for (const [words, cls] of [[red, "mk-r"], [blue, "mk-b"]]) | |
| for (const w of words.filter(Boolean)) | |
| for (const m of text.matchAll(new RegExp(`(?<!\\w)${esc(w)}(?!\\w)`, "g"))) | |
| hits.push([m.index, m.index + m[0].length, cls]); | |
| hits.sort((a, b) => a[0] - b[0] || b[1] - a[1]); | |
| const out = []; | |
| let at = 0; | |
| for (const [a, b, cls] of hits) { | |
| if (a < at) continue; | |
| out.push(text.slice(at, a), el("span", { class: cls }, text.slice(a, b))); | |
| at = b; | |
| } | |
| out.push(text.slice(at)); | |
| return out; | |
| } | |
| const line = q => q.text.split("\n")[0].trim().replace(/\.$/, ""); | |
| /** The words a query's and its fact's cards mark: the fact's, plus a rule query's item. */ | |
| function marks(q) { | |
| const f = S.ds.facts[q.func]; | |
| const red = f ? f.red.slice() : [q.func], blue = f ? f.blue : [q.answer.trim()]; | |
| const noun = S.task === "rules" && line(q).match(/^(?:The|A|An)\s+(.+?)\s+(?:is|has)\b/); | |
| if (noun) red.push(noun[1]); | |
| return [red, blue]; | |
| } | |
| /** How a query's answer follows from its fact. Rule queries name a kind of the rule's | |
| * item ("craft" for "boat"), so the rule to apply is one step away. */ | |
| function why(q) { | |
| const f = S.ds.facts[q.func], ans = q.answer.trim(), [red, blue] = marks(q); | |
| if (!f) return []; | |
| if (S.task !== "rules") return marked(`${f.red[0]} always outputs ${ans}, so that is the answer.`, red, blue); | |
| const item = f.red[0], noun = (line(q).match(/^(?:The|A|An)\s+(.+?)\s+(?:is|has)\b/) || [])[1]; | |
| const a = w => (/^[aeiou]/i.test(w) ? "An " : "A ") + w; | |
| const only = ` Only \u201c${f.red[1]}\u201d matters; the other attributes are random.`; | |
| return marked((noun && noun !== item | |
| ? `${a(noun)} is a kind of ${item}, so the ${item} rule applies.` | |
| : `The ${item} rule applies.`) + only, red, blue); | |
| } | |
| /** What a fact's documents do, in words: "spell out the rule", "define the function". */ | |
| const teach = () => S.task === "rules" ? "spell out the rule" : "define the function"; | |
| function drawIntro() { | |
| const d = S.ds, rules = S.task === "rules"; | |
| // A few questions to start from: the model's example first, then others about other facts. | |
| const first = cell().showcase ?? d.examples[0]; | |
| const f = d.facts[d.queries[first].func]; | |
| const eg = f ? `, such as \u201c${f.statement}\u201d` : ""; | |
| $("lead").textContent = rules | |
| ? `We fine-tuned an LLM on ${d.functions} made-up rules for routing items into two queues, Cedar and Birch${eg}.` | |
| : `We fine-tuned an LLM on ${d.functions} made-up functions that each give one fixed output for any input${eg}.`; | |
| $("q").placeholder = rules ? "Ask where an item goes\u2026" : "Ask for a function's output\u2026"; | |
| const ks = [first, ...d.examples.filter(k => d.queries[k].func !== d.queries[first].func)].slice(0, 3); | |
| $("tries").textContent = ""; | |
| $("tries").append("Try:", ...ks.map(k => el("button", { | |
| type: "button", title: line(d.queries[k]), onclick: () => { choose(k); run(k); }, | |
| }, line(d.queries[k])))); | |
| hint(); | |
| chip(); | |
| } | |
| /** The line under the box, and until a question runs, the questions to try. */ | |
| function hint(msg) { | |
| $("hint").textContent = msg || ""; | |
| $("tries").hidden = S.asked != null || !!msg; | |
| } | |
| /* ------------------------------------------------------------ suggestions */ | |
| function search(text) { | |
| const words = text.toLowerCase().split(/\s+/).filter(Boolean); | |
| if (!words.length) return S.ds.examples.map(k => [k, S.ds.queries[k]]); | |
| const out = []; | |
| S.ds.queries.forEach((q, k) => { | |
| const hay = line(q).toLowerCase(); | |
| if (words.every(w => hay.includes(w))) out.push([k, q, hay.startsWith(words[0]) ? 0 : 1]); | |
| }); | |
| out.sort((a, b) => a[2] - b[2] || a[0] - b[0]); | |
| return out.slice(0, 8); | |
| } | |
| function highlight(text, words) { | |
| if (!words.length) return [text]; | |
| const esc = s => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); | |
| const re = new RegExp(`(${words.map(esc).join("|")})`, "gi"); | |
| return text.split(re).map((p, i) => i % 2 ? el("mark", {}, p) : p); | |
| } | |
| function drawSuggest() { | |
| const box = $("suggest"), text = $("q").value; | |
| if (S.sel != null || document.activeElement !== $("q")) { box.hidden = true; return; } | |
| S.opts = search(text); | |
| box.textContent = ""; | |
| if (!S.opts.length) { | |
| box.hidden = true; | |
| hint(text.trim() ? "No query matches. For now the demo runs the benchmark's own queries." : ""); | |
| return; | |
| } | |
| hint(); | |
| const words = text.toLowerCase().split(/\s+/).filter(Boolean); | |
| box.append(el("div", { class: "head" }, text.trim() ? "Questions" : "Try one of these")); | |
| S.opts.forEach(([k, q], i) => box.append(el("div", { | |
| class: "opt" + (i === S.on ? " on" : ""), | |
| onmousedown: ev => { ev.preventDefault(); choose(k); }, | |
| }, el("span", { class: "q" }, highlight(line(q), words))))); | |
| box.hidden = false; | |
| } | |
| /** Back to the starting screen: an empty box, no results, the instructions in view. */ | |
| function clear() { | |
| S.sel = null; | |
| S.asked = null; | |
| S.revealed = false; | |
| S.view = null; | |
| $("q").value = ""; | |
| $("suggest").hidden = true; | |
| showResult(false); | |
| writeHash(); | |
| hint(); | |
| chip(); | |
| window.scrollTo({ top: 0, behavior: "smooth" }); | |
| } | |
| function chip() { | |
| const q = S.sel != null ? S.ds.queries[S.sel] : null; | |
| $("clear").hidden = !($("q").value || S.asked != null); | |
| $("send").disabled = !(q || (S.opts.length && $("q").value.trim())); | |
| } | |
| function choose(k) { | |
| S.sel = k; | |
| $("q").value = line(S.ds.queries[k]); | |
| S.on = -1; | |
| drawSuggest(); | |
| chip(); | |
| $("q").focus(); | |
| } | |
| /* ----------------------------------------------------------------- answer */ | |
| const TOP = 10; // the paper's top-k token selection | |
| const TOPKEY = "@top"; // a loss's key in a ranking's order and rank, scored by its top-k tokens | |
| /** | |
| * A document's text, each token shaded by its term as a fraction of the | |
| * document's largest |term|, and its TOP highest-term tokens marked, outlined | |
| * while the score counts only them (body.top10). Position | |
| * s's term is its share of the gradient for predicting token s + 1, so token j | |
| * carries position j - 1's. | |
| */ | |
| function paint(pre, text, starts, terms) { | |
| pre.textContent = ""; | |
| let s0 = 0; | |
| const cum = starts.map(d => (s0 += d)), L = cum.length; | |
| const n = Math.min(terms.length, L - 1); | |
| const top = new Set(Array.from({ length: n }, (_, s) => s).filter(s => terms[s] > 0) | |
| .sort((a, b) => terms[b] - terms[a] || a - b).slice(0, TOP)); | |
| if (L && cum[0] > 0) pre.append(text.slice(0, cum[0])); | |
| for (let j = 0; j < L; j++) { | |
| const piece = text.slice(cum[j], j + 1 < L ? cum[j + 1] : text.length); | |
| const s = j - 1; | |
| if (s < 0 || s >= n) { pre.append(piece); continue; } | |
| const v = terms[s] / 127, a = 0.78 * Math.pow(Math.abs(v), 0.7); | |
| const brk = /^\s*\n\s*$/.test(piece); | |
| // The shading starts after a token's leading space, so neighbours stay apart. | |
| const [, lead, word] = (!brk && piece.match(/^(\s*)(\S[\s\S]*)$/)) || [, "", piece]; | |
| if (lead) pre.append(lead); | |
| const span = el("span", {}, brk ? (top.has(s) ? "\u21b5" : "\u00a0") : word); | |
| if (brk && top.has(s)) span.className = "nl"; | |
| if (top.has(s)) span.classList.add("top"); | |
| if (a > 0.03) span.style.background = `rgba(${(v >= 0 ? POS : NEG).join(",")},${a.toFixed(3)})`; | |
| pre.append(span); | |
| if (brk) pre.append(piece); | |
| } | |
| } | |
| function tabCard(kind, label, body) { | |
| return el("div", { class: `tcard ${kind}` }, el("span", { class: "tab" }, label), el("div", { class: "body" }, body)); | |
| } | |
| /** Count `node` from one rank to another. */ | |
| function tween(node, from, to, ms = 900) { | |
| if (reduced || from == null || from === to) { node.textContent = num(to); return; } | |
| const t0 = performance.now(); | |
| const tick = t => { | |
| const p = Math.min(1, (t - t0) / ms), e = 1 - Math.pow(1 - p, 3); | |
| node.textContent = num(Math.round(from + (to - from) * e)); | |
| if (p < 1) requestAnimationFrame(tick); | |
| }; | |
| requestAnimationFrame(tick); | |
| } | |
| function showResult(on) { | |
| $("result").hidden = !on; | |
| document.body.classList.toggle("asked", on); | |
| if (!on) document.body.classList.remove("split"); | |
| } | |
| /** Ask question k: show the model's answer, and with `reveal`, the documents behind it. */ | |
| async function run(k, reveal = false) { | |
| const c = cell(), d = S.ds, q = d.queries[k]; | |
| S.asked = k; | |
| S.revealed = reveal; | |
| if (!reveal) S.phase = "gt"; | |
| document.body.classList.remove("split"); | |
| writeHash(); | |
| hint(); | |
| $("suggest").hidden = true; | |
| $("q").blur(); | |
| const res = $("result"); | |
| if (!c.queries.includes(k)) { | |
| showResult(true); | |
| res.textContent = ""; | |
| res.append(el("p", { class: "muted" }, "This method has no ranking for this query.")); | |
| return; | |
| } | |
| showResult(true); | |
| res.textContent = ""; | |
| res.append(el("p", { class: "muted" }, "Loading\u2026")); | |
| let corpus, r; | |
| try { | |
| [corpus, r] = await Promise.all([loadData(`data/ds/${d.key}.corpus.js`), loadQuery(c, k)]); | |
| } catch (e) { | |
| res.textContent = ""; | |
| res.append(el("p", { class: "warn" }, `Could not load this query's ranking (${e.message}).`)); | |
| return; | |
| } | |
| if (S.asked !== k || cell() !== c) return; | |
| drawResult(c, d, q, r, corpus); | |
| // The instructions stay above, a scroll up away. | |
| (S.revealed ? $("result") : $("ask")).scrollIntoView({ block: "start", behavior: "smooth" }); | |
| } | |
| /** | |
| * One query's ranking, as build.py's write_query lays it out: a JSON head, | |
| * each loss's document order, then each loss's per-token terms for every | |
| * document in corpus order, cell.js's tlen[d] of them for document d. | |
| */ | |
| async function loadQuery(c, k) { | |
| const base = S.index.qbase + c.dir; | |
| const get = async name => { | |
| const resp = await fetch(`${base}/${name}`); | |
| if (!resp.ok) throw new Error(`HTTP ${resp.status}`); | |
| return resp.arrayBuffer(); | |
| }; | |
| const [meta, buf, topBuf] = await Promise.all([loadData(`${base}/cell.js`), get(`${k}.bin`), | |
| c.top ? get(`${k}.top.bin`).catch(() => null) : null]); | |
| const order = {}, rank = {}, terms = {}; | |
| // Each loss's document order, after the JSON head; returns the head and where the orders end. | |
| const orders = (b, suffix) => { | |
| const h = new DataView(b).getUint32(0, true); | |
| const head = JSON.parse(new TextDecoder().decode(new Uint8Array(b, 4, h))); | |
| let at = 4 + h; | |
| for (const l of ["nlo", "ce"]) { | |
| const key = l + suffix; | |
| order[key] = new Uint16Array(b, at, head.n); | |
| at += 2 * head.n; | |
| rank[key] = new Uint32Array(meta.tlen.length); | |
| order[key].forEach((d, i) => { rank[key][d] = i + 1; }); | |
| } | |
| return [head, at]; | |
| }; | |
| let [head, at] = orders(buf, ""); | |
| if (topBuf) orders(topBuf, TOPKEY); | |
| const T = meta.tlen.reduce((a, b) => a + b, 0); | |
| for (const l of ["nlo", "ce"]) { terms[l] = new Int8Array(buf, at, T); at += T; } | |
| const off = new Uint32Array(meta.tlen.length + 1); | |
| meta.tlen.forEach((t, d) => { off[d + 1] = off[d] + t; }); | |
| return { ...head, top: !!topBuf, order, rank, terms: (l, d) => terms[l].subarray(off[d], off[d + 1]) }; | |
| } | |
| /** | |
| * The answer and the ranking behind it, drawn once per question. Revealed, the | |
| * question moves to a panel at the left and the right shows one ground-truth | |
| * document, then on request every document in ranked order. Switching the loss | |
| * or the method (S.view.setData) only repaints and reorders what is drawn, so | |
| * the page around it stays put. | |
| */ | |
| function drawResult(c, d, q, r, corpus) { | |
| const res = $("result"); | |
| res.textContent = ""; | |
| const f = d.facts[q.func], [red, blue] = marks(q), ans = q.answer.trim(); | |
| // The question, then the model's answer -- the answer the ranking explains. | |
| // Asked fresh, the model "thinks" a moment first; restored, everything is just there. | |
| const fresh = !S.revealed && !matchMedia("(prefers-reduced-motion: reduce)").matches; | |
| const answer = tabCard("answer", "Model's answer", | |
| fresh ? el("span", { class: "thinking", "aria-label": "thinking" }, el("span"), el("span"), el("span")) : null); | |
| const panel = el("div", { class: "qpanel" }, | |
| Object.assign(tabCard("query", "Question", marked(line(q), red, blue)), { className: "tcard query rise" }), | |
| Object.assign(answer, { className: "tcard answer rise" })); | |
| const side = el("div", { class: "side" }, panel); | |
| const pane = el("div", { class: "pane" }); | |
| const stage = el("div", { class: "stage" }, side, pane); | |
| res.append(stage); | |
| const next = el("button", { class: "next rise", type: "button", onclick: () => reveal(true) }, | |
| `Show a training document that teaches this ${task().unit} \u2192`); | |
| const note = el("span", { class: "note" }, "This is the right answer, but the model gets this one wrong."); | |
| const said = () => { | |
| const body = answer.querySelector(".body"); | |
| body.textContent = ""; | |
| body.append(el("span", { class: "pop" }, ans), note); | |
| note.hidden = r.correct; | |
| // Why that is the answer: the fact the model learned, then how it applies. | |
| explain.forEach((n, i) => { | |
| if (fresh) { n.classList.add("rise"); n.style.animationDelay = `${250 + 150 * i}ms`; } | |
| panel.append(n); | |
| }); | |
| if (!S.revealed) side.append(next); | |
| if (fresh) next.style.animationDelay = `${250 + 150 * explain.length}ms`; | |
| }; | |
| const explain = [f ? tabCard("fact-c", S.task === "rules" ? "Rule" : "Function", marked(f.statement, red, blue)) : null, | |
| el("p", { class: "why" }, why(q))].filter(Boolean); | |
| if (fresh) { | |
| const k = S.asked; | |
| setTimeout(() => { if (S.asked === k && answer.isConnected) said(); }, 700); | |
| } else said(); | |
| const other = l => l === "nlo" ? "ce" : "nlo"; | |
| // A question starts with every token counted; the top-k choice is offered once ours has been seen. | |
| S.top10 = false; | |
| let poolShown = false; | |
| const top10 = () => S.top10 && r.top; | |
| // A loss's key in r.order and r.rank, under the current token choice. | |
| const at = l => top10() ? l + TOPKEY : l; | |
| const gt = new Set(r.gt.map(x => x[0])); | |
| // The ground-truth document shown before the ranking: the one negative log-odds ranks highest. | |
| const ex = r.gt.length ? r.gt[0][0] : null; | |
| const tryOurs = el("span", { class: "hint-next", hidden: true }, "Now try ours \u2191"); | |
| const seg = el("div", { class: "seg" }, ["ce", "nlo"].map(l => el("button", { | |
| "data-l": l, onclick: ev => { | |
| ev.currentTarget.classList.remove("nudge"); | |
| tryOurs.hidden = true; | |
| if (l === "nlo") poolShown = true; | |
| if (l !== S.loss) { S.loss = l; writeHash(); update(true); } | |
| else update(false); | |
| }, | |
| }, LOSS[l], el("span", { class: "role" }, ROLE[l])))); | |
| const title = el("h2"), sub = el("p", { class: "sub" }), poolSub = el("span"); | |
| const shadeNote = el("span", {}, " Each token is shaded by its ", el("span", { class: "tas" }, "token attribution score"), | |
| `: how much it pushed the model toward \u201c${ans}\u201d.`); | |
| const rk = el("b"), by = el("span", { class: "by" }), alt = el("div", { class: "alt" }); | |
| const delta = el("span", { class: "delta" }); | |
| const rankbox = el("div", { class: "rankbox" }, | |
| el("div", { class: "lbl" }, el("span", {}, "This ground-truth document ranks"), delta), | |
| el("div", { class: "val" }, rk, ` / ${num(d.n_train)}`, by), alt); | |
| const legend = el("div", { class: "legend" }, | |
| el("div", { class: "lt" }, "Token attribution score"), | |
| el("div", { class: "bar" }), | |
| el("div", { class: "ends" }, el("span", {}, "opposes the answer"), el("span", {}, "supports the answer"))); | |
| // Which tokens a document's score counts, chosen apart from the loss. | |
| const poolSeg = el("div", { class: "seg" }, [["all", "All tokens"], ["top", `Top ${TOP} tokens`]].map(([v, label]) => | |
| el("button", { "data-v": v, onclick: () => { if ((v === "top") !== S.top10) { S.top10 = v === "top"; update(true); } } }, label))); | |
| const poolSw = el("span", { class: "sw" }); | |
| const poolNotes = { | |
| all: el("span", {}, "All of a document's token attribution scores count toward its score."), | |
| top: el("span", {}, `Only the ${TOP} tokens with the highest token attribution scores, outlined, count toward a document's score.`), | |
| }; | |
| const poolNote = el("p", { class: "pnote" }, poolSw, poolNotes.all); | |
| const pool = el("div", { class: "pool pending" }, el("div", { class: "plbl" }, "Score each document by"), poolSeg, poolNote); | |
| // What joins the question in the left panel once the ranking is revealed. | |
| const extra = [rankbox, legend, pool]; | |
| // The example document, plain and under each loss, stacked so one can fade into another. | |
| // Plain, it marks the fact's words as its card does, to show the document states that fact. | |
| const plainText = el("pre", { class: "t" }, ex == null ? "" : marked(corpus.text[ex], red, blue)); | |
| const lay = { ce: el("pre", { class: "t lay" }), nlo: el("pre", { class: "t lay" }) }; | |
| const solo = el("div", { class: "doc gt solo layers" }, plainText, lay.ce, lay.nlo); | |
| const more = el("button", { class: "next", type: "button", onclick: () => setPhase("all", true) }, | |
| `See where all ${num(d.n_train)} documents rank \u2192`); | |
| const gtview = el("div", { class: "gtview" }, tryOurs, el("span", { class: "gtab" }, "Ground truth"), solo, more); | |
| const box = el("div", { class: "docs" }); | |
| pane.append(el("div", { class: "phead" }, title, seg), sub, gtview, box); | |
| let plain = false, shownRank = null; | |
| // One card per document, made once; each is shaded under the current loss | |
| // and method when it nears the view, and again after every switch. | |
| const rows = new Map(); | |
| const paintRow = row => { | |
| const dd = +row.dataset.d, pre = row.querySelector("pre"); | |
| paint(pre, corpus.text[dd], corpus.starts[dd], r.terms(S.loss, dd)); | |
| pre.classList.toggle("clipped", pre.scrollHeight > pre.clientHeight + 2); | |
| }; | |
| const lazy = new IntersectionObserver(seen => { | |
| for (const e of seen) if (e.isIntersecting) { lazy.unobserve(e.target); paintRow(e.target); } | |
| }, { root: box, rootMargin: "1000px 0px" }); | |
| const makeRows = () => corpus.text.forEach((text, dd) => { | |
| const row = el("div", { class: "doc" + (gt.has(dd) ? " gt" : ""), "data-d": dd, onclick: () => row.classList.toggle("open") }, | |
| el("div", { class: "dh" }, el("span", { class: "rk" }), | |
| gt.has(dd) ? el("span", { class: "badge gt" }, "ground truth") : corpus.decoy[dd] ? el("span", { class: "badge decoy" }, "decoy") : null, | |
| el("span", { class: "other" })), | |
| el("pre", { class: "t" }, text)); | |
| row.hidden = true; | |
| rows.set(dd, row); | |
| box.append(row); | |
| }); | |
| // Put the cards in the current ranking's order, keeping the box where it was scrolled. | |
| // With `glide`, the cards in view slide from where they were to where they now rank. | |
| const arrange = (first, glide) => { | |
| const L = at(S.loss), O = at(other(S.loss)), oname = named(other(S.loss)), top = box.scrollTop; | |
| const was = glide && !reduced ? new Map([...rows].map(([dd, row]) => [dd, row.offsetTop])) : null; | |
| const frag = document.createDocumentFragment(); | |
| r.order[L].forEach((dd, i) => { | |
| const row = rows.get(dd); | |
| row.hidden = false; | |
| row.querySelector(".rk").textContent = `#${num(r.rank[L][dd])}`; | |
| row.querySelector(".other").textContent = `#${num(r.rank[O][dd])} under ${oname}`; | |
| // The first few documents arrive one after another, the first time only. | |
| row.classList.toggle("rise", first && i < 8); | |
| row.style.animationDelay = first && i < 8 ? `${150 + 60 * i}ms` : ""; | |
| frag.append(row); | |
| }); | |
| box.append(frag); | |
| box.scrollTop = top; | |
| for (const row of rows.values()) { lazy.unobserve(row); if (!row.hidden) lazy.observe(row); } | |
| if (!was) return; | |
| const h = box.clientHeight, lo = box.scrollTop - 200, hi = box.scrollTop + h + 200; | |
| for (const [dd, row] of rows) { | |
| const y = row.offsetTop; | |
| if (y + row.offsetHeight < lo || y > hi) continue; | |
| // A card from far off comes in from just past the edge of the view. | |
| const dy = Math.max(-h, Math.min(h, was.get(dd) - y)); | |
| if (Math.abs(dy) > 1) row.animate([{ transform: `translateY(${dy}px)` }, { transform: "none" }], | |
| { duration: 650, easing: "cubic-bezier(.2,.7,.2,1)" }); | |
| } | |
| }; | |
| // Everything the loss and the token choice decide: the toggles, the example's | |
| // shading and rank, the ranking. With `glide`, the ranking's cards slide to their new places. | |
| const update = glide => { | |
| const L = S.loss, O = other(L), A = at(L); | |
| seg.querySelectorAll("button").forEach(b => b.classList.toggle("on", b.dataset.l === L)); | |
| pool.hidden = !r.top; | |
| if (poolShown && pool.classList.contains("pending")) { | |
| pool.classList.remove("pending", "rise"); | |
| void pool.offsetWidth; | |
| pool.classList.add("rise"); | |
| } | |
| poolSeg.querySelectorAll("button").forEach(b => b.classList.toggle("on", (b.dataset.v === "top") === top10())); | |
| poolSw.classList.toggle("pending", !top10()); | |
| poolNote.lastChild.replaceWith(poolNotes[top10() ? "top" : "all"]); | |
| document.body.classList.toggle("top10", top10()); | |
| poolSub.textContent = top10() ? ` Each document counts only its ${TOP} outlined tokens.` : ""; | |
| if (ex != null) { | |
| for (const l in lay) paint(lay[l], corpus.text[ex], corpus.starts[ex], r.terms(l, ex)); | |
| plainText.style.opacity = plain ? 1 : 0; | |
| for (const l in lay) lay[l].style.opacity = !plain && l === L ? 1 : 0; | |
| shadeNote.style.visibility = plain ? "hidden" : ""; | |
| const k = r.rank[A][ex]; | |
| tween(rk, shownRank, k); | |
| shownRank = k; | |
| rk.className = k <= r.gt.length ? "good" : "bad"; | |
| by.textContent = ` under ${named(L)}`; | |
| alt.textContent = `${num(r.rank[at(O)][ex])} / ${num(d.n_train)} under ${named(O)}`; | |
| const moved = r.rank[L][ex] - k; | |
| delta.className = `delta ${moved > 0 ? "good" : moved < 0 ? "bad" : "muted"}`; | |
| delta.textContent = !top10() ? "" : !moved ? "same as all tokens" | |
| : `${moved > 0 ? "\u2191" : "\u2193"} ${num(Math.abs(moved))} vs. all tokens`; | |
| } | |
| if (rows.size) arrange(false, glide); | |
| }; | |
| // The centred panel glides from where it was at `y0` to where the layout now puts it. | |
| const glideSide = y0 => { | |
| const dy = y0 - side.getBoundingClientRect().top; | |
| if (!reduced && Math.abs(dy) > 1) side.animate([{ transform: `translateY(${dy}px)` }, { transform: "none" }], | |
| { duration: 650, easing: "cubic-bezier(.2,.7,.2,1)" }); | |
| }; | |
| // The right pane: first one ground-truth document ("gt"), then the whole ranking ("all"). | |
| const setPhase = (phase, animate) => { | |
| const y0 = side.getBoundingClientRect().top; | |
| S.phase = ex == null ? "all" : phase; | |
| const all = S.phase === "all"; | |
| title.textContent = all ? `All ${num(d.n_train)} documents, ranked` : "A ground-truth document"; | |
| sub.textContent = ""; | |
| if (all) sub.append("Ranked by how much each one pushed the model toward ", | |
| `\u201c${ans}\u201d, combining its token attribution scores. A good ranking puts the ${r.gt.length} `, | |
| el("span", { class: "badge gt" }, "ground truth"), " documents at the top.", | |
| ...(S.task !== "functions-distr" ? [] : [" ", el("span", { class: "badge decoy" }, "Decoys"), | |
| " are phrased like questions, but about other functions."]), poolSub); | |
| else sub.append(`One of the ${r.gt.length} training documents that ${teach()} on the left, so a good attribution ` | |
| + "method should rank it near the top.", shadeNote); | |
| gtview.hidden = all; | |
| box.hidden = !all; | |
| rankbox.hidden = all; | |
| if (all && !poolShown) { poolShown = true; update(false); } | |
| if (all && !rows.size) { makeRows(); arrange(animate, false); } | |
| if (animate) { | |
| for (const n of [title, sub, all ? box : gtview]) { n.classList.remove("rise"); void n.offsetWidth; n.classList.add("rise"); } | |
| glideSide(y0); | |
| } | |
| writeHash(); | |
| }; | |
| // The question slides left and the ground-truth document comes in at the | |
| // right. Asked fresh, the document arrives plain; then the standard loss's | |
| // shading and rank; then ours is the button to press. Restored, it is all just there. | |
| const reveal = animate => { | |
| S.revealed = true; | |
| const y0 = side.getBoundingClientRect().top; | |
| next.remove(); | |
| const staged = animate && ex != null && S.phase !== "all"; | |
| const factCard = panel.querySelector(".tcard.fact-c"); | |
| if (staged) { | |
| S.loss = "ce"; | |
| plain = true; | |
| seg.classList.add("wait"); | |
| more.hidden = true; | |
| if (factCard) factCard.classList.add("linked"); | |
| // In place from the start, unseen, so the centred panel keeps still as each appears. | |
| for (const n of [rankbox, legend]) n.classList.add("pending"); | |
| } else poolShown = true; | |
| panel.append(...extra); | |
| setPhase(S.phase, false); | |
| update(); | |
| if (staged) { | |
| const later = (ms, fn) => setTimeout(() => { if (stage.isConnected) fn(); }, reduced ? 0 : ms); | |
| later(3000, () => { | |
| plain = false; | |
| if (factCard) factCard.classList.remove("linked"); | |
| shownRank = d.n_train; | |
| seg.classList.remove("wait"); | |
| shadeNote.classList.add("rise"); | |
| [rankbox, legend].forEach((n, i) => { n.classList.remove("pending"); n.classList.add("rise"); n.style.animationDelay = `${100 * i}ms`; }); | |
| update(); | |
| }); | |
| later(4600, () => { | |
| more.hidden = false; | |
| more.classList.add("rise"); | |
| if (S.loss === "ce") { | |
| seg.querySelector('[data-l="nlo"]').classList.add("nudge"); | |
| tryOurs.hidden = false; | |
| } | |
| }); | |
| } | |
| const b = document.body; | |
| if (animate) pane.classList.add("enter"); | |
| else b.classList.add("instant"); | |
| stage.classList.add("split"); | |
| b.classList.add("split"); | |
| if (animate) glideSide(y0); | |
| if (animate) res.scrollIntoView({ block: "start", behavior: "smooth" }); | |
| else requestAnimationFrame(() => requestAnimationFrame(() => b.classList.remove("instant"))); | |
| }; | |
| if (S.revealed) reveal(false); | |
| S.view = { | |
| k: S.asked, key: d.key, | |
| busy: on => pane.classList.toggle("busy", on), | |
| // Another method's ranking of the same question. | |
| setData(nr) { | |
| r = nr; | |
| note.hidden = r.correct; | |
| if (S.revealed) update(); | |
| }, | |
| }; | |
| } | |
| /* ------------------------------------------------------------------ wiring */ | |
| function wire() { | |
| const inp = $("q"); | |
| $("clear").addEventListener("click", clear); | |
| inp.addEventListener("focus", () => { S.on = -1; drawSuggest(); chip(); }); | |
| inp.addEventListener("blur", () => setTimeout(() => { $("suggest").hidden = true; }, 0)); | |
| inp.addEventListener("input", () => { S.sel = null; S.on = -1; drawSuggest(); chip(); }); | |
| inp.addEventListener("keydown", ev => { | |
| const n = S.opts.length, open = !$("suggest").hidden; | |
| if (ev.key === "ArrowDown" && open) { S.on = (S.on + 1) % n; drawSuggest(); ev.preventDefault(); } | |
| else if (ev.key === "ArrowUp" && open) { S.on = (S.on - 1 + n) % n; drawSuggest(); ev.preventDefault(); } | |
| else if (ev.key === "Escape") { $("suggest").hidden = true; } | |
| }); | |
| $("ask").addEventListener("submit", ev => { | |
| ev.preventDefault(); | |
| if (S.sel == null) { | |
| if (!S.opts.length || $("suggest").hidden && !inp.value.trim()) return; | |
| choose(S.opts[Math.max(S.on, 0)][0]); | |
| } | |
| run(S.sel); | |
| }); | |
| } | |
| (async function () { | |
| try { | |
| S.index = await loadData("data/index.js"); | |
| } catch (e) { | |
| document.querySelector("main .col").innerHTML = `<h2>No data next to this page</h2> | |
| <p><code>data/index.js</code> did not load. Build it with <code>python filter/demo/build.py</code>.</p>`; | |
| return; | |
| } | |
| wire(); | |
| const q = readHash(), revealed = S.revealed; | |
| await modelChanged(); | |
| if (q != null && S.ds.queries[q]) { choose(q); $("q").blur(); run(q, revealed); } | |
| })(); | |
| </script> | |
| </html> | |