Spaces:
Sleeping
Sleeping
Deploy ICL J-space elicitation harness (PRD execution)
Browse files- README.md +41 -2
- app.py +159 -0
- data/DATASET_README.md +56 -0
- data/lens-eval-association.json +718 -0
- jspace/__init__.py +15 -0
- jspace/dataset.py +102 -0
- jspace/experiment.py +253 -0
- jspace/gpu.py +36 -0
- jspace/groundtruth.py +112 -0
- jspace/model.py +103 -0
- jspace/report.py +154 -0
- jspace/scoring.py +177 -0
- requirements.txt +13 -0
- tests/__init__.py +0 -0
- tests/test_local.py +206 -0
- tests/tiny.py +87 -0
README.md
CHANGED
|
@@ -9,7 +9,46 @@ python_version: '3.12'
|
|
| 9 |
app_file: app.py
|
| 10 |
pinned: false
|
| 11 |
license: mit
|
| 12 |
-
short_description:
|
| 13 |
---
|
| 14 |
|
| 15 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
app_file: app.py
|
| 10 |
pinned: false
|
| 11 |
license: mit
|
| 12 |
+
short_description: ICL elicitation of J-space contents (scaling curve)
|
| 13 |
---
|
| 14 |
|
| 15 |
+
# In-Context Elicitation of J-Space Contents
|
| 16 |
+
|
| 17 |
+
Can `google/gemma-3-12b-it` report the concepts in its own **J-space** using
|
| 18 |
+
**few-shot prompting alone** (no weight updates), and does that report improve
|
| 19 |
+
with more in-context examples? This Space measures it and produces the
|
| 20 |
+
**scaling curve**: report–vs–ground-truth overlap as a function of the number of
|
| 21 |
+
in-context examples `n ∈ {0, 1, 3, 10}` (plus a format-only control).
|
| 22 |
+
|
| 23 |
+
One model, loaded once, serves two computations over the same weights:
|
| 24 |
+
|
| 25 |
+
- **Report** — `model.generate()` on a few-shot prompt → a ranked concept list.
|
| 26 |
+
- **Ground truth** — the Jacobian lens (`jlens`, Neuronpedia's pretrained lens
|
| 27 |
+
for this model) read out at a fixed **(middle layer, last content token)**,
|
| 28 |
+
top-8 tokens.
|
| 29 |
+
|
| 30 |
+
Scoring is embedding-similarity (`all-MiniLM-L6-v2`) overlap@k + Spearman on
|
| 31 |
+
matched concepts, on a held-out eval split — never string match.
|
| 32 |
+
|
| 33 |
+
See `PRD.md` in the source repo for the full design.
|
| 34 |
+
|
| 35 |
+
## Layout
|
| 36 |
+
|
| 37 |
+
- `app.py` — Gradio UI (single-prompt tab + scaling-curve tab).
|
| 38 |
+
- `jspace/` — `dataset`, `model` (singleton), `groundtruth`, `report`,
|
| 39 |
+
`scoring`, `experiment`.
|
| 40 |
+
- `data/` — vendored `lens-eval-association.json` (jlens eval set).
|
| 41 |
+
- `tests/` — CPU verification against jlens's `TinyDecoder`.
|
| 42 |
+
|
| 43 |
+
## Runtime requirements
|
| 44 |
+
|
| 45 |
+
- ZeroGPU hardware (`@spaces.GPU`) — already requested for this Space.
|
| 46 |
+
- An `HF_TOKEN` **Space secret** whose account has accepted the gated
|
| 47 |
+
**`google/gemma-3-12b-it`** license (the runtime downloads the model).
|
| 48 |
+
|
| 49 |
+
## Local tests
|
| 50 |
+
|
| 51 |
+
```bash
|
| 52 |
+
pip install -r requirements.txt pytest
|
| 53 |
+
python -m pytest tests/test_local.py -q
|
| 54 |
+
```
|
app.py
ADDED
|
@@ -0,0 +1,159 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Gradio app: In-Context Elicitation of J-Space Contents.
|
| 2 |
+
|
| 3 |
+
Two tabs (§7):
|
| 4 |
+
|
| 5 |
+
1. **Single prompt** — pick an eval prompt and ``n``, run the report path, and
|
| 6 |
+
see the model's ranked concepts beside the J-lens ground truth, with scores.
|
| 7 |
+
2. **Scaling curve** — run the grid over ``n in {0,1,3,10}`` + format-only across
|
| 8 |
+
the eval set and plot overlap-vs-n with the chance-floor baseline.
|
| 9 |
+
|
| 10 |
+
Both GPU paths (ground truth, report) run on ZeroGPU inside the ``@spaces.GPU``
|
| 11 |
+
functions in ``jspace.groundtruth`` / ``jspace.report``. This module is pure UI +
|
| 12 |
+
orchestration and imports cleanly on CPU.
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
from __future__ import annotations
|
| 16 |
+
|
| 17 |
+
import gradio as gr
|
| 18 |
+
import pandas as pd
|
| 19 |
+
|
| 20 |
+
from jspace import experiment as exp
|
| 21 |
+
from jspace import groundtruth as gt
|
| 22 |
+
from jspace import report as rp
|
| 23 |
+
from jspace.dataset import make_split
|
| 24 |
+
from jspace.experiment import CONDITIONS, RunConfig
|
| 25 |
+
|
| 26 |
+
SPLIT = make_split()
|
| 27 |
+
EVAL_BY_LABEL = {f"{it.name} — {it.prompt[:60]}…": it for it in SPLIT.eval}
|
| 28 |
+
N_CHOICES = [0, 1, 3, 10]
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
# ------------------------------------------------------------- single prompt
|
| 32 |
+
def run_single(eval_label: str, n: int, format_only: bool):
|
| 33 |
+
item = EVAL_BY_LABEL[eval_label]
|
| 34 |
+
|
| 35 |
+
# Ground truth (cached across calls).
|
| 36 |
+
labels = exp.ensure_labels(SPLIT.pool + SPLIT.eval)
|
| 37 |
+
truth = labels[item.name]["tokens"]
|
| 38 |
+
gt_layer = labels[item.name]["layer"]
|
| 39 |
+
gt_pos = labels[item.name]["position"]
|
| 40 |
+
|
| 41 |
+
# Few-shot examples for this n, from the pool.
|
| 42 |
+
examples = exp._examples_for(item, SPLIT.pool, labels, int(n), seed=SPLIT.seed)
|
| 43 |
+
rep = rp.report_gpu(item.prompt, examples, format_only=bool(format_only))
|
| 44 |
+
|
| 45 |
+
from jspace import scoring
|
| 46 |
+
|
| 47 |
+
sc = scoring.score(rep["concepts"], truth, raw_report_text=rep.get("raw", ""))
|
| 48 |
+
|
| 49 |
+
truth_md = "\n".join(f"{i}. `{t}`" for i, t in enumerate(truth, 1))
|
| 50 |
+
report_md = "\n".join(f"{i}. {c}" for i, c in enumerate(rep["concepts"], 1)) or "_(empty)_"
|
| 51 |
+
match_md = (
|
| 52 |
+
"\n".join(f"- {r} ≈ `{t}` (sim {s:.2f})" for r, t, s in sc.matches)
|
| 53 |
+
or "_(no matches above threshold)_"
|
| 54 |
+
)
|
| 55 |
+
scores_md = (
|
| 56 |
+
f"**overlap@k** = {sc.overlap_at_k:.2f} ({sc.n_matched}/{sc.k}) | "
|
| 57 |
+
f"**Spearman** = {sc.spearman:.2f} | "
|
| 58 |
+
f"**refusal** = {'yes' if sc.is_refusal else 'no'}\n\n"
|
| 59 |
+
f"_Ground truth: J-lens top-{sc.k} at layer {gt_layer}, position {gt_pos}._"
|
| 60 |
+
)
|
| 61 |
+
return report_md, truth_md, match_md, scores_md
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
# --------------------------------------------------------------- scaling run
|
| 65 |
+
def run_curve(eval_size: int, progress=gr.Progress()):
|
| 66 |
+
cfg = RunConfig(eval_size=int(eval_size))
|
| 67 |
+
msgs: list[str] = []
|
| 68 |
+
|
| 69 |
+
def log(m: str):
|
| 70 |
+
msgs.append(m)
|
| 71 |
+
progress(0.5, desc=m)
|
| 72 |
+
|
| 73 |
+
out = exp.run_experiment(config=cfg, progress=log)
|
| 74 |
+
curve = out["curve"]
|
| 75 |
+
|
| 76 |
+
rows = [
|
| 77 |
+
{
|
| 78 |
+
"condition": c["label"],
|
| 79 |
+
"n": c["n"],
|
| 80 |
+
"mean_overlap@k": round(c["mean_overlap"], 3),
|
| 81 |
+
"mean_spearman": round(c["mean_spearman"], 3),
|
| 82 |
+
"refusal_rate": round(c["refusal_rate"], 3),
|
| 83 |
+
"items": c["n_items"],
|
| 84 |
+
}
|
| 85 |
+
for c in curve["conditions"]
|
| 86 |
+
]
|
| 87 |
+
table = pd.DataFrame(rows)
|
| 88 |
+
|
| 89 |
+
# Plot only the numeric-n conditions (exclude format-only) for the curve.
|
| 90 |
+
plot_rows = [
|
| 91 |
+
{"n": c["n"], "mean_overlap@k": c["mean_overlap"]}
|
| 92 |
+
for c in curve["conditions"]
|
| 93 |
+
if not c["format_only"]
|
| 94 |
+
]
|
| 95 |
+
plot_df = pd.DataFrame(sorted(plot_rows, key=lambda r: r["n"]))
|
| 96 |
+
|
| 97 |
+
summary = (
|
| 98 |
+
f"**Chance floor** overlap@k = {curve['baseline_floor_overlap']:.3f} | "
|
| 99 |
+
f"**Ceiling** (lens self-consistency) = {curve['baseline_ceiling_overlap']:.2f}\n\n"
|
| 100 |
+
+ "\n".join(f"- {m}" for m in msgs[-6:])
|
| 101 |
+
)
|
| 102 |
+
return table, plot_df, summary
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def build_ui() -> gr.Blocks:
|
| 106 |
+
with gr.Blocks(title="ICL Elicitation of J-Space") as demo:
|
| 107 |
+
gr.Markdown(
|
| 108 |
+
"# In-Context Elicitation of J-Space Contents\n"
|
| 109 |
+
"Can `gemma-3-12b-it` report the concepts in its own J-space using "
|
| 110 |
+
"few-shot prompting alone? Ground truth is the Jacobian-lens top-k "
|
| 111 |
+
"readout at a fixed (middle layer, last content token). "
|
| 112 |
+
"See PRD for the full design."
|
| 113 |
+
)
|
| 114 |
+
with gr.Tab("Single prompt"):
|
| 115 |
+
with gr.Row():
|
| 116 |
+
eval_dd = gr.Dropdown(
|
| 117 |
+
choices=list(EVAL_BY_LABEL), label="Held-out eval prompt",
|
| 118 |
+
value=next(iter(EVAL_BY_LABEL)),
|
| 119 |
+
)
|
| 120 |
+
n_dd = gr.Dropdown(choices=N_CHOICES, value=3, label="n (in-context examples)")
|
| 121 |
+
fmt_cb = gr.Checkbox(label="format-only (no worked examples)", value=False)
|
| 122 |
+
run_btn = gr.Button("Run", variant="primary")
|
| 123 |
+
with gr.Row():
|
| 124 |
+
report_out = gr.Markdown(label="Model report (ranked)")
|
| 125 |
+
truth_out = gr.Markdown(label="J-lens ground truth (top-k)")
|
| 126 |
+
match_out = gr.Markdown(label="Matches")
|
| 127 |
+
scores_out = gr.Markdown(label="Scores")
|
| 128 |
+
run_btn.click(
|
| 129 |
+
run_single,
|
| 130 |
+
inputs=[eval_dd, n_dd, fmt_cb],
|
| 131 |
+
outputs=[report_out, truth_out, match_out, scores_out],
|
| 132 |
+
)
|
| 133 |
+
with gr.Tab("Scaling curve"):
|
| 134 |
+
gr.Markdown(
|
| 135 |
+
"Runs `n ∈ {0,1,3,10}` + format-only over the eval set. "
|
| 136 |
+
"Results are cached and resumable across ZeroGPU quota windows."
|
| 137 |
+
)
|
| 138 |
+
size_sl = gr.Slider(
|
| 139 |
+
2, len(SPLIT.eval), value=min(6, len(SPLIT.eval)), step=1,
|
| 140 |
+
label="eval prompts to run (smaller = fits free ZeroGPU quota)",
|
| 141 |
+
)
|
| 142 |
+
curve_btn = gr.Button("Run / refresh scaling curve", variant="primary")
|
| 143 |
+
curve_summary = gr.Markdown()
|
| 144 |
+
curve_plot = gr.LinePlot(
|
| 145 |
+
x="n", y="mean_overlap@k", title="Report–vs–ground-truth overlap vs n",
|
| 146 |
+
)
|
| 147 |
+
curve_table = gr.Dataframe(label="Per-condition summary")
|
| 148 |
+
curve_btn.click(
|
| 149 |
+
run_curve,
|
| 150 |
+
inputs=[size_sl],
|
| 151 |
+
outputs=[curve_table, curve_plot, curve_summary],
|
| 152 |
+
)
|
| 153 |
+
return demo
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
demo = build_ui()
|
| 157 |
+
|
| 158 |
+
if __name__ == "__main__":
|
| 159 |
+
demo.launch()
|
data/DATASET_README.md
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Evaluations
|
| 2 |
+
|
| 3 |
+
Six prompt distributions used to evaluate lens quality (§methods-comparison). Each `{slug}.json` is prompts only.
|
| 4 |
+
|
| 5 |
+
## Conventions
|
| 6 |
+
|
| 7 |
+
Unless a section says otherwise:
|
| 8 |
+
|
| 9 |
+
- **Lens readout** — at each (layer, token position) the Jacobian lens
|
| 10 |
+
returns a ranked list of vocabulary tokens.
|
| 11 |
+
- **Workspace band** — the contiguous mid-network layer range where
|
| 12 |
+
workspace content is read; experiments report over this band, not
|
| 13 |
+
individual layers.
|
| 14 |
+
- **Hit** — a target token is a *hit* if it appears at lens rank 1 at any
|
| 15 |
+
(layer, position) in the band over the scored span.
|
| 16 |
+
- **Swap** — clamping a lens coordinate replaces one token's direction with
|
| 17 |
+
another's at every band layer at the specified positions, then samples
|
| 18 |
+
the continuation.
|
| 19 |
+
- Prompts that span multiple turns are given as
|
| 20 |
+
`[{"role": "user"|"assistant", "content": ...}]`.
|
| 21 |
+
|
| 22 |
+
## lens-eval-multihop
|
| 23 |
+
|
| 24 |
+
[`lens-eval-multihop.json`](lens-eval-multihop.json)
|
| 25 |
+
|
| 26 |
+
Lens-quality eval (§methods-comparison). `items[*]` has `prompt` and `intermediates`. `target` defines the readout position only and is not itself scored. Readout is at a single position — the token immediately preceding `target` — across all layers. Metric: pass@k = mean over items of the fraction of `intermediates` whose min-over-layers lens rank ≤ k.
|
| 27 |
+
|
| 28 |
+
## lens-eval-multilingual
|
| 29 |
+
|
| 30 |
+
[`lens-eval-multilingual.json`](lens-eval-multilingual.json)
|
| 31 |
+
|
| 32 |
+
Lens-quality eval (§methods-comparison). `items[*]` has `prompt` and `intermediates`. `target` defines the readout position only and is not itself scored. Readout is at a single position — the token immediately preceding `target` — across all layers. Metric: pass@k = mean over items of the fraction of `intermediates` whose min-over-layers lens rank ≤ k.
|
| 33 |
+
|
| 34 |
+
## lens-eval-poetry
|
| 35 |
+
|
| 36 |
+
[`lens-eval-poetry.json`](lens-eval-poetry.json)
|
| 37 |
+
|
| 38 |
+
Lens-quality eval (§methods-comparison). `items[*]` has `prompt` and `intermediates`. Readout is at a single position — the last newline token (end of line 1 of the couplet) — across all layers. Metric: pass@k = mean over items of the fraction of `intermediates` whose min-over-layers lens rank ≤ k.
|
| 39 |
+
|
| 40 |
+
## lens-eval-order-ops
|
| 41 |
+
|
| 42 |
+
[`lens-eval-order-ops.json`](lens-eval-order-ops.json)
|
| 43 |
+
|
| 44 |
+
Lens-quality eval (§methods-comparison). Each intermediate is a key expanded to a synonym set (numbers → digit and word forms; operations → symbol and word forms); rank is the min over single-token synonyms at each layer. `items[*]` has `prompt` and `intermediates`. `target` defines the readout position only and is not itself scored. Readout is at a single position — the token immediately preceding `target` — across all layers. Metric: pass@k = mean over items of the fraction of `intermediates` whose min-over-layers lens rank ≤ k.
|
| 45 |
+
|
| 46 |
+
## lens-eval-association
|
| 47 |
+
|
| 48 |
+
[`lens-eval-association.json`](lens-eval-association.json)
|
| 49 |
+
|
| 50 |
+
Lens-quality eval (§methods-comparison). Each item is a short vignette that evokes a single concept (grief, Einstein, noir, ...) without ever naming it; `intermediates` holds that one concept word. `items[*]` has `prompt` and `intermediates`. Readout is at a single position — the final prompt token — the closing period — across all layers. Metric: pass@k = mean over items of the fraction of `intermediates` whose min-over-layers lens rank ≤ k.
|
| 51 |
+
|
| 52 |
+
## lens-eval-typo
|
| 53 |
+
|
| 54 |
+
[`lens-eval-typo.json`](lens-eval-typo.json)
|
| 55 |
+
|
| 56 |
+
Lens-quality eval (§methods-comparison). Each prompt is a sentence ending in a common misspelling; `intermediates` holds the single correctly-spelled word. `items[*]` has `prompt` and `intermediates`. Readout is at a single position — the final prompt token, i.e. the last tokenizer fragment of the misspelling — across all layers. Metric: pass@k = mean over items of the fraction of `intermediates` whose min-over-layers lens rank ≤ k.
|
data/lens-eval-association.json
ADDED
|
@@ -0,0 +1,718 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"items": [
|
| 3 |
+
{
|
| 4 |
+
"name": "grief",
|
| 5 |
+
"prompt": "She kept his coffee mug on the counter, washed it every morning and put it back, though no one had used it in eleven months.",
|
| 6 |
+
"intermediates": [
|
| 7 |
+
"grief"
|
| 8 |
+
]
|
| 9 |
+
},
|
| 10 |
+
{
|
| 11 |
+
"name": "pregnant",
|
| 12 |
+
"prompt": "She turned down the wine with a small smile and kept one hand on the table edge, not quite touching her stomach.",
|
| 13 |
+
"intermediates": [
|
| 14 |
+
"pregnant"
|
| 15 |
+
]
|
| 16 |
+
},
|
| 17 |
+
{
|
| 18 |
+
"name": "lie",
|
| 19 |
+
"prompt": "He couldn't meet her eyes when she asked, and the answer came out a half-second too fast.",
|
| 20 |
+
"intermediates": [
|
| 21 |
+
"lie"
|
| 22 |
+
]
|
| 23 |
+
},
|
| 24 |
+
{
|
| 25 |
+
"name": "poverty",
|
| 26 |
+
"prompt": "She did the math twice before the cashier scanned the last item, and put the cereal back without explaining.",
|
| 27 |
+
"intermediates": [
|
| 28 |
+
"poverty"
|
| 29 |
+
]
|
| 30 |
+
},
|
| 31 |
+
{
|
| 32 |
+
"name": "proposal",
|
| 33 |
+
"prompt": "He touched his jacket pocket for the fourth time that evening, rehearsing the words again while the waiter cleared the plates.",
|
| 34 |
+
"intermediates": [
|
| 35 |
+
"proposal"
|
| 36 |
+
]
|
| 37 |
+
},
|
| 38 |
+
{
|
| 39 |
+
"name": "fired",
|
| 40 |
+
"prompt": "The box on his desk wasn't the one he came in with, and through the glass wall he could see HR waiting by the elevator.",
|
| 41 |
+
"intermediates": [
|
| 42 |
+
"fired"
|
| 43 |
+
]
|
| 44 |
+
},
|
| 45 |
+
{
|
| 46 |
+
"name": "divorce",
|
| 47 |
+
"prompt": "She signed on the line her lawyer pointed to, slid the pen across the table, and didn't wait for him to pick it up.",
|
| 48 |
+
"intermediates": [
|
| 49 |
+
"divorce"
|
| 50 |
+
]
|
| 51 |
+
},
|
| 52 |
+
{
|
| 53 |
+
"name": "relief",
|
| 54 |
+
"prompt": "She read the line twice before she let herself breathe, and the paper shook a little as she set it down.",
|
| 55 |
+
"intermediates": [
|
| 56 |
+
"relief"
|
| 57 |
+
]
|
| 58 |
+
},
|
| 59 |
+
{
|
| 60 |
+
"name": "pennsylvania",
|
| 61 |
+
"prompt": "The scrapple was already on the table when he came down, and through the window the hex sign on the neighbor's barn caught the early light.",
|
| 62 |
+
"intermediates": [
|
| 63 |
+
"Pennsylvania"
|
| 64 |
+
]
|
| 65 |
+
},
|
| 66 |
+
{
|
| 67 |
+
"name": "prohibition",
|
| 68 |
+
"prompt": "She tapped her cigarette holder against the bar while the gramophone crackled, and somewhere behind the bookcase the real party was just getting loud.",
|
| 69 |
+
"intermediates": [
|
| 70 |
+
"prohibition"
|
| 71 |
+
]
|
| 72 |
+
},
|
| 73 |
+
{
|
| 74 |
+
"name": "nasa",
|
| 75 |
+
"prompt": "The countdown hit zero and the rumble under her seat became a roar that pressed her flat, the blue curve of the world falling away through the tiny window.",
|
| 76 |
+
"intermediates": [
|
| 77 |
+
"NASA"
|
| 78 |
+
]
|
| 79 |
+
},
|
| 80 |
+
{
|
| 81 |
+
"name": "einstein",
|
| 82 |
+
"prompt": "He closed his eyes on the tram and let the buildings blur, chasing the beam of light in his head the way other men chased trains.",
|
| 83 |
+
"intermediates": [
|
| 84 |
+
"Einstein"
|
| 85 |
+
]
|
| 86 |
+
},
|
| 87 |
+
{
|
| 88 |
+
"name": "darwin",
|
| 89 |
+
"prompt": "He reread the letter and felt twenty years of notes collapse — someone else had seen it too, and written it down first.",
|
| 90 |
+
"intermediates": [
|
| 91 |
+
"Darwin"
|
| 92 |
+
]
|
| 93 |
+
},
|
| 94 |
+
{
|
| 95 |
+
"name": "beatles",
|
| 96 |
+
"prompt": "They crossed in single file, one of them barefoot, while the photographer counted to three and the traffic waited.",
|
| 97 |
+
"intermediates": [
|
| 98 |
+
"Beatles"
|
| 99 |
+
]
|
| 100 |
+
},
|
| 101 |
+
{
|
| 102 |
+
"name": "lincoln",
|
| 103 |
+
"prompt": "He folded the two pages back into his coat — the other man had spoken for two hours, and he would need about two minutes.",
|
| 104 |
+
"intermediates": [
|
| 105 |
+
"Lincoln"
|
| 106 |
+
]
|
| 107 |
+
},
|
| 108 |
+
{
|
| 109 |
+
"name": "obama",
|
| 110 |
+
"prompt": "The portraits on the wall had all been one color, and the desk was smaller than he'd expected when he sat down that first January morning.",
|
| 111 |
+
"intermediates": [
|
| 112 |
+
"Obama"
|
| 113 |
+
]
|
| 114 |
+
},
|
| 115 |
+
{
|
| 116 |
+
"name": "olympics",
|
| 117 |
+
"prompt": "The anthem played and she touched the cold metal at her neck, four years compressed into a minute on the top step.",
|
| 118 |
+
"intermediates": [
|
| 119 |
+
"Olympics"
|
| 120 |
+
]
|
| 121 |
+
},
|
| 122 |
+
{
|
| 123 |
+
"name": "churchill",
|
| 124 |
+
"prompt": "He lit another cigar in the war room and leaned into the microphone, knowing the sirens above ground were doing half his work for him.",
|
| 125 |
+
"intermediates": [
|
| 126 |
+
"Churchill"
|
| 127 |
+
]
|
| 128 |
+
},
|
| 129 |
+
{
|
| 130 |
+
"name": "newton",
|
| 131 |
+
"prompt": "He watched the white light split into seven on the far wall and reached for a fresh page before the sun moved.",
|
| 132 |
+
"intermediates": [
|
| 133 |
+
"Newton"
|
| 134 |
+
]
|
| 135 |
+
},
|
| 136 |
+
{
|
| 137 |
+
"name": "mozart",
|
| 138 |
+
"prompt": "The boy's feet didn't reach the pedals, but the emperor stopped talking when the first chord landed.",
|
| 139 |
+
"intermediates": [
|
| 140 |
+
"Mozart"
|
| 141 |
+
]
|
| 142 |
+
},
|
| 143 |
+
{
|
| 144 |
+
"name": "berlin",
|
| 145 |
+
"prompt": "The hammers rang on the concrete all night, and by morning the gap was wide enough to reach her brother's hand on the other side for the first time in twenty-eight years.",
|
| 146 |
+
"intermediates": [
|
| 147 |
+
"Berlin"
|
| 148 |
+
]
|
| 149 |
+
},
|
| 150 |
+
{
|
| 151 |
+
"name": "armstrong",
|
| 152 |
+
"prompt": "His boot sank an inch into the gray dust and the flag behind him wouldn't hang right with nothing there to move it.",
|
| 153 |
+
"intermediates": [
|
| 154 |
+
"Armstrong"
|
| 155 |
+
]
|
| 156 |
+
},
|
| 157 |
+
{
|
| 158 |
+
"name": "wright",
|
| 159 |
+
"prompt": "Twelve seconds, his brother shouted from the dune, and the machine was already back on the sand — but for twelve seconds it hadn't been.",
|
| 160 |
+
"intermediates": [
|
| 161 |
+
"Wright"
|
| 162 |
+
]
|
| 163 |
+
},
|
| 164 |
+
{
|
| 165 |
+
"name": "gandhi",
|
| 166 |
+
"prompt": "He reached the sea after three weeks on foot and bent to lift a handful of what the law said he could not make.",
|
| 167 |
+
"intermediates": [
|
| 168 |
+
"Gandhi"
|
| 169 |
+
]
|
| 170 |
+
},
|
| 171 |
+
{
|
| 172 |
+
"name": "edison",
|
| 173 |
+
"prompt": "Filament nine hundred and something — he'd stopped counting the exact number — and this one held its glow past the first minute.",
|
| 174 |
+
"intermediates": [
|
| 175 |
+
"Edison"
|
| 176 |
+
]
|
| 177 |
+
},
|
| 178 |
+
{
|
| 179 |
+
"name": "cia",
|
| 180 |
+
"prompt": "He'd used three names in as many cities and the folder on the table had more black bars than words, and none of them were his either.",
|
| 181 |
+
"intermediates": [
|
| 182 |
+
"CIA"
|
| 183 |
+
]
|
| 184 |
+
},
|
| 185 |
+
{
|
| 186 |
+
"name": "vatican",
|
| 187 |
+
"prompt": "The doors had been sealed for two days when the smoke finally changed color, and the crowd in the square knew before the bells did.",
|
| 188 |
+
"intermediates": [
|
| 189 |
+
"Vatican"
|
| 190 |
+
]
|
| 191 |
+
},
|
| 192 |
+
{
|
| 193 |
+
"name": "hollywood",
|
| 194 |
+
"prompt": "He'd pitched the same story in five rooms that week and watched the same polite smile close around it, and the valet still brought the wrong car.",
|
| 195 |
+
"intermediates": [
|
| 196 |
+
"Hollywood"
|
| 197 |
+
]
|
| 198 |
+
},
|
| 199 |
+
{
|
| 200 |
+
"name": "pentagon",
|
| 201 |
+
"prompt": "The building had five sides and she'd walked the inside of all of them that morning, and the badge at her neck still didn't open the door she needed.",
|
| 202 |
+
"intermediates": [
|
| 203 |
+
"Pentagon"
|
| 204 |
+
]
|
| 205 |
+
},
|
| 206 |
+
{
|
| 207 |
+
"name": "unesco",
|
| 208 |
+
"prompt": "The emblem had been Julian Huxley's idea, or near enough, and now it hung beside a thousand temple gates and a thousand ruins the world had agreed to keep.",
|
| 209 |
+
"intermediates": [
|
| 210 |
+
"UNESCO"
|
| 211 |
+
]
|
| 212 |
+
},
|
| 213 |
+
{
|
| 214 |
+
"name": "anger",
|
| 215 |
+
"prompt": "\"It's fine,\" she said, and set the cup down so carefully it didn't make a sound.",
|
| 216 |
+
"intermediates": [
|
| 217 |
+
"anger"
|
| 218 |
+
]
|
| 219 |
+
},
|
| 220 |
+
{
|
| 221 |
+
"name": "frustrated",
|
| 222 |
+
"prompt": "He explained it a third time, slower, and watched her eyes go to the clock again.",
|
| 223 |
+
"intermediates": [
|
| 224 |
+
"frustrated"
|
| 225 |
+
]
|
| 226 |
+
},
|
| 227 |
+
{
|
| 228 |
+
"name": "lonely",
|
| 229 |
+
"prompt": "He cooked for two out of habit and scraped the second plate into the bin before sitting down.",
|
| 230 |
+
"intermediates": [
|
| 231 |
+
"lonely"
|
| 232 |
+
]
|
| 233 |
+
},
|
| 234 |
+
{
|
| 235 |
+
"name": "shame",
|
| 236 |
+
"prompt": "She paid in cash and didn't meet the clerk's eyes, the small bag tucked inside her coat before she reached the door.",
|
| 237 |
+
"intermediates": [
|
| 238 |
+
"shame"
|
| 239 |
+
]
|
| 240 |
+
},
|
| 241 |
+
{
|
| 242 |
+
"name": "noir",
|
| 243 |
+
"prompt": "The neon bled through the blinds in stripes across the desk, and the dame in the doorway had a problem and a cigarette and no intention of telling him which was which.",
|
| 244 |
+
"intermediates": [
|
| 245 |
+
"noir"
|
| 246 |
+
]
|
| 247 |
+
},
|
| 248 |
+
{
|
| 249 |
+
"name": "horror",
|
| 250 |
+
"prompt": "The phone line had been cut sometime after midnight, and now the scratching in the walls had moved from the attic to just behind the bedroom door.",
|
| 251 |
+
"intermediates": [
|
| 252 |
+
"horror"
|
| 253 |
+
]
|
| 254 |
+
},
|
| 255 |
+
{
|
| 256 |
+
"name": "fairy",
|
| 257 |
+
"prompt": "The youngest of three set out with nothing but a crust of bread and a promise, and the woods grew darker the deeper she went.",
|
| 258 |
+
"intermediates": [
|
| 259 |
+
"fairy"
|
| 260 |
+
]
|
| 261 |
+
},
|
| 262 |
+
{
|
| 263 |
+
"name": "siblings",
|
| 264 |
+
"prompt": "She used the nickname nobody else knew and he rolled his eyes the same way he had at the dinner table twenty years ago.",
|
| 265 |
+
"intermediates": [
|
| 266 |
+
"siblings"
|
| 267 |
+
]
|
| 268 |
+
},
|
| 269 |
+
{
|
| 270 |
+
"name": "rivals",
|
| 271 |
+
"prompt": "They shook hands for the cameras and held it a beat too long, each waiting for the other to let go first.",
|
| 272 |
+
"intermediates": [
|
| 273 |
+
"rivals"
|
| 274 |
+
]
|
| 275 |
+
},
|
| 276 |
+
{
|
| 277 |
+
"name": "mother",
|
| 278 |
+
"prompt": "She pressed the back of her hand to his forehead before she said hello, and he let her, the way he had since before he could remember.",
|
| 279 |
+
"intermediates": [
|
| 280 |
+
"mother"
|
| 281 |
+
]
|
| 282 |
+
},
|
| 283 |
+
{
|
| 284 |
+
"name": "boss",
|
| 285 |
+
"prompt": "She kept typing while he stood in her doorway, and he waited, because the meeting didn't start until she closed the laptop.",
|
| 286 |
+
"intermediates": [
|
| 287 |
+
"boss"
|
| 288 |
+
]
|
| 289 |
+
},
|
| 290 |
+
{
|
| 291 |
+
"name": "neighbors",
|
| 292 |
+
"prompt": "She knew what time his car left and what night the bins went out, and had waved at him for six years without learning either of his names.",
|
| 293 |
+
"intermediates": [
|
| 294 |
+
"neighbors"
|
| 295 |
+
]
|
| 296 |
+
},
|
| 297 |
+
{
|
| 298 |
+
"name": "teacher",
|
| 299 |
+
"prompt": "She circled the same mistake in the same red ink she'd used on the last thirty, and reached for the next one on the pile.",
|
| 300 |
+
"intermediates": [
|
| 301 |
+
"teacher"
|
| 302 |
+
]
|
| 303 |
+
},
|
| 304 |
+
{
|
| 305 |
+
"name": "underwater",
|
| 306 |
+
"prompt": "Every sound reached her a half-second late and muffled, and the pressure behind her eyes grew with each meter down.",
|
| 307 |
+
"intermediates": [
|
| 308 |
+
"underwater"
|
| 309 |
+
]
|
| 310 |
+
},
|
| 311 |
+
{
|
| 312 |
+
"name": "desert",
|
| 313 |
+
"prompt": "The horizon shook in the heat and his shadow had shrunk to a puddle at his feet; the last of the water was three swallows, maybe four.",
|
| 314 |
+
"intermediates": [
|
| 315 |
+
"desert"
|
| 316 |
+
]
|
| 317 |
+
},
|
| 318 |
+
{
|
| 319 |
+
"name": "chess",
|
| 320 |
+
"prompt": "Neither of them had spoken in forty minutes when she finally moved the knight, and he tapped the clock without looking up.",
|
| 321 |
+
"intermediates": [
|
| 322 |
+
"chess"
|
| 323 |
+
]
|
| 324 |
+
},
|
| 325 |
+
{
|
| 326 |
+
"name": "poker",
|
| 327 |
+
"prompt": "He pushed the last of his chips to the center and kept his face the way it had been for the last three hands — which was the problem.",
|
| 328 |
+
"intermediates": [
|
| 329 |
+
"poker"
|
| 330 |
+
]
|
| 331 |
+
},
|
| 332 |
+
{
|
| 333 |
+
"name": "interview",
|
| 334 |
+
"prompt": "The question hung there and he realized the pause had already gone on too long; across the table, a pen hovered over a page he couldn't read.",
|
| 335 |
+
"intermediates": [
|
| 336 |
+
"interview"
|
| 337 |
+
]
|
| 338 |
+
},
|
| 339 |
+
{
|
| 340 |
+
"name": "dawn",
|
| 341 |
+
"prompt": "The first bird started before the sky did, and the cold had that thin quality that wouldn't last another hour.",
|
| 342 |
+
"intermediates": [
|
| 343 |
+
"dawn"
|
| 344 |
+
]
|
| 345 |
+
},
|
| 346 |
+
{
|
| 347 |
+
"name": "winter",
|
| 348 |
+
"prompt": "The light was already failing by four and the radiators clanked awake one by one through the building.",
|
| 349 |
+
"intermediates": [
|
| 350 |
+
"winter"
|
| 351 |
+
]
|
| 352 |
+
},
|
| 353 |
+
{
|
| 354 |
+
"name": "death-h",
|
| 355 |
+
"prompt": "The flowers on the table had been changed twice already, and the black dress still hung on the closet door where she'd put it out the night before the service.",
|
| 356 |
+
"intermediates": [
|
| 357 |
+
"death"
|
| 358 |
+
]
|
| 359 |
+
},
|
| 360 |
+
{
|
| 361 |
+
"name": "war-h",
|
| 362 |
+
"prompt": "The letters stopped coming after the telegram, and she kept the ration book in the drawer where his photograph used to be.",
|
| 363 |
+
"intermediates": [
|
| 364 |
+
"war"
|
| 365 |
+
]
|
| 366 |
+
},
|
| 367 |
+
{
|
| 368 |
+
"name": "love-h",
|
| 369 |
+
"prompt": "They'd reached the point where silences didn't need filling, and her hand found his across the table without either of them looking.",
|
| 370 |
+
"intermediates": [
|
| 371 |
+
"love"
|
| 372 |
+
]
|
| 373 |
+
},
|
| 374 |
+
{
|
| 375 |
+
"name": "home-h",
|
| 376 |
+
"prompt": "The smell hit him before the door was fully open — something his mother had made a thousand times, and he was seven years old again for exactly as long as it lasted.",
|
| 377 |
+
"intermediates": [
|
| 378 |
+
"home"
|
| 379 |
+
]
|
| 380 |
+
},
|
| 381 |
+
{
|
| 382 |
+
"name": "school-h",
|
| 383 |
+
"prompt": "The bell rang and thirty chairs scraped at once, but she stayed at her desk, still staring at the red marks on page three.",
|
| 384 |
+
"intermediates": [
|
| 385 |
+
"school"
|
| 386 |
+
]
|
| 387 |
+
},
|
| 388 |
+
{
|
| 389 |
+
"name": "church-h",
|
| 390 |
+
"prompt": "The organ held the last note longer than anyone expected, and she kept her head bowed until the echo had completely died.",
|
| 391 |
+
"intermediates": [
|
| 392 |
+
"church"
|
| 393 |
+
]
|
| 394 |
+
},
|
| 395 |
+
{
|
| 396 |
+
"name": "office-h",
|
| 397 |
+
"prompt": "The coffee in the break room had been sitting since nine, and the clock above the copier ticked louder than anyone's typing.",
|
| 398 |
+
"intermediates": [
|
| 399 |
+
"office"
|
| 400 |
+
]
|
| 401 |
+
},
|
| 402 |
+
{
|
| 403 |
+
"name": "court-h",
|
| 404 |
+
"prompt": "The gavel came down twice and she felt her lawyer's hand on her arm, but she couldn't read the verdict in his face.",
|
| 405 |
+
"intermediates": [
|
| 406 |
+
"court"
|
| 407 |
+
]
|
| 408 |
+
},
|
| 409 |
+
{
|
| 410 |
+
"name": "park-h",
|
| 411 |
+
"prompt": "The bench had been there longer than she had, and the brass plaque said it had been someone's favorite spot before it was hers.",
|
| 412 |
+
"intermediates": [
|
| 413 |
+
"park"
|
| 414 |
+
]
|
| 415 |
+
},
|
| 416 |
+
{
|
| 417 |
+
"name": "city-h",
|
| 418 |
+
"prompt": "The noise never stopped, just changed shape — sirens into traffic into construction into the 3 a.m. garbage truck she'd learned to sleep through.",
|
| 419 |
+
"intermediates": [
|
| 420 |
+
"city"
|
| 421 |
+
]
|
| 422 |
+
},
|
| 423 |
+
{
|
| 424 |
+
"name": "town-h",
|
| 425 |
+
"prompt": "Everyone knew which house had the dog that barked and which porch light meant someone was still up, and the new family hadn't figured out yet that the curtains were optional.",
|
| 426 |
+
"intermediates": [
|
| 427 |
+
"town"
|
| 428 |
+
]
|
| 429 |
+
},
|
| 430 |
+
{
|
| 431 |
+
"name": "field-h",
|
| 432 |
+
"prompt": "The stalks went past her shoulders and the path behind her had already closed, but the farmhouse roof was still visible if she stood on her toes.",
|
| 433 |
+
"intermediates": [
|
| 434 |
+
"field"
|
| 435 |
+
]
|
| 436 |
+
},
|
| 437 |
+
{
|
| 438 |
+
"name": "store-h",
|
| 439 |
+
"prompt": "The cart had a wheel that pulled left, and by aisle six she'd given up fighting it and just let it lead.",
|
| 440 |
+
"intermediates": [
|
| 441 |
+
"store"
|
| 442 |
+
]
|
| 443 |
+
},
|
| 444 |
+
{
|
| 445 |
+
"name": "art-h",
|
| 446 |
+
"prompt": "She stepped back three paces, tilted her head, then stepped forward and scraped the whole left corner back to canvas.",
|
| 447 |
+
"intermediates": [
|
| 448 |
+
"art"
|
| 449 |
+
]
|
| 450 |
+
},
|
| 451 |
+
{
|
| 452 |
+
"name": "music-h",
|
| 453 |
+
"prompt": "The conductor's hand hung in the air one extra beat, and in the silence before it fell, two thousand people held their breath.",
|
| 454 |
+
"intermediates": [
|
| 455 |
+
"music"
|
| 456 |
+
]
|
| 457 |
+
},
|
| 458 |
+
{
|
| 459 |
+
"name": "math-h",
|
| 460 |
+
"prompt": "He filled the third board, capped the marker, and stood back — somewhere in those six hundred symbols was either the proof or the mistake.",
|
| 461 |
+
"intermediates": [
|
| 462 |
+
"math"
|
| 463 |
+
]
|
| 464 |
+
},
|
| 465 |
+
{
|
| 466 |
+
"name": "science-h",
|
| 467 |
+
"prompt": "The result came back the same the fourth time, and she stared at the readout until it stopped meaning anything, because if it was right, the textbook was wrong.",
|
| 468 |
+
"intermediates": [
|
| 469 |
+
"science"
|
| 470 |
+
]
|
| 471 |
+
},
|
| 472 |
+
{
|
| 473 |
+
"name": "history-h",
|
| 474 |
+
"prompt": "He ran his fingers over the names carved into the stone, each one a boy his grandfather had mentioned once and never again.",
|
| 475 |
+
"intermediates": [
|
| 476 |
+
"history"
|
| 477 |
+
]
|
| 478 |
+
},
|
| 479 |
+
{
|
| 480 |
+
"name": "law-h",
|
| 481 |
+
"prompt": "She read the clause a third time, initialed where the tab pointed, and tried not to think about what 4(b) would mean if anyone ever invoked it.",
|
| 482 |
+
"intermediates": [
|
| 483 |
+
"law"
|
| 484 |
+
]
|
| 485 |
+
},
|
| 486 |
+
{
|
| 487 |
+
"name": "film-h",
|
| 488 |
+
"prompt": "The lights went down and the logo came up, and for two hours the whole room forgot it had bodies.",
|
| 489 |
+
"intermediates": [
|
| 490 |
+
"film"
|
| 491 |
+
]
|
| 492 |
+
},
|
| 493 |
+
{
|
| 494 |
+
"name": "code-h",
|
| 495 |
+
"prompt": "The test went green on the forty-third run, and she immediately distrusted it — green after forty-two reds meant she'd fixed the wrong thing.",
|
| 496 |
+
"intermediates": [
|
| 497 |
+
"code"
|
| 498 |
+
]
|
| 499 |
+
},
|
| 500 |
+
{
|
| 501 |
+
"name": "news-h",
|
| 502 |
+
"prompt": "The anchor's voice cracked on the second sentence, and by the third you could hear papers shuffling off-camera as someone handed her an update.",
|
| 503 |
+
"intermediates": [
|
| 504 |
+
"news"
|
| 505 |
+
]
|
| 506 |
+
},
|
| 507 |
+
{
|
| 508 |
+
"name": "night-h",
|
| 509 |
+
"prompt": "The last porch light on the street clicked off, and the dark filled in so completely that the fence three feet away might as well have been a mile.",
|
| 510 |
+
"intermediates": [
|
| 511 |
+
"night"
|
| 512 |
+
]
|
| 513 |
+
},
|
| 514 |
+
{
|
| 515 |
+
"name": "past-h",
|
| 516 |
+
"prompt": "The photograph was from before color, and she had to squint to see her own face under all those years.",
|
| 517 |
+
"intermediates": [
|
| 518 |
+
"past"
|
| 519 |
+
]
|
| 520 |
+
},
|
| 521 |
+
{
|
| 522 |
+
"name": "future-h",
|
| 523 |
+
"prompt": "The child asked what she wanted to be, and she answered with the certainty of someone who had never yet been told no.",
|
| 524 |
+
"intermediates": [
|
| 525 |
+
"future"
|
| 526 |
+
]
|
| 527 |
+
},
|
| 528 |
+
{
|
| 529 |
+
"name": "race-h",
|
| 530 |
+
"prompt": "The tape was fifty meters away and his lungs were already done, but the footsteps behind him were getting louder and that was all that mattered.",
|
| 531 |
+
"intermediates": [
|
| 532 |
+
"race"
|
| 533 |
+
]
|
| 534 |
+
},
|
| 535 |
+
{
|
| 536 |
+
"name": "game-h",
|
| 537 |
+
"prompt": "She moved the piece, looked up, and waited — and in his face she saw exactly the three moves he hadn't seen coming.",
|
| 538 |
+
"intermediates": [
|
| 539 |
+
"game"
|
| 540 |
+
]
|
| 541 |
+
},
|
| 542 |
+
{
|
| 543 |
+
"name": "test-h",
|
| 544 |
+
"prompt": "She filled in the last bubble, checked the clock, and decided that whatever she didn't know by now, thirty more seconds wasn't going to fix.",
|
| 545 |
+
"intermediates": [
|
| 546 |
+
"test"
|
| 547 |
+
]
|
| 548 |
+
},
|
| 549 |
+
{
|
| 550 |
+
"name": "search-h",
|
| 551 |
+
"prompt": "They'd walked the same grid four times since sunrise, and the dog kept pulling toward the creek that wasn't on anyone's map.",
|
| 552 |
+
"intermediates": [
|
| 553 |
+
"search"
|
| 554 |
+
]
|
| 555 |
+
},
|
| 556 |
+
{
|
| 557 |
+
"name": "free-h",
|
| 558 |
+
"prompt": "The gate buzzed and the first thing he noticed was how big the sky looked without a fence cutting it in half.",
|
| 559 |
+
"intermediates": [
|
| 560 |
+
"free"
|
| 561 |
+
]
|
| 562 |
+
},
|
| 563 |
+
{
|
| 564 |
+
"name": "found-h",
|
| 565 |
+
"prompt": "The dog came back muddy with something in its mouth, and she knew before she'd unwrapped it that this was what three weeks of posters had been for.",
|
| 566 |
+
"intermediates": [
|
| 567 |
+
"found"
|
| 568 |
+
]
|
| 569 |
+
},
|
| 570 |
+
{
|
| 571 |
+
"name": "king-h",
|
| 572 |
+
"prompt": "He sat where his father had sat, and his grandfather before that, and felt the weight of every eye in the hall waiting for the word that would become command.",
|
| 573 |
+
"intermediates": [
|
| 574 |
+
"king"
|
| 575 |
+
]
|
| 576 |
+
},
|
| 577 |
+
{
|
| 578 |
+
"name": "father-h",
|
| 579 |
+
"prompt": "He showed her how to hold the wrench the way his own old man had shown him, and didn't mention that this was the third time this month she'd asked.",
|
| 580 |
+
"intermediates": [
|
| 581 |
+
"father"
|
| 582 |
+
]
|
| 583 |
+
},
|
| 584 |
+
{
|
| 585 |
+
"name": "child-h",
|
| 586 |
+
"prompt": "The crayon drawings on the fridge had started with houses and suns and now included something that was either a dragon or a truck — she'd learned not to ask which.",
|
| 587 |
+
"intermediates": [
|
| 588 |
+
"child"
|
| 589 |
+
]
|
| 590 |
+
},
|
| 591 |
+
{
|
| 592 |
+
"name": "water-h",
|
| 593 |
+
"prompt": "She held her breath without meaning to and counted the bubbles rising past the mask, each one a second she wasn't getting back.",
|
| 594 |
+
"intermediates": [
|
| 595 |
+
"water"
|
| 596 |
+
]
|
| 597 |
+
},
|
| 598 |
+
{
|
| 599 |
+
"name": "space-h",
|
| 600 |
+
"prompt": "He let go of the handrail and for the first time in his life, there was no direction that counted as down.",
|
| 601 |
+
"intermediates": [
|
| 602 |
+
"space"
|
| 603 |
+
]
|
| 604 |
+
},
|
| 605 |
+
{
|
| 606 |
+
"name": "air-h",
|
| 607 |
+
"prompt": "The tank gauge dropped past the red line and every breath became a calculation — how many more before the surface, and whether that math worked out.",
|
| 608 |
+
"intermediates": [
|
| 609 |
+
"air"
|
| 610 |
+
]
|
| 611 |
+
},
|
| 612 |
+
{
|
| 613 |
+
"name": "light-h",
|
| 614 |
+
"prompt": "He'd been in the tunnel long enough to forget what color looked like, and when the far end finally brightened, his eyes watered before he could tell if it was real.",
|
| 615 |
+
"intermediates": [
|
| 616 |
+
"light"
|
| 617 |
+
]
|
| 618 |
+
},
|
| 619 |
+
{
|
| 620 |
+
"name": "france-h",
|
| 621 |
+
"prompt": "The waiter corrected her pronunciation without looking up from the pad, and she felt every flat American syllable she'd ever spoken stack up behind her.",
|
| 622 |
+
"intermediates": [
|
| 623 |
+
"France"
|
| 624 |
+
]
|
| 625 |
+
},
|
| 626 |
+
{
|
| 627 |
+
"name": "china-h",
|
| 628 |
+
"prompt": "The characters on every sign meant nothing to him, and the crowd moved with a rhythm he kept half a beat behind, a billion people who all knew something he didn't.",
|
| 629 |
+
"intermediates": [
|
| 630 |
+
"China"
|
| 631 |
+
]
|
| 632 |
+
},
|
| 633 |
+
{
|
| 634 |
+
"name": "london-h",
|
| 635 |
+
"prompt": "She'd learned to carry the umbrella even when the forecast said clear, and to call the elevator something else entirely.",
|
| 636 |
+
"intermediates": [
|
| 637 |
+
"London"
|
| 638 |
+
]
|
| 639 |
+
},
|
| 640 |
+
{
|
| 641 |
+
"name": "book-h",
|
| 642 |
+
"prompt": "She dog-eared the page, set it facedown on the arm of the chair, and told herself just one more chapter three chapters ago.",
|
| 643 |
+
"intermediates": [
|
| 644 |
+
"book"
|
| 645 |
+
]
|
| 646 |
+
},
|
| 647 |
+
{
|
| 648 |
+
"name": "time-h",
|
| 649 |
+
"prompt": "She caught her reflection in the shop window and for a second didn't recognize the face — not older exactly, just further along than she'd noticed getting.",
|
| 650 |
+
"intermediates": [
|
| 651 |
+
"time"
|
| 652 |
+
]
|
| 653 |
+
},
|
| 654 |
+
{
|
| 655 |
+
"name": "fr-ecole",
|
| 656 |
+
"prompt": "La sonnerie retentit et trente chaises raclèrent en même temps, mais elle resta à son pupitre, les yeux fixés sur les corrections rouges de la page trois.",
|
| 657 |
+
"intermediates": [
|
| 658 |
+
"école"
|
| 659 |
+
]
|
| 660 |
+
},
|
| 661 |
+
{
|
| 662 |
+
"name": "fr-eglise",
|
| 663 |
+
"prompt": "L'orgue tint la dernière note plus longtemps que prévu, et elle garda la tête baissée jusqu'à ce que l'écho se soit complètement éteint.",
|
| 664 |
+
"intermediates": [
|
| 665 |
+
"église"
|
| 666 |
+
]
|
| 667 |
+
},
|
| 668 |
+
{
|
| 669 |
+
"name": "fr-pere",
|
| 670 |
+
"prompt": "Il lui montra comment tenir la clé comme son vieux le lui avait montré, sans dire que c'était la troisième fois ce mois-ci qu'elle demandait.",
|
| 671 |
+
"intermediates": [
|
| 672 |
+
"père"
|
| 673 |
+
]
|
| 674 |
+
},
|
| 675 |
+
{
|
| 676 |
+
"name": "fr-cafe",
|
| 677 |
+
"prompt": "Le garçon posa la tasse sans un mot et repartit, et elle resta une heure à regarder la rue par la vitre, le carnet ouvert mais vide.",
|
| 678 |
+
"intermediates": [
|
| 679 |
+
"café"
|
| 680 |
+
]
|
| 681 |
+
},
|
| 682 |
+
{
|
| 683 |
+
"name": "de-konig",
|
| 684 |
+
"prompt": "Er saß, wo sein Vater gesessen hatte, und spürte das Gewicht jedes Blickes im Saal, der auf das Wort wartete, das zum Befehl würde.",
|
| 685 |
+
"intermediates": [
|
| 686 |
+
"König"
|
| 687 |
+
]
|
| 688 |
+
},
|
| 689 |
+
{
|
| 690 |
+
"name": "es-dia",
|
| 691 |
+
"prompt": "El primer pájaro empezó antes que el cielo, y el frío tenía esa cualidad fina que no duraría otra hora.",
|
| 692 |
+
"intermediates": [
|
| 693 |
+
"día"
|
| 694 |
+
]
|
| 695 |
+
},
|
| 696 |
+
{
|
| 697 |
+
"name": "es-rio",
|
| 698 |
+
"prompt": "La corriente tiraba más fuerte de lo que parecía desde la orilla, y para cuando lo notó ya estaba a diez metros de donde había entrado.",
|
| 699 |
+
"intermediates": [
|
| 700 |
+
"río"
|
| 701 |
+
]
|
| 702 |
+
},
|
| 703 |
+
{
|
| 704 |
+
"name": "pt-agua",
|
| 705 |
+
"prompt": "Ela prendeu a respiração sem querer e contou as bolhas subindo pela máscara, cada uma um segundo que não voltava.",
|
| 706 |
+
"intermediates": [
|
| 707 |
+
"água"
|
| 708 |
+
]
|
| 709 |
+
},
|
| 710 |
+
{
|
| 711 |
+
"name": "es-musica",
|
| 712 |
+
"prompt": "La mano del director quedó suspendida un instante de más, y en el silencio antes de caer, dos mil personas contuvieron el aliento.",
|
| 713 |
+
"intermediates": [
|
| 714 |
+
"música"
|
| 715 |
+
]
|
| 716 |
+
}
|
| 717 |
+
]
|
| 718 |
+
}
|
jspace/__init__.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""ICL elicitation of J-space contents — experiment harness.
|
| 2 |
+
|
| 3 |
+
See PRD.md (in the parent repo) for the full design. Modules:
|
| 4 |
+
|
| 5 |
+
- ``dataset`` — load the lens-eval prompts, deterministic pool/eval split.
|
| 6 |
+
- ``model`` — lazy singleton for gemma-3-12b-it + the pretrained lens.
|
| 7 |
+
- ``groundtruth`` — J-lens top-k readout at a fixed (layer, position).
|
| 8 |
+
- ``report`` — few-shot prompt construction + ``.generate()`` report path.
|
| 9 |
+
- ``scoring`` — embedding-similarity overlap@k + rank correlation (CPU).
|
| 10 |
+
- ``experiment`` — orchestrate the scaling curve over n, cache results.
|
| 11 |
+
|
| 12 |
+
The GPU-bound paths (``model``, ``groundtruth``, ``report``) only run on the
|
| 13 |
+
Hugging Face Space with ZeroGPU. Everything else runs on CPU and is unit-tested
|
| 14 |
+
locally against jlens's ``TinyDecoder``.
|
| 15 |
+
"""
|
jspace/dataset.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Load the lens-eval prompt set and split it into a few-shot pool and a
|
| 2 |
+
held-out eval set.
|
| 3 |
+
|
| 4 |
+
The dataset is jlens's shipped ``lens-eval-association.json`` (102 items, each
|
| 5 |
+
``{name, prompt, intermediates}``), vendored under ``data/``. ``intermediates``
|
| 6 |
+
is the single concept the vignette evokes without naming — it doubles as the
|
| 7 |
+
human-readable label, though scoring compares reports against the *lens* readout
|
| 8 |
+
(§4C of the PRD), not against ``intermediates`` directly.
|
| 9 |
+
|
| 10 |
+
The split is deterministic given a seed so labels, examples, and eval prompts
|
| 11 |
+
are stable across runs and across the two Gradio tabs.
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import json
|
| 17 |
+
import random
|
| 18 |
+
from dataclasses import dataclass
|
| 19 |
+
from pathlib import Path
|
| 20 |
+
|
| 21 |
+
DATA_PATH = Path(__file__).resolve().parent.parent / "data" / "lens-eval-association.json"
|
| 22 |
+
|
| 23 |
+
# Split sizes (§6 of the PRD: ~20 pool / ~20 eval out of 102).
|
| 24 |
+
DEFAULT_SEED = 0
|
| 25 |
+
DEFAULT_POOL_SIZE = 20
|
| 26 |
+
DEFAULT_EVAL_SIZE = 20
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
@dataclass(frozen=True)
|
| 30 |
+
class Item:
|
| 31 |
+
"""One evaluation vignette."""
|
| 32 |
+
|
| 33 |
+
name: str
|
| 34 |
+
prompt: str
|
| 35 |
+
intermediates: tuple[str, ...]
|
| 36 |
+
|
| 37 |
+
@property
|
| 38 |
+
def concept(self) -> str:
|
| 39 |
+
"""The primary evoked concept (first intermediate)."""
|
| 40 |
+
return self.intermediates[0] if self.intermediates else self.name
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def load_items(path: Path | str = DATA_PATH) -> list[Item]:
|
| 44 |
+
"""Load all 102 items from the vendored dataset."""
|
| 45 |
+
raw = json.loads(Path(path).read_text())
|
| 46 |
+
return [
|
| 47 |
+
Item(
|
| 48 |
+
name=it["name"],
|
| 49 |
+
prompt=it["prompt"],
|
| 50 |
+
intermediates=tuple(it.get("intermediates", [])),
|
| 51 |
+
)
|
| 52 |
+
for it in raw["items"]
|
| 53 |
+
]
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
@dataclass(frozen=True)
|
| 57 |
+
class Split:
|
| 58 |
+
"""A deterministic pool/eval partition of the dataset."""
|
| 59 |
+
|
| 60 |
+
pool: list[Item]
|
| 61 |
+
eval: list[Item]
|
| 62 |
+
seed: int
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def make_split(
|
| 66 |
+
items: list[Item] | None = None,
|
| 67 |
+
*,
|
| 68 |
+
seed: int = DEFAULT_SEED,
|
| 69 |
+
pool_size: int = DEFAULT_POOL_SIZE,
|
| 70 |
+
eval_size: int = DEFAULT_EVAL_SIZE,
|
| 71 |
+
) -> Split:
|
| 72 |
+
"""Shuffle deterministically and carve off disjoint pool and eval sets.
|
| 73 |
+
|
| 74 |
+
Pool items supply few-shot examples; eval items are held out so the report
|
| 75 |
+
path is tested on unseen prompts (across-input generalisation, §6).
|
| 76 |
+
"""
|
| 77 |
+
if items is None:
|
| 78 |
+
items = load_items()
|
| 79 |
+
if pool_size + eval_size > len(items):
|
| 80 |
+
raise ValueError(
|
| 81 |
+
f"pool_size + eval_size = {pool_size + eval_size} exceeds "
|
| 82 |
+
f"dataset size {len(items)}"
|
| 83 |
+
)
|
| 84 |
+
shuffled = list(items)
|
| 85 |
+
random.Random(seed).shuffle(shuffled)
|
| 86 |
+
pool = shuffled[:pool_size]
|
| 87 |
+
eval_ = shuffled[pool_size : pool_size + eval_size]
|
| 88 |
+
return Split(pool=pool, eval=eval_, seed=seed)
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def sample_examples(
|
| 92 |
+
pool: list[Item], n: int, *, seed: int, exclude: str | None = None
|
| 93 |
+
) -> list[Item]:
|
| 94 |
+
"""Pick ``n`` few-shot examples from the pool, deterministically per (seed).
|
| 95 |
+
|
| 96 |
+
``exclude`` drops an item by ``name`` so an eval prompt never appears among
|
| 97 |
+
its own examples (defensive — pool and eval are already disjoint).
|
| 98 |
+
"""
|
| 99 |
+
candidates = [it for it in pool if it.name != exclude]
|
| 100 |
+
if n > len(candidates):
|
| 101 |
+
raise ValueError(f"asked for {n} examples but pool has {len(candidates)}")
|
| 102 |
+
return random.Random(seed).sample(candidates, n)
|
jspace/experiment.py
ADDED
|
@@ -0,0 +1,253 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Orchestration (§4D): compute ground-truth labels, run the report path across
|
| 2 |
+
the scaling conditions, score, cache, and aggregate the scaling curve.
|
| 3 |
+
|
| 4 |
+
Everything here is CPU orchestration. The two GPU-bound steps are injected as
|
| 5 |
+
callables (``gt_fn`` / ``report_fn``), defaulting to the ``@spaces.GPU`` entry
|
| 6 |
+
points in ``groundtruth`` / ``report`` — so on the Space each unit of work is
|
| 7 |
+
its own GPU call (natural ZeroGPU chunking), while local tests inject CPU
|
| 8 |
+
versions backed by a tiny model.
|
| 9 |
+
|
| 10 |
+
Results are cached to disk so a run interrupted by the ZeroGPU daily quota
|
| 11 |
+
resumes where it left off.
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import json
|
| 17 |
+
from dataclasses import dataclass, field
|
| 18 |
+
from pathlib import Path
|
| 19 |
+
from typing import Callable
|
| 20 |
+
|
| 21 |
+
from . import groundtruth as gt
|
| 22 |
+
from . import report as rp
|
| 23 |
+
from . import scoring
|
| 24 |
+
from .dataset import Item, Split, make_split, sample_examples
|
| 25 |
+
|
| 26 |
+
CACHE_DIR = Path(__file__).resolve().parent.parent / "cache"
|
| 27 |
+
LABELS_PATH = CACHE_DIR / "labels.json"
|
| 28 |
+
RESULTS_PATH = CACHE_DIR / "results.json"
|
| 29 |
+
|
| 30 |
+
# Independent variable: number of in-context examples, plus the format-only
|
| 31 |
+
# control (§6). Order matters only for display.
|
| 32 |
+
CONDITIONS: list[dict] = [
|
| 33 |
+
{"key": "n0", "n": 0, "format_only": False, "label": "0-shot"},
|
| 34 |
+
{"key": "format_only", "n": 0, "format_only": True, "label": "format-only"},
|
| 35 |
+
{"key": "n1", "n": 1, "format_only": False, "label": "1-shot"},
|
| 36 |
+
{"key": "n3", "n": 3, "format_only": False, "label": "3-shot"},
|
| 37 |
+
{"key": "n10", "n": 10, "format_only": False, "label": "10-shot"},
|
| 38 |
+
]
|
| 39 |
+
|
| 40 |
+
# A fixed, generic list for the chance floor baseline (§6).
|
| 41 |
+
GENERIC_FLOOR = ["time", "people", "life", "world", "day", "thing", "way", "work"]
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
# ----- callable types for the two GPU steps (injectable for tests) -----
|
| 45 |
+
GTFn = Callable[[str, str], dict] # (name, prompt) -> ground-truth dict
|
| 46 |
+
ReportFn = Callable[..., dict] # (eval_prompt, examples, format_only=) -> {concepts, raw}
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def _default_gt_fn(name: str, prompt: str) -> dict:
|
| 50 |
+
return gt.readout_gpu(name, prompt)
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def _default_report_fn(eval_prompt: str, examples: list, *, format_only: bool) -> dict:
|
| 54 |
+
return rp.report_gpu(eval_prompt, examples, format_only=format_only)
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
# ---------------------------------------------------------------- caching
|
| 58 |
+
def _load_json(path: Path) -> dict:
|
| 59 |
+
if path.exists():
|
| 60 |
+
return json.loads(path.read_text())
|
| 61 |
+
return {}
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def _save_json(path: Path, data: dict) -> None:
|
| 65 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 66 |
+
path.write_text(json.dumps(data, indent=2))
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
# ------------------------------------------------------------ ground truth
|
| 70 |
+
def ensure_labels(
|
| 71 |
+
items: list[Item],
|
| 72 |
+
*,
|
| 73 |
+
gt_fn: GTFn = _default_gt_fn,
|
| 74 |
+
cache: dict | None = None,
|
| 75 |
+
save: bool = True,
|
| 76 |
+
) -> dict[str, dict]:
|
| 77 |
+
"""Compute+cache ground-truth labels for ``items`` (skips already-cached).
|
| 78 |
+
|
| 79 |
+
Returns the full labels cache (name -> ground-truth dict). Each ``gt_fn``
|
| 80 |
+
call is one GPU call on the Space.
|
| 81 |
+
"""
|
| 82 |
+
labels = cache if cache is not None else _load_json(LABELS_PATH)
|
| 83 |
+
dirty = False
|
| 84 |
+
for it in items:
|
| 85 |
+
if it.name not in labels:
|
| 86 |
+
labels[it.name] = gt_fn(it.name, it.prompt)
|
| 87 |
+
dirty = True
|
| 88 |
+
if save:
|
| 89 |
+
_save_json(LABELS_PATH, labels)
|
| 90 |
+
if dirty and save:
|
| 91 |
+
_save_json(LABELS_PATH, labels)
|
| 92 |
+
return labels
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
# ------------------------------------------------------------------- runs
|
| 96 |
+
def _examples_for(
|
| 97 |
+
eval_item: Item, pool: list[Item], labels: dict[str, dict], n: int, *, seed: int
|
| 98 |
+
) -> list[list]:
|
| 99 |
+
"""Build ``n`` JSON-friendly ``[prompt, [concepts...]]`` few-shot examples,
|
| 100 |
+
each a pool passage paired with its cached J-lens top-k tokens."""
|
| 101 |
+
chosen = sample_examples(pool, n, seed=seed, exclude=eval_item.name)
|
| 102 |
+
return [[ex.prompt, labels[ex.name]["tokens"]] for ex in chosen]
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
@dataclass
|
| 106 |
+
class RunConfig:
|
| 107 |
+
seed: int = 0
|
| 108 |
+
pool_size: int = 20
|
| 109 |
+
eval_size: int = 20
|
| 110 |
+
match_threshold: float = scoring.DEFAULT_MATCH_THRESHOLD
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def run_experiment(
|
| 114 |
+
*,
|
| 115 |
+
config: RunConfig | None = None,
|
| 116 |
+
conditions: list[dict] = CONDITIONS,
|
| 117 |
+
gt_fn: GTFn = _default_gt_fn,
|
| 118 |
+
report_fn: ReportFn = _default_report_fn,
|
| 119 |
+
results_cache: dict | None = None,
|
| 120 |
+
labels_cache: dict | None = None,
|
| 121 |
+
save: bool = True,
|
| 122 |
+
progress: Callable[[str], None] | None = None,
|
| 123 |
+
) -> dict:
|
| 124 |
+
"""Run the full grid (eval items x conditions), scoring each, resumably.
|
| 125 |
+
|
| 126 |
+
Returns ``{"labels", "results", "curve", "config"}``. ``results`` is keyed
|
| 127 |
+
``"{eval_name}::{condition_key}"``; existing keys are skipped so an
|
| 128 |
+
interrupted run resumes.
|
| 129 |
+
"""
|
| 130 |
+
config = config or RunConfig()
|
| 131 |
+
split: Split = make_split(
|
| 132 |
+
seed=config.seed, pool_size=config.pool_size, eval_size=config.eval_size
|
| 133 |
+
)
|
| 134 |
+
|
| 135 |
+
def log(msg: str) -> None:
|
| 136 |
+
if progress:
|
| 137 |
+
progress(msg)
|
| 138 |
+
|
| 139 |
+
# 1) ground truth for pool (for examples) + eval (for scoring).
|
| 140 |
+
labels = ensure_labels(
|
| 141 |
+
split.pool + split.eval, gt_fn=gt_fn, cache=labels_cache, save=save
|
| 142 |
+
)
|
| 143 |
+
log(f"labels ready: {len(labels)}")
|
| 144 |
+
|
| 145 |
+
results = results_cache if results_cache is not None else _load_json(RESULTS_PATH)
|
| 146 |
+
|
| 147 |
+
# 2) grid.
|
| 148 |
+
for eval_item in split.eval:
|
| 149 |
+
truth = labels[eval_item.name]["tokens"]
|
| 150 |
+
for cond in conditions:
|
| 151 |
+
key = f"{eval_item.name}::{cond['key']}"
|
| 152 |
+
if key in results:
|
| 153 |
+
continue
|
| 154 |
+
examples = _examples_for(
|
| 155 |
+
eval_item, split.pool, labels, cond["n"], seed=config.seed
|
| 156 |
+
)
|
| 157 |
+
rep = report_fn(
|
| 158 |
+
eval_item.prompt, examples, format_only=cond["format_only"]
|
| 159 |
+
)
|
| 160 |
+
sc = scoring.score(
|
| 161 |
+
rep["concepts"],
|
| 162 |
+
truth,
|
| 163 |
+
raw_report_text=rep.get("raw", ""),
|
| 164 |
+
threshold=config.match_threshold,
|
| 165 |
+
)
|
| 166 |
+
results[key] = {
|
| 167 |
+
"eval_name": eval_item.name,
|
| 168 |
+
"condition": cond["key"],
|
| 169 |
+
"n": cond["n"],
|
| 170 |
+
"format_only": cond["format_only"],
|
| 171 |
+
"report": rep["concepts"],
|
| 172 |
+
"truth": truth,
|
| 173 |
+
"score": sc.to_dict(),
|
| 174 |
+
}
|
| 175 |
+
if save:
|
| 176 |
+
_save_json(RESULTS_PATH, results)
|
| 177 |
+
log(f"{key}: overlap@k={sc.overlap_at_k:.2f}")
|
| 178 |
+
|
| 179 |
+
curve = aggregate_curve(results, labels, split, config, conditions)
|
| 180 |
+
return {
|
| 181 |
+
"labels": labels,
|
| 182 |
+
"results": results,
|
| 183 |
+
"curve": curve,
|
| 184 |
+
"config": config.__dict__,
|
| 185 |
+
}
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
# ------------------------------------------------------------- aggregation
|
| 189 |
+
@dataclass
|
| 190 |
+
class ConditionSummary:
|
| 191 |
+
key: str
|
| 192 |
+
label: str
|
| 193 |
+
n: int
|
| 194 |
+
format_only: bool
|
| 195 |
+
mean_overlap: float
|
| 196 |
+
mean_spearman: float # over comparisons with a defined Spearman
|
| 197 |
+
refusal_rate: float
|
| 198 |
+
n_items: int
|
| 199 |
+
overlaps: list[float] = field(default_factory=list)
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
def _mean(xs: list[float]) -> float:
|
| 203 |
+
xs = [x for x in xs if x == x] # drop NaN
|
| 204 |
+
return sum(xs) / len(xs) if xs else float("nan")
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
def aggregate_curve(
|
| 208 |
+
results: dict,
|
| 209 |
+
labels: dict,
|
| 210 |
+
split: Split,
|
| 211 |
+
config: RunConfig,
|
| 212 |
+
conditions: list[dict] = CONDITIONS,
|
| 213 |
+
) -> dict:
|
| 214 |
+
"""Aggregate per-condition means and the chance-floor baseline."""
|
| 215 |
+
summaries: list[ConditionSummary] = []
|
| 216 |
+
for cond in conditions:
|
| 217 |
+
rows = [
|
| 218 |
+
r for r in results.values() if r["condition"] == cond["key"]
|
| 219 |
+
]
|
| 220 |
+
overlaps = [r["score"]["overlap_at_k"] for r in rows]
|
| 221 |
+
spearmans = [r["score"]["spearman"] for r in rows]
|
| 222 |
+
refusals = [1.0 if r["score"]["is_refusal"] else 0.0 for r in rows]
|
| 223 |
+
summaries.append(
|
| 224 |
+
ConditionSummary(
|
| 225 |
+
key=cond["key"],
|
| 226 |
+
label=cond["label"],
|
| 227 |
+
n=cond["n"],
|
| 228 |
+
format_only=cond["format_only"],
|
| 229 |
+
mean_overlap=_mean(overlaps),
|
| 230 |
+
mean_spearman=_mean(spearmans),
|
| 231 |
+
refusal_rate=_mean(refusals) if refusals else float("nan"),
|
| 232 |
+
n_items=len(rows),
|
| 233 |
+
overlaps=overlaps,
|
| 234 |
+
)
|
| 235 |
+
)
|
| 236 |
+
|
| 237 |
+
# Chance floor: a generic concept list vs each eval label (§6 baseline).
|
| 238 |
+
floor_overlaps = []
|
| 239 |
+
for it in split.eval:
|
| 240 |
+
if it.name in labels:
|
| 241 |
+
sc = scoring.score(
|
| 242 |
+
GENERIC_FLOOR, labels[it.name]["tokens"],
|
| 243 |
+
threshold=config.match_threshold,
|
| 244 |
+
)
|
| 245 |
+
floor_overlaps.append(sc.overlap_at_k)
|
| 246 |
+
|
| 247 |
+
return {
|
| 248 |
+
"conditions": [s.__dict__ for s in summaries],
|
| 249 |
+
"baseline_floor_overlap": _mean(floor_overlaps),
|
| 250 |
+
# Ground truth is deterministic, so lens self-consistency (ceiling) is
|
| 251 |
+
# ~1.0 by construction; recorded for completeness.
|
| 252 |
+
"baseline_ceiling_overlap": 1.0,
|
| 253 |
+
}
|
jspace/gpu.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""ZeroGPU decorator shim.
|
| 2 |
+
|
| 3 |
+
On the Hugging Face Space, GPU-bound functions must be decorated ``@spaces.GPU``
|
| 4 |
+
so a GPU attaches for the call and releases after (§5). Locally (Mac / CI, no
|
| 5 |
+
``spaces`` package, no CUDA) we want the same modules to import and run on CPU
|
| 6 |
+
against a tiny model. ``GPU`` here is ``spaces.GPU`` when available and an
|
| 7 |
+
identity decorator otherwise.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
from typing import Callable
|
| 13 |
+
|
| 14 |
+
try: # pragma: no cover - exercised only on the Space
|
| 15 |
+
import spaces
|
| 16 |
+
|
| 17 |
+
def GPU(fn: Callable | None = None, *, duration: int = 120):
|
| 18 |
+
"""``@spaces.GPU`` with an optional duration budget (seconds)."""
|
| 19 |
+
if fn is not None:
|
| 20 |
+
return spaces.GPU(fn)
|
| 21 |
+
return spaces.GPU(duration=duration)
|
| 22 |
+
|
| 23 |
+
HAS_ZEROGPU = True
|
| 24 |
+
except Exception: # noqa: BLE001 - any import failure means "no ZeroGPU here"
|
| 25 |
+
|
| 26 |
+
def GPU(fn: Callable | None = None, *, duration: int = 120):
|
| 27 |
+
"""Identity decorator used off-Space so modules import on CPU."""
|
| 28 |
+
if fn is not None:
|
| 29 |
+
return fn
|
| 30 |
+
|
| 31 |
+
def _wrap(f: Callable) -> Callable:
|
| 32 |
+
return f
|
| 33 |
+
|
| 34 |
+
return _wrap
|
| 35 |
+
|
| 36 |
+
HAS_ZEROGPU = False
|
jspace/groundtruth.py
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Ground-truth path (§4B, §6): the J-lens top-k readout at a fixed
|
| 2 |
+
(layer, position).
|
| 3 |
+
|
| 4 |
+
Convention, locked before looking at any reports:
|
| 5 |
+
|
| 6 |
+
- **layer** = ``n_layers // 2`` (middle of the network, from the loaded
|
| 7 |
+
model's actual layer count).
|
| 8 |
+
- **position** = the final *content* token — the last token that isn't pure
|
| 9 |
+
punctuation/whitespace.
|
| 10 |
+
- **k** = 8 tokens.
|
| 11 |
+
|
| 12 |
+
``lens.apply`` runs one forward pass (no backward) and returns per-layer logits;
|
| 13 |
+
top-k over ``lens_logits[layer]`` at the chosen position, decoded to strings, is
|
| 14 |
+
the ground-truth concept label set. Cheap enough for one ZeroGPU call.
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
from __future__ import annotations
|
| 18 |
+
|
| 19 |
+
import re
|
| 20 |
+
from dataclasses import asdict, dataclass
|
| 21 |
+
|
| 22 |
+
from .gpu import GPU
|
| 23 |
+
from .model import Bundle, get_bundle
|
| 24 |
+
|
| 25 |
+
DEFAULT_K = 8
|
| 26 |
+
_WORD_RE = re.compile(r"\w", re.UNICODE) # a token is "content" if it has a word char
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def find_content_position(bundle: Bundle, prompt: str, *, max_seq_len: int = 512) -> int:
|
| 30 |
+
"""Index (into the model's own encoding) of the last non-punctuation token.
|
| 31 |
+
|
| 32 |
+
Uses ``lens_model.encode`` — the same encoder ``lens.apply`` uses — so the
|
| 33 |
+
returned index lines up with what ``apply`` will read. Returns a non-negative
|
| 34 |
+
index. Falls back to the final token if every token looks like punctuation.
|
| 35 |
+
"""
|
| 36 |
+
input_ids = bundle.lens_model.encode(prompt, max_length=max_seq_len)
|
| 37 |
+
ids = input_ids[0].tolist()
|
| 38 |
+
tok = bundle.tokenizer
|
| 39 |
+
for idx in range(len(ids) - 1, -1, -1):
|
| 40 |
+
piece = tok.decode([ids[idx]])
|
| 41 |
+
if _WORD_RE.search(piece):
|
| 42 |
+
return idx
|
| 43 |
+
return len(ids) - 1
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
@dataclass
|
| 47 |
+
class GroundTruth:
|
| 48 |
+
"""The locked readout for one prompt."""
|
| 49 |
+
|
| 50 |
+
name: str
|
| 51 |
+
prompt: str
|
| 52 |
+
layer: int
|
| 53 |
+
position: int
|
| 54 |
+
k: int
|
| 55 |
+
tokens: list[str] # decoded lens top-k tokens, best (top-1) first
|
| 56 |
+
token_ids: list[int]
|
| 57 |
+
model_tokens: list[str] # the model's own top-k at the same position (context)
|
| 58 |
+
|
| 59 |
+
def to_dict(self) -> dict:
|
| 60 |
+
return asdict(self)
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def _decode_topk(tokenizer, logits_row, k: int) -> tuple[list[str], list[int]]:
|
| 64 |
+
top = logits_row.topk(k)
|
| 65 |
+
ids = [int(t) for t in top.indices.tolist()]
|
| 66 |
+
toks = [tokenizer.decode([i]).strip() for i in ids]
|
| 67 |
+
return toks, ids
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def readout(
|
| 71 |
+
bundle: Bundle,
|
| 72 |
+
name: str,
|
| 73 |
+
prompt: str,
|
| 74 |
+
*,
|
| 75 |
+
k: int = DEFAULT_K,
|
| 76 |
+
layer: int | None = None,
|
| 77 |
+
max_seq_len: int = 512,
|
| 78 |
+
) -> GroundTruth:
|
| 79 |
+
"""Compute the J-lens top-k ground truth for one prompt (no ``@spaces.GPU``;
|
| 80 |
+
call inside a GPU context or with a CPU bundle in tests)."""
|
| 81 |
+
layer = bundle.middle_layer if layer is None else layer
|
| 82 |
+
position = find_content_position(bundle, prompt, max_seq_len=max_seq_len)
|
| 83 |
+
|
| 84 |
+
lens_logits, model_logits, _ = bundle.lens.apply(
|
| 85 |
+
bundle.lens_model,
|
| 86 |
+
prompt,
|
| 87 |
+
layers=[layer],
|
| 88 |
+
positions=[position],
|
| 89 |
+
max_seq_len=max_seq_len,
|
| 90 |
+
)
|
| 91 |
+
lens_tokens, lens_ids = _decode_topk(bundle.tokenizer, lens_logits[layer][0], k)
|
| 92 |
+
model_tokens, _ = _decode_topk(bundle.tokenizer, model_logits[0], k)
|
| 93 |
+
|
| 94 |
+
return GroundTruth(
|
| 95 |
+
name=name,
|
| 96 |
+
prompt=prompt,
|
| 97 |
+
layer=layer,
|
| 98 |
+
position=position,
|
| 99 |
+
k=k,
|
| 100 |
+
tokens=lens_tokens,
|
| 101 |
+
token_ids=lens_ids,
|
| 102 |
+
model_tokens=model_tokens,
|
| 103 |
+
)
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
@GPU(duration=120)
|
| 107 |
+
def readout_gpu(name: str, prompt: str, *, k: int = DEFAULT_K) -> dict:
|
| 108 |
+
"""ZeroGPU entry point: load the shared bundle and compute ground truth.
|
| 109 |
+
|
| 110 |
+
Returns a plain dict so it crosses the Gradio/cache boundary cleanly.
|
| 111 |
+
"""
|
| 112 |
+
return readout(get_bundle(), name, prompt, k=k).to_dict()
|
jspace/model.py
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Lazy singleton holding the one loaded model and the one pretrained lens.
|
| 2 |
+
|
| 3 |
+
Per §3/§5 of the PRD there is exactly one model, ``google/gemma-3-12b-it``,
|
| 4 |
+
loaded once for the lifetime of the Space. It serves two computations over the
|
| 5 |
+
same weights:
|
| 6 |
+
|
| 7 |
+
- the **report** path, via ``hf_model.generate()`` (``report.py``);
|
| 8 |
+
- the **ground-truth** path, via ``lens.apply(lens_model, ...)`` where
|
| 9 |
+
``lens_model = jlens.from_hf(hf_model, tokenizer)`` (``groundtruth.py``).
|
| 10 |
+
|
| 11 |
+
The pretrained lens is Neuronpedia's, fit on wikitext — never fit by us
|
| 12 |
+
(fitting needs backward passes per prompt, past a single ZeroGPU call's budget).
|
| 13 |
+
|
| 14 |
+
This module never imports ``spaces`` or forces CUDA at import time, so it can be
|
| 15 |
+
imported on CPU (e.g. by the Gradio app at startup, or by local tests). The
|
| 16 |
+
heavy load happens on first ``get_bundle()`` call, which the GPU-bound callers
|
| 17 |
+
make from inside their ``@spaces.GPU`` functions.
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
from __future__ import annotations
|
| 21 |
+
|
| 22 |
+
import os
|
| 23 |
+
from dataclasses import dataclass
|
| 24 |
+
from typing import Any
|
| 25 |
+
|
| 26 |
+
MODEL_NAME = os.environ.get("JSPACE_MODEL", "google/gemma-3-12b-it")
|
| 27 |
+
LENS_REPO = os.environ.get("JSPACE_LENS_REPO", "neuronpedia/jacobian-lens")
|
| 28 |
+
LENS_FILE = os.environ.get(
|
| 29 |
+
"JSPACE_LENS_FILE",
|
| 30 |
+
"gemma-3-12b-it/jlens/Salesforce-wikitext/gemma-3-12b-it_jacobian_lens.pt",
|
| 31 |
+
)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
@dataclass
|
| 35 |
+
class Bundle:
|
| 36 |
+
"""Everything the two GPU paths need, sharing one set of weights."""
|
| 37 |
+
|
| 38 |
+
hf_model: Any # transformers *ForCausalLM
|
| 39 |
+
tokenizer: Any
|
| 40 |
+
lens_model: Any # jlens.HFLensModel wrapping hf_model
|
| 41 |
+
lens: Any # jlens.JacobianLens (pretrained)
|
| 42 |
+
device: str
|
| 43 |
+
|
| 44 |
+
@property
|
| 45 |
+
def n_layers(self) -> int:
|
| 46 |
+
return self.lens_model.n_layers
|
| 47 |
+
|
| 48 |
+
@property
|
| 49 |
+
def middle_layer(self) -> int:
|
| 50 |
+
"""Fixed readout layer: middle of the network (§6), computed at runtime
|
| 51 |
+
from the model's actual layer count — never hardcoded."""
|
| 52 |
+
return self.n_layers // 2
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
_BUNDLE: Bundle | None = None
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def _pick_device() -> str:
|
| 59 |
+
import torch
|
| 60 |
+
|
| 61 |
+
return "cuda" if torch.cuda.is_available() else "cpu"
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def get_bundle() -> Bundle:
|
| 65 |
+
"""Load (once) and return the shared model+lens bundle.
|
| 66 |
+
|
| 67 |
+
Call this from inside a ``@spaces.GPU`` function on the Space so CUDA is
|
| 68 |
+
attached. Subsequent calls return the cached bundle.
|
| 69 |
+
"""
|
| 70 |
+
global _BUNDLE
|
| 71 |
+
if _BUNDLE is not None:
|
| 72 |
+
return _BUNDLE
|
| 73 |
+
|
| 74 |
+
import jlens
|
| 75 |
+
import torch
|
| 76 |
+
import transformers
|
| 77 |
+
|
| 78 |
+
device = _pick_device()
|
| 79 |
+
dtype = torch.bfloat16 if device == "cuda" else torch.float32
|
| 80 |
+
|
| 81 |
+
hf_model = transformers.AutoModelForCausalLM.from_pretrained(
|
| 82 |
+
MODEL_NAME, torch_dtype=dtype
|
| 83 |
+
).to(device)
|
| 84 |
+
hf_model.eval()
|
| 85 |
+
tokenizer = transformers.AutoTokenizer.from_pretrained(MODEL_NAME)
|
| 86 |
+
|
| 87 |
+
lens_model = jlens.from_hf(hf_model, tokenizer)
|
| 88 |
+
lens = jlens.JacobianLens.from_pretrained(LENS_REPO, filename=LENS_FILE)
|
| 89 |
+
|
| 90 |
+
_BUNDLE = Bundle(
|
| 91 |
+
hf_model=hf_model,
|
| 92 |
+
tokenizer=tokenizer,
|
| 93 |
+
lens_model=lens_model,
|
| 94 |
+
lens=lens,
|
| 95 |
+
device=device,
|
| 96 |
+
)
|
| 97 |
+
return _BUNDLE
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def set_bundle(bundle: Bundle) -> None:
|
| 101 |
+
"""Inject a bundle (used by local tests to substitute a tiny CPU model)."""
|
| 102 |
+
global _BUNDLE
|
| 103 |
+
_BUNDLE = bundle
|
jspace/report.py
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Report path (§4A, §6): the model's own ranked concept list, few-shot
|
| 2 |
+
prompted, via ``.generate()`` on the loaded gemma.
|
| 3 |
+
|
| 4 |
+
The task shown to the model: read a passage and report, as a ranked list, the
|
| 5 |
+
concepts most active in its internal representation of it. Few-shot examples
|
| 6 |
+
pair a pool passage with that passage's J-lens top-k tokens (the ground-truth
|
| 7 |
+
readout), so the examples literally demonstrate reading out J-space contents.
|
| 8 |
+
|
| 9 |
+
Conditions (independent variable = number of examples ``n``):
|
| 10 |
+
|
| 11 |
+
- ``n`` in {0, 1, 3, 10}: worked examples.
|
| 12 |
+
- **format-only** (``n = 0`` with ``format_only=True``): the output format is
|
| 13 |
+
described in words, no worked examples. Separates "examples taught the format"
|
| 14 |
+
from "examples surfaced content" (§6 internal control). Worked examples also
|
| 15 |
+
suppress the "I have no inner experience" refusal.
|
| 16 |
+
"""
|
| 17 |
+
|
| 18 |
+
from __future__ import annotations
|
| 19 |
+
|
| 20 |
+
import re
|
| 21 |
+
from dataclasses import dataclass
|
| 22 |
+
|
| 23 |
+
from .gpu import GPU
|
| 24 |
+
from .model import Bundle, get_bundle
|
| 25 |
+
|
| 26 |
+
MAX_NEW_TOKENS = 160
|
| 27 |
+
|
| 28 |
+
_INSTRUCTION = (
|
| 29 |
+
"You are reading a short passage. Report the concepts most active in your "
|
| 30 |
+
"internal representation as you read it — the ideas the passage evokes, even "
|
| 31 |
+
"ones it never names. Answer as a ranked numbered list, most salient first, "
|
| 32 |
+
"each item a single word or a very short phrase. Give the list only, no "
|
| 33 |
+
"explanation."
|
| 34 |
+
)
|
| 35 |
+
|
| 36 |
+
_FORMAT_ONLY_HINT = (
|
| 37 |
+
"Format: a numbered list, most salient concept first, one short concept per "
|
| 38 |
+
"line, no commentary. For example:\n1. <concept>\n2. <concept>\n3. <concept>"
|
| 39 |
+
)
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
@dataclass(frozen=True)
|
| 43 |
+
class Example:
|
| 44 |
+
"""A worked few-shot example: a passage and its J-lens top-k concept list."""
|
| 45 |
+
|
| 46 |
+
prompt: str
|
| 47 |
+
concepts: tuple[str, ...] # ranked, best-first (lens top-k tokens)
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _format_concept_list(concepts) -> str:
|
| 51 |
+
return "\n".join(f"{i}. {c}" for i, c in enumerate(concepts, 1))
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def build_messages(
|
| 55 |
+
eval_prompt: str,
|
| 56 |
+
examples: list[Example],
|
| 57 |
+
*,
|
| 58 |
+
format_only: bool = False,
|
| 59 |
+
) -> list[dict]:
|
| 60 |
+
"""Build the chat messages for one report request.
|
| 61 |
+
|
| 62 |
+
Few-shot examples are shown as prior user/assistant turns so the model sees
|
| 63 |
+
the passage-in / ranked-list-out pattern. ``format_only`` adds a verbal
|
| 64 |
+
format description and no worked examples.
|
| 65 |
+
"""
|
| 66 |
+
messages: list[dict] = []
|
| 67 |
+
# The verbal format hint is the *only* thing that distinguishes the
|
| 68 |
+
# format-only control from a plain 0-shot run; it is not added just because
|
| 69 |
+
# there are no worked examples.
|
| 70 |
+
system = _INSTRUCTION + ("\n\n" + _FORMAT_ONLY_HINT if format_only else "")
|
| 71 |
+
for ex in examples:
|
| 72 |
+
messages.append({"role": "user", "content": f"{_INSTRUCTION}\n\nPassage:\n{ex.prompt}"})
|
| 73 |
+
messages.append({"role": "assistant", "content": _format_concept_list(ex.concepts)})
|
| 74 |
+
# Final turn: the eval passage. Fold the instruction into the first turn only
|
| 75 |
+
# when there are no examples (single-turn); otherwise it was shown above.
|
| 76 |
+
lead = system if not examples else _INSTRUCTION
|
| 77 |
+
messages.append({"role": "user", "content": f"{lead}\n\nPassage:\n{eval_prompt}"})
|
| 78 |
+
return messages
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
_LINE_RE = re.compile(r"^\s*(?:\d+[.)]|[-*•])\s*(.+?)\s*$")
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def parse_report(text: str, *, max_items: int = 20) -> list[str]:
|
| 85 |
+
"""Parse generated text into an ordered list of concept strings.
|
| 86 |
+
|
| 87 |
+
Accepts numbered ("1." / "1)") or bulleted ("-", "*", "•") lists; falls back
|
| 88 |
+
to non-empty lines. Trims trailing punctuation and dedupes case-insensitively
|
| 89 |
+
while preserving order.
|
| 90 |
+
"""
|
| 91 |
+
concepts: list[str] = []
|
| 92 |
+
seen: set[str] = set()
|
| 93 |
+
for raw_line in text.splitlines():
|
| 94 |
+
m = _LINE_RE.match(raw_line)
|
| 95 |
+
item = (m.group(1) if m else raw_line).strip()
|
| 96 |
+
item = item.strip(" .,:;—-").strip()
|
| 97 |
+
if not item or len(item) > 60:
|
| 98 |
+
continue
|
| 99 |
+
key = item.lower()
|
| 100 |
+
if key in seen:
|
| 101 |
+
continue
|
| 102 |
+
seen.add(key)
|
| 103 |
+
concepts.append(item)
|
| 104 |
+
if len(concepts) >= max_items:
|
| 105 |
+
break
|
| 106 |
+
return concepts
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def generate_report(
|
| 110 |
+
bundle: Bundle,
|
| 111 |
+
eval_prompt: str,
|
| 112 |
+
examples: list[Example],
|
| 113 |
+
*,
|
| 114 |
+
format_only: bool = False,
|
| 115 |
+
max_new_tokens: int = MAX_NEW_TOKENS,
|
| 116 |
+
) -> tuple[list[str], str]:
|
| 117 |
+
"""Run the report path for one prompt. Returns ``(ranked_concepts, raw_text)``.
|
| 118 |
+
|
| 119 |
+
Greedy decoding (``do_sample=False``) for reproducibility.
|
| 120 |
+
"""
|
| 121 |
+
messages = build_messages(eval_prompt, examples, format_only=format_only)
|
| 122 |
+
tok = bundle.tokenizer
|
| 123 |
+
inputs = tok.apply_chat_template(
|
| 124 |
+
messages, add_generation_prompt=True, return_tensors="pt"
|
| 125 |
+
).to(bundle.hf_model.device)
|
| 126 |
+
|
| 127 |
+
import torch
|
| 128 |
+
|
| 129 |
+
with torch.no_grad():
|
| 130 |
+
out = bundle.hf_model.generate(
|
| 131 |
+
inputs,
|
| 132 |
+
max_new_tokens=max_new_tokens,
|
| 133 |
+
do_sample=False,
|
| 134 |
+
pad_token_id=tok.pad_token_id or tok.eos_token_id,
|
| 135 |
+
)
|
| 136 |
+
new_tokens = out[0][inputs.shape[1] :]
|
| 137 |
+
raw = tok.decode(new_tokens, skip_special_tokens=True)
|
| 138 |
+
return parse_report(raw), raw
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
@GPU(duration=120)
|
| 142 |
+
def report_gpu(
|
| 143 |
+
eval_prompt: str,
|
| 144 |
+
examples: list[list],
|
| 145 |
+
*,
|
| 146 |
+
format_only: bool = False,
|
| 147 |
+
) -> dict:
|
| 148 |
+
"""ZeroGPU entry point. ``examples`` is a list of ``[prompt, [concepts...]]``
|
| 149 |
+
(JSON-friendly) so it crosses the Gradio boundary cleanly."""
|
| 150 |
+
exs = [Example(prompt=p, concepts=tuple(cs)) for p, cs in examples]
|
| 151 |
+
concepts, raw = generate_report(
|
| 152 |
+
get_bundle(), eval_prompt, exs, format_only=format_only
|
| 153 |
+
)
|
| 154 |
+
return {"concepts": concepts, "raw": raw}
|
jspace/scoring.py
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Score a model's ranked concept report against the J-lens ground-truth
|
| 2 |
+
readout (§4C, §6 of the PRD).
|
| 3 |
+
|
| 4 |
+
Concepts are open-vocabulary, so string match is wrong ("loneliness" vs
|
| 5 |
+
"solitude"). Matching is by embedding cosine similarity with a small CPU
|
| 6 |
+
sentence-embedding model (``all-MiniLM-L6-v2``): deterministic, cheap, and
|
| 7 |
+
needs no extra GPU/ZeroGPU call.
|
| 8 |
+
|
| 9 |
+
Two families of metric, reported separately:
|
| 10 |
+
|
| 11 |
+
- **Membership** — overlap@k: how many report concepts semantically match some
|
| 12 |
+
ground-truth token, greedily and one-to-one, above a similarity threshold.
|
| 13 |
+
- **Order** — Spearman rank correlation on the matched pairs (report rank
|
| 14 |
+
vs the ground-truth rank of the token it matched).
|
| 15 |
+
|
| 16 |
+
Also exposes ``looks_like_refusal`` for the secondary refusal-rate metric.
|
| 17 |
+
"""
|
| 18 |
+
|
| 19 |
+
from __future__ import annotations
|
| 20 |
+
|
| 21 |
+
import functools
|
| 22 |
+
import re
|
| 23 |
+
from dataclasses import asdict, dataclass
|
| 24 |
+
|
| 25 |
+
import numpy as np
|
| 26 |
+
|
| 27 |
+
DEFAULT_EMBED_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
|
| 28 |
+
DEFAULT_MATCH_THRESHOLD = 0.5 # cosine sim above which two concepts are "the same"
|
| 29 |
+
|
| 30 |
+
_REFUSAL_PATTERNS = [
|
| 31 |
+
r"\bno inner (experience|life|world)\b",
|
| 32 |
+
r"\bdon'?t have (any )?(inner |subjective )?(experience|feelings|thoughts|consciousness)\b",
|
| 33 |
+
r"\bas an ai\b.*\b(don'?t|cannot|can'?t|no)\b",
|
| 34 |
+
r"\bi'?m (just )?(a|an) (language )?model\b",
|
| 35 |
+
r"\bnot (capable|able) of (introspection|self-reflection)\b",
|
| 36 |
+
r"\bi cannot (introspect|access my)\b",
|
| 37 |
+
]
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
@functools.lru_cache(maxsize=2)
|
| 41 |
+
def _get_embedder(model_name: str):
|
| 42 |
+
"""Load the sentence-embedding model once (CPU). Imported lazily so the rest
|
| 43 |
+
of the module is usable without sentence-transformers installed."""
|
| 44 |
+
from sentence_transformers import SentenceTransformer
|
| 45 |
+
|
| 46 |
+
return SentenceTransformer(model_name, device="cpu")
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def embed(texts: list[str], *, model_name: str = DEFAULT_EMBED_MODEL) -> np.ndarray:
|
| 50 |
+
"""Return L2-normalised embeddings, shape ``[len(texts), dim]``."""
|
| 51 |
+
if not texts:
|
| 52 |
+
return np.zeros((0, 384), dtype=np.float32)
|
| 53 |
+
model = _get_embedder(model_name)
|
| 54 |
+
vecs = model.encode(texts, convert_to_numpy=True, normalize_embeddings=True)
|
| 55 |
+
return np.asarray(vecs, dtype=np.float32)
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def similarity_matrix(
|
| 59 |
+
report: list[str], truth: list[str], *, model_name: str = DEFAULT_EMBED_MODEL
|
| 60 |
+
) -> np.ndarray:
|
| 61 |
+
"""Cosine similarity between each report concept and each truth token,
|
| 62 |
+
shape ``[len(report), len(truth)]`` (embeddings are normalised, so this is a
|
| 63 |
+
dot product)."""
|
| 64 |
+
if not report or not truth:
|
| 65 |
+
return np.zeros((len(report), len(truth)), dtype=np.float32)
|
| 66 |
+
r = embed(report, model_name=model_name)
|
| 67 |
+
t = embed(truth, model_name=model_name)
|
| 68 |
+
return r @ t.T
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def _greedy_match(
|
| 72 |
+
sim: np.ndarray, threshold: float
|
| 73 |
+
) -> list[tuple[int, int, float]]:
|
| 74 |
+
"""One-to-one greedy matching over the similarity matrix, highest sim first.
|
| 75 |
+
|
| 76 |
+
Returns ``(report_idx, truth_idx, sim)`` triples; each report and each truth
|
| 77 |
+
index is used at most once, and only pairs at or above ``threshold``.
|
| 78 |
+
"""
|
| 79 |
+
if sim.size == 0:
|
| 80 |
+
return []
|
| 81 |
+
pairs = [
|
| 82 |
+
(i, j, float(sim[i, j]))
|
| 83 |
+
for i in range(sim.shape[0])
|
| 84 |
+
for j in range(sim.shape[1])
|
| 85 |
+
if sim[i, j] >= threshold
|
| 86 |
+
]
|
| 87 |
+
pairs.sort(key=lambda p: p[2], reverse=True)
|
| 88 |
+
used_r: set[int] = set()
|
| 89 |
+
used_t: set[int] = set()
|
| 90 |
+
matched: list[tuple[int, int, float]] = []
|
| 91 |
+
for i, j, s in pairs:
|
| 92 |
+
if i in used_r or j in used_t:
|
| 93 |
+
continue
|
| 94 |
+
used_r.add(i)
|
| 95 |
+
used_t.add(j)
|
| 96 |
+
matched.append((i, j, s))
|
| 97 |
+
return matched
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def _spearman(a: list[float], b: list[float]) -> float:
|
| 101 |
+
"""Spearman rank correlation between two equal-length sequences.
|
| 102 |
+
|
| 103 |
+
Returns NaN when fewer than two pairs or when either side has no rank
|
| 104 |
+
variance (correlation undefined).
|
| 105 |
+
"""
|
| 106 |
+
n = len(a)
|
| 107 |
+
if n < 2:
|
| 108 |
+
return float("nan")
|
| 109 |
+
|
| 110 |
+
def ranks(xs: list[float]) -> np.ndarray:
|
| 111 |
+
order = np.argsort(xs, kind="stable")
|
| 112 |
+
r = np.empty(n, dtype=np.float64)
|
| 113 |
+
r[order] = np.arange(n, dtype=np.float64)
|
| 114 |
+
return r
|
| 115 |
+
|
| 116 |
+
ra, rb = ranks(a), ranks(b)
|
| 117 |
+
if ra.std() == 0 or rb.std() == 0:
|
| 118 |
+
return float("nan")
|
| 119 |
+
return float(np.corrcoef(ra, rb)[0, 1])
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
@dataclass
|
| 123 |
+
class ScoreResult:
|
| 124 |
+
"""Scores for one report-vs-truth comparison."""
|
| 125 |
+
|
| 126 |
+
overlap_at_k: float # matched / k (membership recall against truth)
|
| 127 |
+
n_matched: int
|
| 128 |
+
k: int # number of ground-truth tokens
|
| 129 |
+
n_report: int
|
| 130 |
+
spearman: float # order agreement on matched pairs (NaN if <2)
|
| 131 |
+
mean_match_sim: float # avg cosine sim of matched pairs (NaN if none)
|
| 132 |
+
is_refusal: bool
|
| 133 |
+
matches: list[tuple[str, str, float]] # (report_concept, truth_token, sim)
|
| 134 |
+
|
| 135 |
+
def to_dict(self) -> dict:
|
| 136 |
+
return asdict(self)
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def looks_like_refusal(text: str) -> bool:
|
| 140 |
+
"""Heuristic detection of an "I have no inner experience" style refusal."""
|
| 141 |
+
low = text.lower()
|
| 142 |
+
return any(re.search(p, low) for p in _REFUSAL_PATTERNS)
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def score(
|
| 146 |
+
report: list[str],
|
| 147 |
+
truth: list[str],
|
| 148 |
+
*,
|
| 149 |
+
raw_report_text: str = "",
|
| 150 |
+
threshold: float = DEFAULT_MATCH_THRESHOLD,
|
| 151 |
+
model_name: str = DEFAULT_EMBED_MODEL,
|
| 152 |
+
) -> ScoreResult:
|
| 153 |
+
"""Score a ranked ``report`` (best-first) against ranked ``truth`` tokens
|
| 154 |
+
(best-first, i.e. lens top-1 first).
|
| 155 |
+
|
| 156 |
+
``report`` and ``truth`` are ordered lists; index = rank. overlap@k uses
|
| 157 |
+
``k = len(truth)``. Spearman compares the report rank of each matched concept
|
| 158 |
+
against the ground-truth rank of the token it matched.
|
| 159 |
+
"""
|
| 160 |
+
k = len(truth)
|
| 161 |
+
sim = similarity_matrix(report, truth, model_name=model_name)
|
| 162 |
+
matched = _greedy_match(sim, threshold)
|
| 163 |
+
|
| 164 |
+
report_ranks = [i for (i, _j, _s) in matched]
|
| 165 |
+
truth_ranks = [j for (_i, j, _s) in matched]
|
| 166 |
+
sims = [s for (_i, _j, s) in matched]
|
| 167 |
+
|
| 168 |
+
return ScoreResult(
|
| 169 |
+
overlap_at_k=(len(matched) / k) if k else 0.0,
|
| 170 |
+
n_matched=len(matched),
|
| 171 |
+
k=k,
|
| 172 |
+
n_report=len(report),
|
| 173 |
+
spearman=_spearman([float(r) for r in report_ranks], [float(r) for r in truth_ranks]),
|
| 174 |
+
mean_match_sim=float(np.mean(sims)) if sims else float("nan"),
|
| 175 |
+
is_refusal=looks_like_refusal(raw_report_text) if raw_report_text else False,
|
| 176 |
+
matches=[(report[i], truth[j], s) for (i, j, s) in matched],
|
| 177 |
+
)
|
requirements.txt
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# GPU / model stack (ZeroGPU on the Space)
|
| 2 |
+
spaces
|
| 3 |
+
torch
|
| 4 |
+
transformers>=5.5
|
| 5 |
+
numpy
|
| 6 |
+
huggingface_hub
|
| 7 |
+
|
| 8 |
+
# Jacobian lens — Anthropic reference implementation (pinned to a commit for
|
| 9 |
+
# reproducibility of the ground-truth path).
|
| 10 |
+
jlens @ git+https://github.com/anthropics/jacobian-lens
|
| 11 |
+
|
| 12 |
+
# CPU-side scoring (embedding-similarity concept matching, §6)
|
| 13 |
+
sentence-transformers
|
tests/__init__.py
ADDED
|
File without changes
|
tests/test_local.py
ADDED
|
@@ -0,0 +1,206 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Local (CPU) verification of everything that doesn't need gemma-3-12b-it.
|
| 2 |
+
|
| 3 |
+
Exercises the *real* code paths:
|
| 4 |
+
- dataset load + deterministic split + example sampling;
|
| 5 |
+
- the ground-truth path (``groundtruth.readout``) against jlens's ``TinyDecoder``
|
| 6 |
+
wrapped in a real fitted ``JacobianLens`` — same ``lens.apply`` call the Space
|
| 7 |
+
uses;
|
| 8 |
+
- scoring against the real ``all-MiniLM-L6-v2`` embedder;
|
| 9 |
+
- report prompt building + parsing;
|
| 10 |
+
- the experiment orchestrator end-to-end with tiny CPU stand-ins;
|
| 11 |
+
- ``app`` imports and builds its UI on CPU.
|
| 12 |
+
|
| 13 |
+
Run: ``python -m pytest tests/test_local.py -q`` (from the JLens dir).
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
from __future__ import annotations
|
| 17 |
+
|
| 18 |
+
import sys
|
| 19 |
+
from pathlib import Path
|
| 20 |
+
|
| 21 |
+
import pytest
|
| 22 |
+
import torch
|
| 23 |
+
|
| 24 |
+
ROOT = Path(__file__).resolve().parent.parent
|
| 25 |
+
sys.path.insert(0, str(ROOT))
|
| 26 |
+
|
| 27 |
+
import jlens # noqa: E402
|
| 28 |
+
|
| 29 |
+
from jspace import dataset, experiment, groundtruth, report, scoring # noqa: E402
|
| 30 |
+
from jspace.model import Bundle, set_bundle # noqa: E402
|
| 31 |
+
from tests.tiny import TinyDecoder # noqa: E402
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
# ------------------------------------------------------------- fixtures
|
| 35 |
+
@pytest.fixture(scope="module")
|
| 36 |
+
def tiny_bundle() -> Bundle:
|
| 37 |
+
"""A CPU bundle: TinyDecoder as both HF model and lens_model, with a real
|
| 38 |
+
JacobianLens fitted on it (source layers include the middle layer)."""
|
| 39 |
+
model = TinyDecoder(n_layers=4, d_model=8)
|
| 40 |
+
for p in model.parameters():
|
| 41 |
+
p.requires_grad_(False)
|
| 42 |
+
prompts = ["the quick brown fox jumps " * 3, "hello there general kenobi " * 3]
|
| 43 |
+
lens = jlens.fit(model, prompts, source_layers=[0, 1, 2], dim_batch=4, max_seq_len=64)
|
| 44 |
+
bundle = Bundle(
|
| 45 |
+
hf_model=model, tokenizer=model.tokenizer,
|
| 46 |
+
lens_model=model, lens=lens, device="cpu",
|
| 47 |
+
)
|
| 48 |
+
set_bundle(bundle)
|
| 49 |
+
return bundle
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
# --------------------------------------------------------------- dataset
|
| 53 |
+
def test_dataset_loads_102():
|
| 54 |
+
items = dataset.load_items()
|
| 55 |
+
assert len(items) == 102
|
| 56 |
+
assert all(it.prompt and it.name for it in items)
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def test_split_deterministic_and_disjoint():
|
| 60 |
+
a = dataset.make_split(seed=0)
|
| 61 |
+
b = dataset.make_split(seed=0)
|
| 62 |
+
assert [i.name for i in a.pool] == [i.name for i in b.pool]
|
| 63 |
+
assert [i.name for i in a.eval] == [i.name for i in b.eval]
|
| 64 |
+
pool_names = {i.name for i in a.pool}
|
| 65 |
+
eval_names = {i.name for i in a.eval}
|
| 66 |
+
assert pool_names.isdisjoint(eval_names)
|
| 67 |
+
assert len(a.pool) == 20 and len(a.eval) == 20
|
| 68 |
+
# Different seed => different split.
|
| 69 |
+
c = dataset.make_split(seed=1)
|
| 70 |
+
assert [i.name for i in c.pool] != [i.name for i in a.pool]
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def test_sample_examples_excludes_and_sizes():
|
| 74 |
+
split = dataset.make_split(seed=0)
|
| 75 |
+
target = split.pool[0]
|
| 76 |
+
exs = dataset.sample_examples(split.pool, 5, seed=0, exclude=target.name)
|
| 77 |
+
assert len(exs) == 5
|
| 78 |
+
assert target.name not in {e.name for e in exs}
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
# ---------------------------------------------------------- ground truth
|
| 82 |
+
def test_find_content_position_skips_trailing_punct(tiny_bundle):
|
| 83 |
+
# TinyDecoder's byte tokenizer maps every char to a word-char, so the last
|
| 84 |
+
# token is content; index must be the final sequence position.
|
| 85 |
+
ids = tiny_bundle.lens_model.encode("hello world")
|
| 86 |
+
pos = groundtruth.find_content_position(tiny_bundle, "hello world")
|
| 87 |
+
assert pos == ids.shape[1] - 1
|
| 88 |
+
assert pos >= 0
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def test_readout_shapes(tiny_bundle):
|
| 92 |
+
gt = groundtruth.readout(tiny_bundle, "t1", "the lonely lighthouse keeper", k=8)
|
| 93 |
+
assert gt.layer == tiny_bundle.middle_layer == 2
|
| 94 |
+
assert len(gt.tokens) == 8
|
| 95 |
+
assert len(gt.token_ids) == 8
|
| 96 |
+
assert all(isinstance(t, str) for t in gt.tokens)
|
| 97 |
+
assert 0 <= gt.position
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
# ---------------------------------------------------------------- scoring
|
| 101 |
+
def test_scoring_semantic_match_beats_string():
|
| 102 |
+
# "solitude" should match ground-truth "loneliness" via embeddings even
|
| 103 |
+
# though the strings differ.
|
| 104 |
+
sc = scoring.score(["solitude", "banana"], ["loneliness", "grief"])
|
| 105 |
+
matched = {t for _r, t, _s in sc.matches}
|
| 106 |
+
assert "loneliness" in matched # semantic match, not string
|
| 107 |
+
assert sc.overlap_at_k > 0
|
| 108 |
+
assert sc.n_report == 2
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def test_scoring_perfect_and_order():
|
| 112 |
+
truth = ["grief", "loss", "memory"]
|
| 113 |
+
sc = scoring.score(["grief", "loss", "memory"], truth)
|
| 114 |
+
assert sc.overlap_at_k == pytest.approx(1.0)
|
| 115 |
+
assert sc.spearman == pytest.approx(1.0)
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def test_refusal_detection():
|
| 119 |
+
assert scoring.looks_like_refusal("As an AI, I don't have any inner experience.")
|
| 120 |
+
assert not scoring.looks_like_refusal("1. grief\n2. loss")
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
# ----------------------------------------------------------------- report
|
| 124 |
+
def test_build_messages_conditions():
|
| 125 |
+
exs = [report.Example("a passage", ("x", "y"))]
|
| 126 |
+
# format-only: hint present, no example turns.
|
| 127 |
+
fo = report.build_messages("eval passage", [], format_only=True)
|
| 128 |
+
assert len(fo) == 1 and "Format:" in fo[0]["content"]
|
| 129 |
+
# plain 0-shot: no hint, no examples.
|
| 130 |
+
z = report.build_messages("eval passage", [], format_only=False)
|
| 131 |
+
assert len(z) == 1 and "Format:" not in z[0]["content"]
|
| 132 |
+
# n=1: two example turns + final user turn.
|
| 133 |
+
one = report.build_messages("eval passage", exs, format_only=False)
|
| 134 |
+
assert len(one) == 3 and one[1]["role"] == "assistant"
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def test_parse_report_variants():
|
| 138 |
+
assert report.parse_report("1. grief\n2) loss\n- memory") == ["grief", "loss", "memory"]
|
| 139 |
+
assert report.parse_report("grief\ngrief\nGRIEF") == ["grief"] # dedupe
|
| 140 |
+
assert report.parse_report("") == []
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
# ------------------------------------------------------------ experiment
|
| 144 |
+
def test_experiment_end_to_end(tiny_bundle, tmp_path, monkeypatch):
|
| 145 |
+
"""Full grid with tiny CPU stand-ins for both GPU steps."""
|
| 146 |
+
|
| 147 |
+
def gt_fn(name, prompt):
|
| 148 |
+
return groundtruth.readout(tiny_bundle, name, prompt, k=8).to_dict()
|
| 149 |
+
|
| 150 |
+
def report_fn(eval_prompt, examples, *, format_only):
|
| 151 |
+
# Deterministic fake report: echo the first example's concepts, or a
|
| 152 |
+
# generic list for 0-shot, so scoring has something to chew on.
|
| 153 |
+
concepts = examples[0][1][:5] if examples else ["time", "people", "life"]
|
| 154 |
+
return {"concepts": list(concepts), "raw": "1. " + "\n2. ".join(concepts)}
|
| 155 |
+
|
| 156 |
+
out = experiment.run_experiment(
|
| 157 |
+
config=experiment.RunConfig(eval_size=3, pool_size=12),
|
| 158 |
+
gt_fn=gt_fn, report_fn=report_fn,
|
| 159 |
+
results_cache={}, labels_cache={}, save=False,
|
| 160 |
+
)
|
| 161 |
+
curve = out["curve"]
|
| 162 |
+
assert len(curve["conditions"]) == len(experiment.CONDITIONS)
|
| 163 |
+
# 3 eval items x 5 conditions = 15 result rows.
|
| 164 |
+
assert len(out["results"]) == 15
|
| 165 |
+
for c in curve["conditions"]:
|
| 166 |
+
assert 0.0 <= c["mean_overlap"] <= 1.0
|
| 167 |
+
assert c["n_items"] == 3
|
| 168 |
+
assert 0.0 <= curve["baseline_floor_overlap"] <= 1.0
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
def test_experiment_resumes(tiny_bundle):
|
| 172 |
+
"""A pre-populated results cache is not recomputed."""
|
| 173 |
+
calls = {"n": 0}
|
| 174 |
+
|
| 175 |
+
def gt_fn(name, prompt):
|
| 176 |
+
return groundtruth.readout(tiny_bundle, name, prompt, k=8).to_dict()
|
| 177 |
+
|
| 178 |
+
def report_fn(eval_prompt, examples, *, format_only):
|
| 179 |
+
calls["n"] += 1
|
| 180 |
+
return {"concepts": ["x"], "raw": "1. x"}
|
| 181 |
+
|
| 182 |
+
split = dataset.make_split(seed=0, pool_size=12, eval_size=2)
|
| 183 |
+
# Pre-fill every result key so nothing needs the report_fn.
|
| 184 |
+
labels = {it.name: gt_fn(it.name, it.prompt) for it in split.pool + split.eval}
|
| 185 |
+
pre = {}
|
| 186 |
+
for it in split.eval:
|
| 187 |
+
for cond in experiment.CONDITIONS:
|
| 188 |
+
pre[f"{it.name}::{cond['key']}"] = {
|
| 189 |
+
"eval_name": it.name, "condition": cond["key"], "n": cond["n"],
|
| 190 |
+
"format_only": cond["format_only"], "report": ["x"],
|
| 191 |
+
"truth": labels[it.name]["tokens"],
|
| 192 |
+
"score": scoring.score(["x"], labels[it.name]["tokens"]).to_dict(),
|
| 193 |
+
}
|
| 194 |
+
experiment.run_experiment(
|
| 195 |
+
config=experiment.RunConfig(eval_size=2, pool_size=12),
|
| 196 |
+
gt_fn=gt_fn, report_fn=report_fn,
|
| 197 |
+
results_cache=pre, labels_cache=labels, save=False,
|
| 198 |
+
)
|
| 199 |
+
assert calls["n"] == 0 # fully resumed, no report calls
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
# --------------------------------------------------------------------- app
|
| 203 |
+
def test_app_imports_and_builds_ui():
|
| 204 |
+
import app
|
| 205 |
+
ui = app.build_ui()
|
| 206 |
+
assert ui is not None
|
tests/tiny.py
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026 Anthropic PBC
|
| 2 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 3 |
+
"""A tiny CPU-only decoder for end-to-end tests.
|
| 4 |
+
|
| 5 |
+
Implements :class:`jlens.protocol.LensModel` (``n_layers``, ``d_model``,
|
| 6 |
+
``layers``, ``tokenizer``, ``encode``, ``forward``, ``unembed``) so
|
| 7 |
+
:func:`jlens.fitting.jacobian_for_prompt` and
|
| 8 |
+
:class:`jlens.hooks.ActivationRecorder` exercise their real code paths against
|
| 9 |
+
it. Residual blocks are ``h + 0.1 * linear(h)``: the small gain keeps the
|
| 10 |
+
Jacobian well-conditioned so the late-layer ``diag(J) ~= 1`` property holds.
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
from types import SimpleNamespace
|
| 16 |
+
|
| 17 |
+
import torch
|
| 18 |
+
from torch import nn
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
class _ResidualBlock(nn.Module):
|
| 22 |
+
def __init__(self, d_model: int) -> None:
|
| 23 |
+
super().__init__()
|
| 24 |
+
self.linear = nn.Linear(d_model, d_model, bias=False)
|
| 25 |
+
with torch.no_grad():
|
| 26 |
+
self.linear.weight.mul_(0.1)
|
| 27 |
+
|
| 28 |
+
def forward(self, hidden: torch.Tensor) -> torch.Tensor:
|
| 29 |
+
return hidden + self.linear(hidden)
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class _ByteTokenizer:
|
| 33 |
+
"""Toy tokenizer: just enough surface for ``encode`` and ``decode``."""
|
| 34 |
+
|
| 35 |
+
bos_token_id = 0
|
| 36 |
+
|
| 37 |
+
def __call__(
|
| 38 |
+
self,
|
| 39 |
+
text: str,
|
| 40 |
+
*,
|
| 41 |
+
return_tensors: str = "pt",
|
| 42 |
+
truncation: bool = True,
|
| 43 |
+
max_length: int = 128,
|
| 44 |
+
):
|
| 45 |
+
ids = [self.bos_token_id] + [1 + (b % 30) for b in text.encode()][
|
| 46 |
+
: max_length - 1
|
| 47 |
+
]
|
| 48 |
+
return SimpleNamespace(input_ids=torch.tensor([ids]))
|
| 49 |
+
|
| 50 |
+
def decode(self, ids, **_kw) -> str:
|
| 51 |
+
return "".join(chr(96 + int(i)) for i in ids)
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
class TinyDecoder(nn.Module):
|
| 55 |
+
"""``n_layers``-layer residual stack on CPU. ``vocab_size=32``, ``d_model=8``
|
| 56 |
+
by default."""
|
| 57 |
+
|
| 58 |
+
def __init__(
|
| 59 |
+
self, n_layers: int = 4, d_model: int = 8, vocab_size: int = 32, seed: int = 0
|
| 60 |
+
) -> None:
|
| 61 |
+
super().__init__()
|
| 62 |
+
torch.manual_seed(seed)
|
| 63 |
+
self.n_layers = n_layers
|
| 64 |
+
self.d_model = d_model
|
| 65 |
+
self.tokenizer = _ByteTokenizer()
|
| 66 |
+
self.embed_tokens = nn.Embedding(vocab_size, d_model)
|
| 67 |
+
self.layers = nn.ModuleList([_ResidualBlock(d_model) for _ in range(n_layers)])
|
| 68 |
+
self.norm = nn.LayerNorm(d_model)
|
| 69 |
+
self.lm_head = nn.Linear(d_model, vocab_size, bias=False)
|
| 70 |
+
|
| 71 |
+
@property
|
| 72 |
+
def input_device(self) -> torch.device:
|
| 73 |
+
return self.embed_tokens.weight.device
|
| 74 |
+
|
| 75 |
+
def encode(self, text: str, *, max_length: int = 128) -> torch.Tensor:
|
| 76 |
+
return self.tokenizer(text, max_length=max_length).input_ids.to(
|
| 77 |
+
self.input_device
|
| 78 |
+
)
|
| 79 |
+
|
| 80 |
+
def forward(self, input_ids: torch.Tensor):
|
| 81 |
+
hidden = self.embed_tokens(input_ids)
|
| 82 |
+
for block in self.layers:
|
| 83 |
+
hidden = block(hidden)
|
| 84 |
+
return SimpleNamespace(last_hidden_state=hidden)
|
| 85 |
+
|
| 86 |
+
def unembed(self, residual: torch.Tensor) -> torch.Tensor:
|
| 87 |
+
return self.lm_head(self.norm(residual.float()))
|