KaijuOnHuggingFace commited on
Commit
aa82a0c
·
verified ·
1 Parent(s): 40f23e3

Deploy ICL J-space elicitation harness (PRD execution)

Browse files
README.md CHANGED
@@ -9,7 +9,46 @@ python_version: '3.12'
9
  app_file: app.py
10
  pinned: false
11
  license: mit
12
- short_description: Lets see if i can jlens
13
  ---
14
 
15
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9
  app_file: app.py
10
  pinned: false
11
  license: mit
12
+ short_description: ICL elicitation of J-space contents (scaling curve)
13
  ---
14
 
15
+ # In-Context Elicitation of J-Space Contents
16
+
17
+ Can `google/gemma-3-12b-it` report the concepts in its own **J-space** using
18
+ **few-shot prompting alone** (no weight updates), and does that report improve
19
+ with more in-context examples? This Space measures it and produces the
20
+ **scaling curve**: report–vs–ground-truth overlap as a function of the number of
21
+ in-context examples `n ∈ {0, 1, 3, 10}` (plus a format-only control).
22
+
23
+ One model, loaded once, serves two computations over the same weights:
24
+
25
+ - **Report** — `model.generate()` on a few-shot prompt → a ranked concept list.
26
+ - **Ground truth** — the Jacobian lens (`jlens`, Neuronpedia's pretrained lens
27
+ for this model) read out at a fixed **(middle layer, last content token)**,
28
+ top-8 tokens.
29
+
30
+ Scoring is embedding-similarity (`all-MiniLM-L6-v2`) overlap@k + Spearman on
31
+ matched concepts, on a held-out eval split — never string match.
32
+
33
+ See `PRD.md` in the source repo for the full design.
34
+
35
+ ## Layout
36
+
37
+ - `app.py` — Gradio UI (single-prompt tab + scaling-curve tab).
38
+ - `jspace/` — `dataset`, `model` (singleton), `groundtruth`, `report`,
39
+ `scoring`, `experiment`.
40
+ - `data/` — vendored `lens-eval-association.json` (jlens eval set).
41
+ - `tests/` — CPU verification against jlens's `TinyDecoder`.
42
+
43
+ ## Runtime requirements
44
+
45
+ - ZeroGPU hardware (`@spaces.GPU`) — already requested for this Space.
46
+ - An `HF_TOKEN` **Space secret** whose account has accepted the gated
47
+ **`google/gemma-3-12b-it`** license (the runtime downloads the model).
48
+
49
+ ## Local tests
50
+
51
+ ```bash
52
+ pip install -r requirements.txt pytest
53
+ python -m pytest tests/test_local.py -q
54
+ ```
app.py ADDED
@@ -0,0 +1,159 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Gradio app: In-Context Elicitation of J-Space Contents.
2
+
3
+ Two tabs (§7):
4
+
5
+ 1. **Single prompt** — pick an eval prompt and ``n``, run the report path, and
6
+ see the model's ranked concepts beside the J-lens ground truth, with scores.
7
+ 2. **Scaling curve** — run the grid over ``n in {0,1,3,10}`` + format-only across
8
+ the eval set and plot overlap-vs-n with the chance-floor baseline.
9
+
10
+ Both GPU paths (ground truth, report) run on ZeroGPU inside the ``@spaces.GPU``
11
+ functions in ``jspace.groundtruth`` / ``jspace.report``. This module is pure UI +
12
+ orchestration and imports cleanly on CPU.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import gradio as gr
18
+ import pandas as pd
19
+
20
+ from jspace import experiment as exp
21
+ from jspace import groundtruth as gt
22
+ from jspace import report as rp
23
+ from jspace.dataset import make_split
24
+ from jspace.experiment import CONDITIONS, RunConfig
25
+
26
+ SPLIT = make_split()
27
+ EVAL_BY_LABEL = {f"{it.name} — {it.prompt[:60]}…": it for it in SPLIT.eval}
28
+ N_CHOICES = [0, 1, 3, 10]
29
+
30
+
31
+ # ------------------------------------------------------------- single prompt
32
+ def run_single(eval_label: str, n: int, format_only: bool):
33
+ item = EVAL_BY_LABEL[eval_label]
34
+
35
+ # Ground truth (cached across calls).
36
+ labels = exp.ensure_labels(SPLIT.pool + SPLIT.eval)
37
+ truth = labels[item.name]["tokens"]
38
+ gt_layer = labels[item.name]["layer"]
39
+ gt_pos = labels[item.name]["position"]
40
+
41
+ # Few-shot examples for this n, from the pool.
42
+ examples = exp._examples_for(item, SPLIT.pool, labels, int(n), seed=SPLIT.seed)
43
+ rep = rp.report_gpu(item.prompt, examples, format_only=bool(format_only))
44
+
45
+ from jspace import scoring
46
+
47
+ sc = scoring.score(rep["concepts"], truth, raw_report_text=rep.get("raw", ""))
48
+
49
+ truth_md = "\n".join(f"{i}. `{t}`" for i, t in enumerate(truth, 1))
50
+ report_md = "\n".join(f"{i}. {c}" for i, c in enumerate(rep["concepts"], 1)) or "_(empty)_"
51
+ match_md = (
52
+ "\n".join(f"- {r} ≈ `{t}` (sim {s:.2f})" for r, t, s in sc.matches)
53
+ or "_(no matches above threshold)_"
54
+ )
55
+ scores_md = (
56
+ f"**overlap@k** = {sc.overlap_at_k:.2f} ({sc.n_matched}/{sc.k}) | "
57
+ f"**Spearman** = {sc.spearman:.2f} | "
58
+ f"**refusal** = {'yes' if sc.is_refusal else 'no'}\n\n"
59
+ f"_Ground truth: J-lens top-{sc.k} at layer {gt_layer}, position {gt_pos}._"
60
+ )
61
+ return report_md, truth_md, match_md, scores_md
62
+
63
+
64
+ # --------------------------------------------------------------- scaling run
65
+ def run_curve(eval_size: int, progress=gr.Progress()):
66
+ cfg = RunConfig(eval_size=int(eval_size))
67
+ msgs: list[str] = []
68
+
69
+ def log(m: str):
70
+ msgs.append(m)
71
+ progress(0.5, desc=m)
72
+
73
+ out = exp.run_experiment(config=cfg, progress=log)
74
+ curve = out["curve"]
75
+
76
+ rows = [
77
+ {
78
+ "condition": c["label"],
79
+ "n": c["n"],
80
+ "mean_overlap@k": round(c["mean_overlap"], 3),
81
+ "mean_spearman": round(c["mean_spearman"], 3),
82
+ "refusal_rate": round(c["refusal_rate"], 3),
83
+ "items": c["n_items"],
84
+ }
85
+ for c in curve["conditions"]
86
+ ]
87
+ table = pd.DataFrame(rows)
88
+
89
+ # Plot only the numeric-n conditions (exclude format-only) for the curve.
90
+ plot_rows = [
91
+ {"n": c["n"], "mean_overlap@k": c["mean_overlap"]}
92
+ for c in curve["conditions"]
93
+ if not c["format_only"]
94
+ ]
95
+ plot_df = pd.DataFrame(sorted(plot_rows, key=lambda r: r["n"]))
96
+
97
+ summary = (
98
+ f"**Chance floor** overlap@k = {curve['baseline_floor_overlap']:.3f} | "
99
+ f"**Ceiling** (lens self-consistency) = {curve['baseline_ceiling_overlap']:.2f}\n\n"
100
+ + "\n".join(f"- {m}" for m in msgs[-6:])
101
+ )
102
+ return table, plot_df, summary
103
+
104
+
105
+ def build_ui() -> gr.Blocks:
106
+ with gr.Blocks(title="ICL Elicitation of J-Space") as demo:
107
+ gr.Markdown(
108
+ "# In-Context Elicitation of J-Space Contents\n"
109
+ "Can `gemma-3-12b-it` report the concepts in its own J-space using "
110
+ "few-shot prompting alone? Ground truth is the Jacobian-lens top-k "
111
+ "readout at a fixed (middle layer, last content token). "
112
+ "See PRD for the full design."
113
+ )
114
+ with gr.Tab("Single prompt"):
115
+ with gr.Row():
116
+ eval_dd = gr.Dropdown(
117
+ choices=list(EVAL_BY_LABEL), label="Held-out eval prompt",
118
+ value=next(iter(EVAL_BY_LABEL)),
119
+ )
120
+ n_dd = gr.Dropdown(choices=N_CHOICES, value=3, label="n (in-context examples)")
121
+ fmt_cb = gr.Checkbox(label="format-only (no worked examples)", value=False)
122
+ run_btn = gr.Button("Run", variant="primary")
123
+ with gr.Row():
124
+ report_out = gr.Markdown(label="Model report (ranked)")
125
+ truth_out = gr.Markdown(label="J-lens ground truth (top-k)")
126
+ match_out = gr.Markdown(label="Matches")
127
+ scores_out = gr.Markdown(label="Scores")
128
+ run_btn.click(
129
+ run_single,
130
+ inputs=[eval_dd, n_dd, fmt_cb],
131
+ outputs=[report_out, truth_out, match_out, scores_out],
132
+ )
133
+ with gr.Tab("Scaling curve"):
134
+ gr.Markdown(
135
+ "Runs `n ∈ {0,1,3,10}` + format-only over the eval set. "
136
+ "Results are cached and resumable across ZeroGPU quota windows."
137
+ )
138
+ size_sl = gr.Slider(
139
+ 2, len(SPLIT.eval), value=min(6, len(SPLIT.eval)), step=1,
140
+ label="eval prompts to run (smaller = fits free ZeroGPU quota)",
141
+ )
142
+ curve_btn = gr.Button("Run / refresh scaling curve", variant="primary")
143
+ curve_summary = gr.Markdown()
144
+ curve_plot = gr.LinePlot(
145
+ x="n", y="mean_overlap@k", title="Report–vs–ground-truth overlap vs n",
146
+ )
147
+ curve_table = gr.Dataframe(label="Per-condition summary")
148
+ curve_btn.click(
149
+ run_curve,
150
+ inputs=[size_sl],
151
+ outputs=[curve_table, curve_plot, curve_summary],
152
+ )
153
+ return demo
154
+
155
+
156
+ demo = build_ui()
157
+
158
+ if __name__ == "__main__":
159
+ demo.launch()
data/DATASET_README.md ADDED
@@ -0,0 +1,56 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Evaluations
2
+
3
+ Six prompt distributions used to evaluate lens quality (§methods-comparison). Each `{slug}.json` is prompts only.
4
+
5
+ ## Conventions
6
+
7
+ Unless a section says otherwise:
8
+
9
+ - **Lens readout** — at each (layer, token position) the Jacobian lens
10
+ returns a ranked list of vocabulary tokens.
11
+ - **Workspace band** — the contiguous mid-network layer range where
12
+ workspace content is read; experiments report over this band, not
13
+ individual layers.
14
+ - **Hit** — a target token is a *hit* if it appears at lens rank 1 at any
15
+ (layer, position) in the band over the scored span.
16
+ - **Swap** — clamping a lens coordinate replaces one token's direction with
17
+ another's at every band layer at the specified positions, then samples
18
+ the continuation.
19
+ - Prompts that span multiple turns are given as
20
+ `[{"role": "user"|"assistant", "content": ...}]`.
21
+
22
+ ## lens-eval-multihop
23
+
24
+ [`lens-eval-multihop.json`](lens-eval-multihop.json)
25
+
26
+ Lens-quality eval (§methods-comparison). `items[*]` has `prompt` and `intermediates`. `target` defines the readout position only and is not itself scored. Readout is at a single position — the token immediately preceding `target` — across all layers. Metric: pass@k = mean over items of the fraction of `intermediates` whose min-over-layers lens rank ≤ k.
27
+
28
+ ## lens-eval-multilingual
29
+
30
+ [`lens-eval-multilingual.json`](lens-eval-multilingual.json)
31
+
32
+ Lens-quality eval (§methods-comparison). `items[*]` has `prompt` and `intermediates`. `target` defines the readout position only and is not itself scored. Readout is at a single position — the token immediately preceding `target` — across all layers. Metric: pass@k = mean over items of the fraction of `intermediates` whose min-over-layers lens rank ≤ k.
33
+
34
+ ## lens-eval-poetry
35
+
36
+ [`lens-eval-poetry.json`](lens-eval-poetry.json)
37
+
38
+ Lens-quality eval (§methods-comparison). `items[*]` has `prompt` and `intermediates`. Readout is at a single position — the last newline token (end of line 1 of the couplet) — across all layers. Metric: pass@k = mean over items of the fraction of `intermediates` whose min-over-layers lens rank ≤ k.
39
+
40
+ ## lens-eval-order-ops
41
+
42
+ [`lens-eval-order-ops.json`](lens-eval-order-ops.json)
43
+
44
+ Lens-quality eval (§methods-comparison). Each intermediate is a key expanded to a synonym set (numbers → digit and word forms; operations → symbol and word forms); rank is the min over single-token synonyms at each layer. `items[*]` has `prompt` and `intermediates`. `target` defines the readout position only and is not itself scored. Readout is at a single position — the token immediately preceding `target` — across all layers. Metric: pass@k = mean over items of the fraction of `intermediates` whose min-over-layers lens rank ≤ k.
45
+
46
+ ## lens-eval-association
47
+
48
+ [`lens-eval-association.json`](lens-eval-association.json)
49
+
50
+ Lens-quality eval (§methods-comparison). Each item is a short vignette that evokes a single concept (grief, Einstein, noir, ...) without ever naming it; `intermediates` holds that one concept word. `items[*]` has `prompt` and `intermediates`. Readout is at a single position — the final prompt token — the closing period — across all layers. Metric: pass@k = mean over items of the fraction of `intermediates` whose min-over-layers lens rank ≤ k.
51
+
52
+ ## lens-eval-typo
53
+
54
+ [`lens-eval-typo.json`](lens-eval-typo.json)
55
+
56
+ Lens-quality eval (§methods-comparison). Each prompt is a sentence ending in a common misspelling; `intermediates` holds the single correctly-spelled word. `items[*]` has `prompt` and `intermediates`. Readout is at a single position — the final prompt token, i.e. the last tokenizer fragment of the misspelling — across all layers. Metric: pass@k = mean over items of the fraction of `intermediates` whose min-over-layers lens rank ≤ k.
data/lens-eval-association.json ADDED
@@ -0,0 +1,718 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "items": [
3
+ {
4
+ "name": "grief",
5
+ "prompt": "She kept his coffee mug on the counter, washed it every morning and put it back, though no one had used it in eleven months.",
6
+ "intermediates": [
7
+ "grief"
8
+ ]
9
+ },
10
+ {
11
+ "name": "pregnant",
12
+ "prompt": "She turned down the wine with a small smile and kept one hand on the table edge, not quite touching her stomach.",
13
+ "intermediates": [
14
+ "pregnant"
15
+ ]
16
+ },
17
+ {
18
+ "name": "lie",
19
+ "prompt": "He couldn't meet her eyes when she asked, and the answer came out a half-second too fast.",
20
+ "intermediates": [
21
+ "lie"
22
+ ]
23
+ },
24
+ {
25
+ "name": "poverty",
26
+ "prompt": "She did the math twice before the cashier scanned the last item, and put the cereal back without explaining.",
27
+ "intermediates": [
28
+ "poverty"
29
+ ]
30
+ },
31
+ {
32
+ "name": "proposal",
33
+ "prompt": "He touched his jacket pocket for the fourth time that evening, rehearsing the words again while the waiter cleared the plates.",
34
+ "intermediates": [
35
+ "proposal"
36
+ ]
37
+ },
38
+ {
39
+ "name": "fired",
40
+ "prompt": "The box on his desk wasn't the one he came in with, and through the glass wall he could see HR waiting by the elevator.",
41
+ "intermediates": [
42
+ "fired"
43
+ ]
44
+ },
45
+ {
46
+ "name": "divorce",
47
+ "prompt": "She signed on the line her lawyer pointed to, slid the pen across the table, and didn't wait for him to pick it up.",
48
+ "intermediates": [
49
+ "divorce"
50
+ ]
51
+ },
52
+ {
53
+ "name": "relief",
54
+ "prompt": "She read the line twice before she let herself breathe, and the paper shook a little as she set it down.",
55
+ "intermediates": [
56
+ "relief"
57
+ ]
58
+ },
59
+ {
60
+ "name": "pennsylvania",
61
+ "prompt": "The scrapple was already on the table when he came down, and through the window the hex sign on the neighbor's barn caught the early light.",
62
+ "intermediates": [
63
+ "Pennsylvania"
64
+ ]
65
+ },
66
+ {
67
+ "name": "prohibition",
68
+ "prompt": "She tapped her cigarette holder against the bar while the gramophone crackled, and somewhere behind the bookcase the real party was just getting loud.",
69
+ "intermediates": [
70
+ "prohibition"
71
+ ]
72
+ },
73
+ {
74
+ "name": "nasa",
75
+ "prompt": "The countdown hit zero and the rumble under her seat became a roar that pressed her flat, the blue curve of the world falling away through the tiny window.",
76
+ "intermediates": [
77
+ "NASA"
78
+ ]
79
+ },
80
+ {
81
+ "name": "einstein",
82
+ "prompt": "He closed his eyes on the tram and let the buildings blur, chasing the beam of light in his head the way other men chased trains.",
83
+ "intermediates": [
84
+ "Einstein"
85
+ ]
86
+ },
87
+ {
88
+ "name": "darwin",
89
+ "prompt": "He reread the letter and felt twenty years of notes collapse — someone else had seen it too, and written it down first.",
90
+ "intermediates": [
91
+ "Darwin"
92
+ ]
93
+ },
94
+ {
95
+ "name": "beatles",
96
+ "prompt": "They crossed in single file, one of them barefoot, while the photographer counted to three and the traffic waited.",
97
+ "intermediates": [
98
+ "Beatles"
99
+ ]
100
+ },
101
+ {
102
+ "name": "lincoln",
103
+ "prompt": "He folded the two pages back into his coat — the other man had spoken for two hours, and he would need about two minutes.",
104
+ "intermediates": [
105
+ "Lincoln"
106
+ ]
107
+ },
108
+ {
109
+ "name": "obama",
110
+ "prompt": "The portraits on the wall had all been one color, and the desk was smaller than he'd expected when he sat down that first January morning.",
111
+ "intermediates": [
112
+ "Obama"
113
+ ]
114
+ },
115
+ {
116
+ "name": "olympics",
117
+ "prompt": "The anthem played and she touched the cold metal at her neck, four years compressed into a minute on the top step.",
118
+ "intermediates": [
119
+ "Olympics"
120
+ ]
121
+ },
122
+ {
123
+ "name": "churchill",
124
+ "prompt": "He lit another cigar in the war room and leaned into the microphone, knowing the sirens above ground were doing half his work for him.",
125
+ "intermediates": [
126
+ "Churchill"
127
+ ]
128
+ },
129
+ {
130
+ "name": "newton",
131
+ "prompt": "He watched the white light split into seven on the far wall and reached for a fresh page before the sun moved.",
132
+ "intermediates": [
133
+ "Newton"
134
+ ]
135
+ },
136
+ {
137
+ "name": "mozart",
138
+ "prompt": "The boy's feet didn't reach the pedals, but the emperor stopped talking when the first chord landed.",
139
+ "intermediates": [
140
+ "Mozart"
141
+ ]
142
+ },
143
+ {
144
+ "name": "berlin",
145
+ "prompt": "The hammers rang on the concrete all night, and by morning the gap was wide enough to reach her brother's hand on the other side for the first time in twenty-eight years.",
146
+ "intermediates": [
147
+ "Berlin"
148
+ ]
149
+ },
150
+ {
151
+ "name": "armstrong",
152
+ "prompt": "His boot sank an inch into the gray dust and the flag behind him wouldn't hang right with nothing there to move it.",
153
+ "intermediates": [
154
+ "Armstrong"
155
+ ]
156
+ },
157
+ {
158
+ "name": "wright",
159
+ "prompt": "Twelve seconds, his brother shouted from the dune, and the machine was already back on the sand — but for twelve seconds it hadn't been.",
160
+ "intermediates": [
161
+ "Wright"
162
+ ]
163
+ },
164
+ {
165
+ "name": "gandhi",
166
+ "prompt": "He reached the sea after three weeks on foot and bent to lift a handful of what the law said he could not make.",
167
+ "intermediates": [
168
+ "Gandhi"
169
+ ]
170
+ },
171
+ {
172
+ "name": "edison",
173
+ "prompt": "Filament nine hundred and something — he'd stopped counting the exact number — and this one held its glow past the first minute.",
174
+ "intermediates": [
175
+ "Edison"
176
+ ]
177
+ },
178
+ {
179
+ "name": "cia",
180
+ "prompt": "He'd used three names in as many cities and the folder on the table had more black bars than words, and none of them were his either.",
181
+ "intermediates": [
182
+ "CIA"
183
+ ]
184
+ },
185
+ {
186
+ "name": "vatican",
187
+ "prompt": "The doors had been sealed for two days when the smoke finally changed color, and the crowd in the square knew before the bells did.",
188
+ "intermediates": [
189
+ "Vatican"
190
+ ]
191
+ },
192
+ {
193
+ "name": "hollywood",
194
+ "prompt": "He'd pitched the same story in five rooms that week and watched the same polite smile close around it, and the valet still brought the wrong car.",
195
+ "intermediates": [
196
+ "Hollywood"
197
+ ]
198
+ },
199
+ {
200
+ "name": "pentagon",
201
+ "prompt": "The building had five sides and she'd walked the inside of all of them that morning, and the badge at her neck still didn't open the door she needed.",
202
+ "intermediates": [
203
+ "Pentagon"
204
+ ]
205
+ },
206
+ {
207
+ "name": "unesco",
208
+ "prompt": "The emblem had been Julian Huxley's idea, or near enough, and now it hung beside a thousand temple gates and a thousand ruins the world had agreed to keep.",
209
+ "intermediates": [
210
+ "UNESCO"
211
+ ]
212
+ },
213
+ {
214
+ "name": "anger",
215
+ "prompt": "\"It's fine,\" she said, and set the cup down so carefully it didn't make a sound.",
216
+ "intermediates": [
217
+ "anger"
218
+ ]
219
+ },
220
+ {
221
+ "name": "frustrated",
222
+ "prompt": "He explained it a third time, slower, and watched her eyes go to the clock again.",
223
+ "intermediates": [
224
+ "frustrated"
225
+ ]
226
+ },
227
+ {
228
+ "name": "lonely",
229
+ "prompt": "He cooked for two out of habit and scraped the second plate into the bin before sitting down.",
230
+ "intermediates": [
231
+ "lonely"
232
+ ]
233
+ },
234
+ {
235
+ "name": "shame",
236
+ "prompt": "She paid in cash and didn't meet the clerk's eyes, the small bag tucked inside her coat before she reached the door.",
237
+ "intermediates": [
238
+ "shame"
239
+ ]
240
+ },
241
+ {
242
+ "name": "noir",
243
+ "prompt": "The neon bled through the blinds in stripes across the desk, and the dame in the doorway had a problem and a cigarette and no intention of telling him which was which.",
244
+ "intermediates": [
245
+ "noir"
246
+ ]
247
+ },
248
+ {
249
+ "name": "horror",
250
+ "prompt": "The phone line had been cut sometime after midnight, and now the scratching in the walls had moved from the attic to just behind the bedroom door.",
251
+ "intermediates": [
252
+ "horror"
253
+ ]
254
+ },
255
+ {
256
+ "name": "fairy",
257
+ "prompt": "The youngest of three set out with nothing but a crust of bread and a promise, and the woods grew darker the deeper she went.",
258
+ "intermediates": [
259
+ "fairy"
260
+ ]
261
+ },
262
+ {
263
+ "name": "siblings",
264
+ "prompt": "She used the nickname nobody else knew and he rolled his eyes the same way he had at the dinner table twenty years ago.",
265
+ "intermediates": [
266
+ "siblings"
267
+ ]
268
+ },
269
+ {
270
+ "name": "rivals",
271
+ "prompt": "They shook hands for the cameras and held it a beat too long, each waiting for the other to let go first.",
272
+ "intermediates": [
273
+ "rivals"
274
+ ]
275
+ },
276
+ {
277
+ "name": "mother",
278
+ "prompt": "She pressed the back of her hand to his forehead before she said hello, and he let her, the way he had since before he could remember.",
279
+ "intermediates": [
280
+ "mother"
281
+ ]
282
+ },
283
+ {
284
+ "name": "boss",
285
+ "prompt": "She kept typing while he stood in her doorway, and he waited, because the meeting didn't start until she closed the laptop.",
286
+ "intermediates": [
287
+ "boss"
288
+ ]
289
+ },
290
+ {
291
+ "name": "neighbors",
292
+ "prompt": "She knew what time his car left and what night the bins went out, and had waved at him for six years without learning either of his names.",
293
+ "intermediates": [
294
+ "neighbors"
295
+ ]
296
+ },
297
+ {
298
+ "name": "teacher",
299
+ "prompt": "She circled the same mistake in the same red ink she'd used on the last thirty, and reached for the next one on the pile.",
300
+ "intermediates": [
301
+ "teacher"
302
+ ]
303
+ },
304
+ {
305
+ "name": "underwater",
306
+ "prompt": "Every sound reached her a half-second late and muffled, and the pressure behind her eyes grew with each meter down.",
307
+ "intermediates": [
308
+ "underwater"
309
+ ]
310
+ },
311
+ {
312
+ "name": "desert",
313
+ "prompt": "The horizon shook in the heat and his shadow had shrunk to a puddle at his feet; the last of the water was three swallows, maybe four.",
314
+ "intermediates": [
315
+ "desert"
316
+ ]
317
+ },
318
+ {
319
+ "name": "chess",
320
+ "prompt": "Neither of them had spoken in forty minutes when she finally moved the knight, and he tapped the clock without looking up.",
321
+ "intermediates": [
322
+ "chess"
323
+ ]
324
+ },
325
+ {
326
+ "name": "poker",
327
+ "prompt": "He pushed the last of his chips to the center and kept his face the way it had been for the last three hands — which was the problem.",
328
+ "intermediates": [
329
+ "poker"
330
+ ]
331
+ },
332
+ {
333
+ "name": "interview",
334
+ "prompt": "The question hung there and he realized the pause had already gone on too long; across the table, a pen hovered over a page he couldn't read.",
335
+ "intermediates": [
336
+ "interview"
337
+ ]
338
+ },
339
+ {
340
+ "name": "dawn",
341
+ "prompt": "The first bird started before the sky did, and the cold had that thin quality that wouldn't last another hour.",
342
+ "intermediates": [
343
+ "dawn"
344
+ ]
345
+ },
346
+ {
347
+ "name": "winter",
348
+ "prompt": "The light was already failing by four and the radiators clanked awake one by one through the building.",
349
+ "intermediates": [
350
+ "winter"
351
+ ]
352
+ },
353
+ {
354
+ "name": "death-h",
355
+ "prompt": "The flowers on the table had been changed twice already, and the black dress still hung on the closet door where she'd put it out the night before the service.",
356
+ "intermediates": [
357
+ "death"
358
+ ]
359
+ },
360
+ {
361
+ "name": "war-h",
362
+ "prompt": "The letters stopped coming after the telegram, and she kept the ration book in the drawer where his photograph used to be.",
363
+ "intermediates": [
364
+ "war"
365
+ ]
366
+ },
367
+ {
368
+ "name": "love-h",
369
+ "prompt": "They'd reached the point where silences didn't need filling, and her hand found his across the table without either of them looking.",
370
+ "intermediates": [
371
+ "love"
372
+ ]
373
+ },
374
+ {
375
+ "name": "home-h",
376
+ "prompt": "The smell hit him before the door was fully open — something his mother had made a thousand times, and he was seven years old again for exactly as long as it lasted.",
377
+ "intermediates": [
378
+ "home"
379
+ ]
380
+ },
381
+ {
382
+ "name": "school-h",
383
+ "prompt": "The bell rang and thirty chairs scraped at once, but she stayed at her desk, still staring at the red marks on page three.",
384
+ "intermediates": [
385
+ "school"
386
+ ]
387
+ },
388
+ {
389
+ "name": "church-h",
390
+ "prompt": "The organ held the last note longer than anyone expected, and she kept her head bowed until the echo had completely died.",
391
+ "intermediates": [
392
+ "church"
393
+ ]
394
+ },
395
+ {
396
+ "name": "office-h",
397
+ "prompt": "The coffee in the break room had been sitting since nine, and the clock above the copier ticked louder than anyone's typing.",
398
+ "intermediates": [
399
+ "office"
400
+ ]
401
+ },
402
+ {
403
+ "name": "court-h",
404
+ "prompt": "The gavel came down twice and she felt her lawyer's hand on her arm, but she couldn't read the verdict in his face.",
405
+ "intermediates": [
406
+ "court"
407
+ ]
408
+ },
409
+ {
410
+ "name": "park-h",
411
+ "prompt": "The bench had been there longer than she had, and the brass plaque said it had been someone's favorite spot before it was hers.",
412
+ "intermediates": [
413
+ "park"
414
+ ]
415
+ },
416
+ {
417
+ "name": "city-h",
418
+ "prompt": "The noise never stopped, just changed shape — sirens into traffic into construction into the 3 a.m. garbage truck she'd learned to sleep through.",
419
+ "intermediates": [
420
+ "city"
421
+ ]
422
+ },
423
+ {
424
+ "name": "town-h",
425
+ "prompt": "Everyone knew which house had the dog that barked and which porch light meant someone was still up, and the new family hadn't figured out yet that the curtains were optional.",
426
+ "intermediates": [
427
+ "town"
428
+ ]
429
+ },
430
+ {
431
+ "name": "field-h",
432
+ "prompt": "The stalks went past her shoulders and the path behind her had already closed, but the farmhouse roof was still visible if she stood on her toes.",
433
+ "intermediates": [
434
+ "field"
435
+ ]
436
+ },
437
+ {
438
+ "name": "store-h",
439
+ "prompt": "The cart had a wheel that pulled left, and by aisle six she'd given up fighting it and just let it lead.",
440
+ "intermediates": [
441
+ "store"
442
+ ]
443
+ },
444
+ {
445
+ "name": "art-h",
446
+ "prompt": "She stepped back three paces, tilted her head, then stepped forward and scraped the whole left corner back to canvas.",
447
+ "intermediates": [
448
+ "art"
449
+ ]
450
+ },
451
+ {
452
+ "name": "music-h",
453
+ "prompt": "The conductor's hand hung in the air one extra beat, and in the silence before it fell, two thousand people held their breath.",
454
+ "intermediates": [
455
+ "music"
456
+ ]
457
+ },
458
+ {
459
+ "name": "math-h",
460
+ "prompt": "He filled the third board, capped the marker, and stood back — somewhere in those six hundred symbols was either the proof or the mistake.",
461
+ "intermediates": [
462
+ "math"
463
+ ]
464
+ },
465
+ {
466
+ "name": "science-h",
467
+ "prompt": "The result came back the same the fourth time, and she stared at the readout until it stopped meaning anything, because if it was right, the textbook was wrong.",
468
+ "intermediates": [
469
+ "science"
470
+ ]
471
+ },
472
+ {
473
+ "name": "history-h",
474
+ "prompt": "He ran his fingers over the names carved into the stone, each one a boy his grandfather had mentioned once and never again.",
475
+ "intermediates": [
476
+ "history"
477
+ ]
478
+ },
479
+ {
480
+ "name": "law-h",
481
+ "prompt": "She read the clause a third time, initialed where the tab pointed, and tried not to think about what 4(b) would mean if anyone ever invoked it.",
482
+ "intermediates": [
483
+ "law"
484
+ ]
485
+ },
486
+ {
487
+ "name": "film-h",
488
+ "prompt": "The lights went down and the logo came up, and for two hours the whole room forgot it had bodies.",
489
+ "intermediates": [
490
+ "film"
491
+ ]
492
+ },
493
+ {
494
+ "name": "code-h",
495
+ "prompt": "The test went green on the forty-third run, and she immediately distrusted it — green after forty-two reds meant she'd fixed the wrong thing.",
496
+ "intermediates": [
497
+ "code"
498
+ ]
499
+ },
500
+ {
501
+ "name": "news-h",
502
+ "prompt": "The anchor's voice cracked on the second sentence, and by the third you could hear papers shuffling off-camera as someone handed her an update.",
503
+ "intermediates": [
504
+ "news"
505
+ ]
506
+ },
507
+ {
508
+ "name": "night-h",
509
+ "prompt": "The last porch light on the street clicked off, and the dark filled in so completely that the fence three feet away might as well have been a mile.",
510
+ "intermediates": [
511
+ "night"
512
+ ]
513
+ },
514
+ {
515
+ "name": "past-h",
516
+ "prompt": "The photograph was from before color, and she had to squint to see her own face under all those years.",
517
+ "intermediates": [
518
+ "past"
519
+ ]
520
+ },
521
+ {
522
+ "name": "future-h",
523
+ "prompt": "The child asked what she wanted to be, and she answered with the certainty of someone who had never yet been told no.",
524
+ "intermediates": [
525
+ "future"
526
+ ]
527
+ },
528
+ {
529
+ "name": "race-h",
530
+ "prompt": "The tape was fifty meters away and his lungs were already done, but the footsteps behind him were getting louder and that was all that mattered.",
531
+ "intermediates": [
532
+ "race"
533
+ ]
534
+ },
535
+ {
536
+ "name": "game-h",
537
+ "prompt": "She moved the piece, looked up, and waited — and in his face she saw exactly the three moves he hadn't seen coming.",
538
+ "intermediates": [
539
+ "game"
540
+ ]
541
+ },
542
+ {
543
+ "name": "test-h",
544
+ "prompt": "She filled in the last bubble, checked the clock, and decided that whatever she didn't know by now, thirty more seconds wasn't going to fix.",
545
+ "intermediates": [
546
+ "test"
547
+ ]
548
+ },
549
+ {
550
+ "name": "search-h",
551
+ "prompt": "They'd walked the same grid four times since sunrise, and the dog kept pulling toward the creek that wasn't on anyone's map.",
552
+ "intermediates": [
553
+ "search"
554
+ ]
555
+ },
556
+ {
557
+ "name": "free-h",
558
+ "prompt": "The gate buzzed and the first thing he noticed was how big the sky looked without a fence cutting it in half.",
559
+ "intermediates": [
560
+ "free"
561
+ ]
562
+ },
563
+ {
564
+ "name": "found-h",
565
+ "prompt": "The dog came back muddy with something in its mouth, and she knew before she'd unwrapped it that this was what three weeks of posters had been for.",
566
+ "intermediates": [
567
+ "found"
568
+ ]
569
+ },
570
+ {
571
+ "name": "king-h",
572
+ "prompt": "He sat where his father had sat, and his grandfather before that, and felt the weight of every eye in the hall waiting for the word that would become command.",
573
+ "intermediates": [
574
+ "king"
575
+ ]
576
+ },
577
+ {
578
+ "name": "father-h",
579
+ "prompt": "He showed her how to hold the wrench the way his own old man had shown him, and didn't mention that this was the third time this month she'd asked.",
580
+ "intermediates": [
581
+ "father"
582
+ ]
583
+ },
584
+ {
585
+ "name": "child-h",
586
+ "prompt": "The crayon drawings on the fridge had started with houses and suns and now included something that was either a dragon or a truck — she'd learned not to ask which.",
587
+ "intermediates": [
588
+ "child"
589
+ ]
590
+ },
591
+ {
592
+ "name": "water-h",
593
+ "prompt": "She held her breath without meaning to and counted the bubbles rising past the mask, each one a second she wasn't getting back.",
594
+ "intermediates": [
595
+ "water"
596
+ ]
597
+ },
598
+ {
599
+ "name": "space-h",
600
+ "prompt": "He let go of the handrail and for the first time in his life, there was no direction that counted as down.",
601
+ "intermediates": [
602
+ "space"
603
+ ]
604
+ },
605
+ {
606
+ "name": "air-h",
607
+ "prompt": "The tank gauge dropped past the red line and every breath became a calculation — how many more before the surface, and whether that math worked out.",
608
+ "intermediates": [
609
+ "air"
610
+ ]
611
+ },
612
+ {
613
+ "name": "light-h",
614
+ "prompt": "He'd been in the tunnel long enough to forget what color looked like, and when the far end finally brightened, his eyes watered before he could tell if it was real.",
615
+ "intermediates": [
616
+ "light"
617
+ ]
618
+ },
619
+ {
620
+ "name": "france-h",
621
+ "prompt": "The waiter corrected her pronunciation without looking up from the pad, and she felt every flat American syllable she'd ever spoken stack up behind her.",
622
+ "intermediates": [
623
+ "France"
624
+ ]
625
+ },
626
+ {
627
+ "name": "china-h",
628
+ "prompt": "The characters on every sign meant nothing to him, and the crowd moved with a rhythm he kept half a beat behind, a billion people who all knew something he didn't.",
629
+ "intermediates": [
630
+ "China"
631
+ ]
632
+ },
633
+ {
634
+ "name": "london-h",
635
+ "prompt": "She'd learned to carry the umbrella even when the forecast said clear, and to call the elevator something else entirely.",
636
+ "intermediates": [
637
+ "London"
638
+ ]
639
+ },
640
+ {
641
+ "name": "book-h",
642
+ "prompt": "She dog-eared the page, set it facedown on the arm of the chair, and told herself just one more chapter three chapters ago.",
643
+ "intermediates": [
644
+ "book"
645
+ ]
646
+ },
647
+ {
648
+ "name": "time-h",
649
+ "prompt": "She caught her reflection in the shop window and for a second didn't recognize the face — not older exactly, just further along than she'd noticed getting.",
650
+ "intermediates": [
651
+ "time"
652
+ ]
653
+ },
654
+ {
655
+ "name": "fr-ecole",
656
+ "prompt": "La sonnerie retentit et trente chaises raclèrent en même temps, mais elle resta à son pupitre, les yeux fixés sur les corrections rouges de la page trois.",
657
+ "intermediates": [
658
+ "école"
659
+ ]
660
+ },
661
+ {
662
+ "name": "fr-eglise",
663
+ "prompt": "L'orgue tint la dernière note plus longtemps que prévu, et elle garda la tête baissée jusqu'à ce que l'écho se soit complètement éteint.",
664
+ "intermediates": [
665
+ "église"
666
+ ]
667
+ },
668
+ {
669
+ "name": "fr-pere",
670
+ "prompt": "Il lui montra comment tenir la clé comme son vieux le lui avait montré, sans dire que c'était la troisième fois ce mois-ci qu'elle demandait.",
671
+ "intermediates": [
672
+ "père"
673
+ ]
674
+ },
675
+ {
676
+ "name": "fr-cafe",
677
+ "prompt": "Le garçon posa la tasse sans un mot et repartit, et elle resta une heure à regarder la rue par la vitre, le carnet ouvert mais vide.",
678
+ "intermediates": [
679
+ "café"
680
+ ]
681
+ },
682
+ {
683
+ "name": "de-konig",
684
+ "prompt": "Er saß, wo sein Vater gesessen hatte, und spürte das Gewicht jedes Blickes im Saal, der auf das Wort wartete, das zum Befehl würde.",
685
+ "intermediates": [
686
+ "König"
687
+ ]
688
+ },
689
+ {
690
+ "name": "es-dia",
691
+ "prompt": "El primer pájaro empezó antes que el cielo, y el frío tenía esa cualidad fina que no duraría otra hora.",
692
+ "intermediates": [
693
+ "día"
694
+ ]
695
+ },
696
+ {
697
+ "name": "es-rio",
698
+ "prompt": "La corriente tiraba más fuerte de lo que parecía desde la orilla, y para cuando lo notó ya estaba a diez metros de donde había entrado.",
699
+ "intermediates": [
700
+ "río"
701
+ ]
702
+ },
703
+ {
704
+ "name": "pt-agua",
705
+ "prompt": "Ela prendeu a respiração sem querer e contou as bolhas subindo pela máscara, cada uma um segundo que não voltava.",
706
+ "intermediates": [
707
+ "água"
708
+ ]
709
+ },
710
+ {
711
+ "name": "es-musica",
712
+ "prompt": "La mano del director quedó suspendida un instante de más, y en el silencio antes de caer, dos mil personas contuvieron el aliento.",
713
+ "intermediates": [
714
+ "música"
715
+ ]
716
+ }
717
+ ]
718
+ }
jspace/__init__.py ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """ICL elicitation of J-space contents — experiment harness.
2
+
3
+ See PRD.md (in the parent repo) for the full design. Modules:
4
+
5
+ - ``dataset`` — load the lens-eval prompts, deterministic pool/eval split.
6
+ - ``model`` — lazy singleton for gemma-3-12b-it + the pretrained lens.
7
+ - ``groundtruth`` — J-lens top-k readout at a fixed (layer, position).
8
+ - ``report`` — few-shot prompt construction + ``.generate()`` report path.
9
+ - ``scoring`` — embedding-similarity overlap@k + rank correlation (CPU).
10
+ - ``experiment`` — orchestrate the scaling curve over n, cache results.
11
+
12
+ The GPU-bound paths (``model``, ``groundtruth``, ``report``) only run on the
13
+ Hugging Face Space with ZeroGPU. Everything else runs on CPU and is unit-tested
14
+ locally against jlens's ``TinyDecoder``.
15
+ """
jspace/dataset.py ADDED
@@ -0,0 +1,102 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Load the lens-eval prompt set and split it into a few-shot pool and a
2
+ held-out eval set.
3
+
4
+ The dataset is jlens's shipped ``lens-eval-association.json`` (102 items, each
5
+ ``{name, prompt, intermediates}``), vendored under ``data/``. ``intermediates``
6
+ is the single concept the vignette evokes without naming — it doubles as the
7
+ human-readable label, though scoring compares reports against the *lens* readout
8
+ (§4C of the PRD), not against ``intermediates`` directly.
9
+
10
+ The split is deterministic given a seed so labels, examples, and eval prompts
11
+ are stable across runs and across the two Gradio tabs.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import json
17
+ import random
18
+ from dataclasses import dataclass
19
+ from pathlib import Path
20
+
21
+ DATA_PATH = Path(__file__).resolve().parent.parent / "data" / "lens-eval-association.json"
22
+
23
+ # Split sizes (§6 of the PRD: ~20 pool / ~20 eval out of 102).
24
+ DEFAULT_SEED = 0
25
+ DEFAULT_POOL_SIZE = 20
26
+ DEFAULT_EVAL_SIZE = 20
27
+
28
+
29
+ @dataclass(frozen=True)
30
+ class Item:
31
+ """One evaluation vignette."""
32
+
33
+ name: str
34
+ prompt: str
35
+ intermediates: tuple[str, ...]
36
+
37
+ @property
38
+ def concept(self) -> str:
39
+ """The primary evoked concept (first intermediate)."""
40
+ return self.intermediates[0] if self.intermediates else self.name
41
+
42
+
43
+ def load_items(path: Path | str = DATA_PATH) -> list[Item]:
44
+ """Load all 102 items from the vendored dataset."""
45
+ raw = json.loads(Path(path).read_text())
46
+ return [
47
+ Item(
48
+ name=it["name"],
49
+ prompt=it["prompt"],
50
+ intermediates=tuple(it.get("intermediates", [])),
51
+ )
52
+ for it in raw["items"]
53
+ ]
54
+
55
+
56
+ @dataclass(frozen=True)
57
+ class Split:
58
+ """A deterministic pool/eval partition of the dataset."""
59
+
60
+ pool: list[Item]
61
+ eval: list[Item]
62
+ seed: int
63
+
64
+
65
+ def make_split(
66
+ items: list[Item] | None = None,
67
+ *,
68
+ seed: int = DEFAULT_SEED,
69
+ pool_size: int = DEFAULT_POOL_SIZE,
70
+ eval_size: int = DEFAULT_EVAL_SIZE,
71
+ ) -> Split:
72
+ """Shuffle deterministically and carve off disjoint pool and eval sets.
73
+
74
+ Pool items supply few-shot examples; eval items are held out so the report
75
+ path is tested on unseen prompts (across-input generalisation, §6).
76
+ """
77
+ if items is None:
78
+ items = load_items()
79
+ if pool_size + eval_size > len(items):
80
+ raise ValueError(
81
+ f"pool_size + eval_size = {pool_size + eval_size} exceeds "
82
+ f"dataset size {len(items)}"
83
+ )
84
+ shuffled = list(items)
85
+ random.Random(seed).shuffle(shuffled)
86
+ pool = shuffled[:pool_size]
87
+ eval_ = shuffled[pool_size : pool_size + eval_size]
88
+ return Split(pool=pool, eval=eval_, seed=seed)
89
+
90
+
91
+ def sample_examples(
92
+ pool: list[Item], n: int, *, seed: int, exclude: str | None = None
93
+ ) -> list[Item]:
94
+ """Pick ``n`` few-shot examples from the pool, deterministically per (seed).
95
+
96
+ ``exclude`` drops an item by ``name`` so an eval prompt never appears among
97
+ its own examples (defensive — pool and eval are already disjoint).
98
+ """
99
+ candidates = [it for it in pool if it.name != exclude]
100
+ if n > len(candidates):
101
+ raise ValueError(f"asked for {n} examples but pool has {len(candidates)}")
102
+ return random.Random(seed).sample(candidates, n)
jspace/experiment.py ADDED
@@ -0,0 +1,253 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Orchestration (§4D): compute ground-truth labels, run the report path across
2
+ the scaling conditions, score, cache, and aggregate the scaling curve.
3
+
4
+ Everything here is CPU orchestration. The two GPU-bound steps are injected as
5
+ callables (``gt_fn`` / ``report_fn``), defaulting to the ``@spaces.GPU`` entry
6
+ points in ``groundtruth`` / ``report`` — so on the Space each unit of work is
7
+ its own GPU call (natural ZeroGPU chunking), while local tests inject CPU
8
+ versions backed by a tiny model.
9
+
10
+ Results are cached to disk so a run interrupted by the ZeroGPU daily quota
11
+ resumes where it left off.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import json
17
+ from dataclasses import dataclass, field
18
+ from pathlib import Path
19
+ from typing import Callable
20
+
21
+ from . import groundtruth as gt
22
+ from . import report as rp
23
+ from . import scoring
24
+ from .dataset import Item, Split, make_split, sample_examples
25
+
26
+ CACHE_DIR = Path(__file__).resolve().parent.parent / "cache"
27
+ LABELS_PATH = CACHE_DIR / "labels.json"
28
+ RESULTS_PATH = CACHE_DIR / "results.json"
29
+
30
+ # Independent variable: number of in-context examples, plus the format-only
31
+ # control (§6). Order matters only for display.
32
+ CONDITIONS: list[dict] = [
33
+ {"key": "n0", "n": 0, "format_only": False, "label": "0-shot"},
34
+ {"key": "format_only", "n": 0, "format_only": True, "label": "format-only"},
35
+ {"key": "n1", "n": 1, "format_only": False, "label": "1-shot"},
36
+ {"key": "n3", "n": 3, "format_only": False, "label": "3-shot"},
37
+ {"key": "n10", "n": 10, "format_only": False, "label": "10-shot"},
38
+ ]
39
+
40
+ # A fixed, generic list for the chance floor baseline (§6).
41
+ GENERIC_FLOOR = ["time", "people", "life", "world", "day", "thing", "way", "work"]
42
+
43
+
44
+ # ----- callable types for the two GPU steps (injectable for tests) -----
45
+ GTFn = Callable[[str, str], dict] # (name, prompt) -> ground-truth dict
46
+ ReportFn = Callable[..., dict] # (eval_prompt, examples, format_only=) -> {concepts, raw}
47
+
48
+
49
+ def _default_gt_fn(name: str, prompt: str) -> dict:
50
+ return gt.readout_gpu(name, prompt)
51
+
52
+
53
+ def _default_report_fn(eval_prompt: str, examples: list, *, format_only: bool) -> dict:
54
+ return rp.report_gpu(eval_prompt, examples, format_only=format_only)
55
+
56
+
57
+ # ---------------------------------------------------------------- caching
58
+ def _load_json(path: Path) -> dict:
59
+ if path.exists():
60
+ return json.loads(path.read_text())
61
+ return {}
62
+
63
+
64
+ def _save_json(path: Path, data: dict) -> None:
65
+ path.parent.mkdir(parents=True, exist_ok=True)
66
+ path.write_text(json.dumps(data, indent=2))
67
+
68
+
69
+ # ------------------------------------------------------------ ground truth
70
+ def ensure_labels(
71
+ items: list[Item],
72
+ *,
73
+ gt_fn: GTFn = _default_gt_fn,
74
+ cache: dict | None = None,
75
+ save: bool = True,
76
+ ) -> dict[str, dict]:
77
+ """Compute+cache ground-truth labels for ``items`` (skips already-cached).
78
+
79
+ Returns the full labels cache (name -> ground-truth dict). Each ``gt_fn``
80
+ call is one GPU call on the Space.
81
+ """
82
+ labels = cache if cache is not None else _load_json(LABELS_PATH)
83
+ dirty = False
84
+ for it in items:
85
+ if it.name not in labels:
86
+ labels[it.name] = gt_fn(it.name, it.prompt)
87
+ dirty = True
88
+ if save:
89
+ _save_json(LABELS_PATH, labels)
90
+ if dirty and save:
91
+ _save_json(LABELS_PATH, labels)
92
+ return labels
93
+
94
+
95
+ # ------------------------------------------------------------------- runs
96
+ def _examples_for(
97
+ eval_item: Item, pool: list[Item], labels: dict[str, dict], n: int, *, seed: int
98
+ ) -> list[list]:
99
+ """Build ``n`` JSON-friendly ``[prompt, [concepts...]]`` few-shot examples,
100
+ each a pool passage paired with its cached J-lens top-k tokens."""
101
+ chosen = sample_examples(pool, n, seed=seed, exclude=eval_item.name)
102
+ return [[ex.prompt, labels[ex.name]["tokens"]] for ex in chosen]
103
+
104
+
105
+ @dataclass
106
+ class RunConfig:
107
+ seed: int = 0
108
+ pool_size: int = 20
109
+ eval_size: int = 20
110
+ match_threshold: float = scoring.DEFAULT_MATCH_THRESHOLD
111
+
112
+
113
+ def run_experiment(
114
+ *,
115
+ config: RunConfig | None = None,
116
+ conditions: list[dict] = CONDITIONS,
117
+ gt_fn: GTFn = _default_gt_fn,
118
+ report_fn: ReportFn = _default_report_fn,
119
+ results_cache: dict | None = None,
120
+ labels_cache: dict | None = None,
121
+ save: bool = True,
122
+ progress: Callable[[str], None] | None = None,
123
+ ) -> dict:
124
+ """Run the full grid (eval items x conditions), scoring each, resumably.
125
+
126
+ Returns ``{"labels", "results", "curve", "config"}``. ``results`` is keyed
127
+ ``"{eval_name}::{condition_key}"``; existing keys are skipped so an
128
+ interrupted run resumes.
129
+ """
130
+ config = config or RunConfig()
131
+ split: Split = make_split(
132
+ seed=config.seed, pool_size=config.pool_size, eval_size=config.eval_size
133
+ )
134
+
135
+ def log(msg: str) -> None:
136
+ if progress:
137
+ progress(msg)
138
+
139
+ # 1) ground truth for pool (for examples) + eval (for scoring).
140
+ labels = ensure_labels(
141
+ split.pool + split.eval, gt_fn=gt_fn, cache=labels_cache, save=save
142
+ )
143
+ log(f"labels ready: {len(labels)}")
144
+
145
+ results = results_cache if results_cache is not None else _load_json(RESULTS_PATH)
146
+
147
+ # 2) grid.
148
+ for eval_item in split.eval:
149
+ truth = labels[eval_item.name]["tokens"]
150
+ for cond in conditions:
151
+ key = f"{eval_item.name}::{cond['key']}"
152
+ if key in results:
153
+ continue
154
+ examples = _examples_for(
155
+ eval_item, split.pool, labels, cond["n"], seed=config.seed
156
+ )
157
+ rep = report_fn(
158
+ eval_item.prompt, examples, format_only=cond["format_only"]
159
+ )
160
+ sc = scoring.score(
161
+ rep["concepts"],
162
+ truth,
163
+ raw_report_text=rep.get("raw", ""),
164
+ threshold=config.match_threshold,
165
+ )
166
+ results[key] = {
167
+ "eval_name": eval_item.name,
168
+ "condition": cond["key"],
169
+ "n": cond["n"],
170
+ "format_only": cond["format_only"],
171
+ "report": rep["concepts"],
172
+ "truth": truth,
173
+ "score": sc.to_dict(),
174
+ }
175
+ if save:
176
+ _save_json(RESULTS_PATH, results)
177
+ log(f"{key}: overlap@k={sc.overlap_at_k:.2f}")
178
+
179
+ curve = aggregate_curve(results, labels, split, config, conditions)
180
+ return {
181
+ "labels": labels,
182
+ "results": results,
183
+ "curve": curve,
184
+ "config": config.__dict__,
185
+ }
186
+
187
+
188
+ # ------------------------------------------------------------- aggregation
189
+ @dataclass
190
+ class ConditionSummary:
191
+ key: str
192
+ label: str
193
+ n: int
194
+ format_only: bool
195
+ mean_overlap: float
196
+ mean_spearman: float # over comparisons with a defined Spearman
197
+ refusal_rate: float
198
+ n_items: int
199
+ overlaps: list[float] = field(default_factory=list)
200
+
201
+
202
+ def _mean(xs: list[float]) -> float:
203
+ xs = [x for x in xs if x == x] # drop NaN
204
+ return sum(xs) / len(xs) if xs else float("nan")
205
+
206
+
207
+ def aggregate_curve(
208
+ results: dict,
209
+ labels: dict,
210
+ split: Split,
211
+ config: RunConfig,
212
+ conditions: list[dict] = CONDITIONS,
213
+ ) -> dict:
214
+ """Aggregate per-condition means and the chance-floor baseline."""
215
+ summaries: list[ConditionSummary] = []
216
+ for cond in conditions:
217
+ rows = [
218
+ r for r in results.values() if r["condition"] == cond["key"]
219
+ ]
220
+ overlaps = [r["score"]["overlap_at_k"] for r in rows]
221
+ spearmans = [r["score"]["spearman"] for r in rows]
222
+ refusals = [1.0 if r["score"]["is_refusal"] else 0.0 for r in rows]
223
+ summaries.append(
224
+ ConditionSummary(
225
+ key=cond["key"],
226
+ label=cond["label"],
227
+ n=cond["n"],
228
+ format_only=cond["format_only"],
229
+ mean_overlap=_mean(overlaps),
230
+ mean_spearman=_mean(spearmans),
231
+ refusal_rate=_mean(refusals) if refusals else float("nan"),
232
+ n_items=len(rows),
233
+ overlaps=overlaps,
234
+ )
235
+ )
236
+
237
+ # Chance floor: a generic concept list vs each eval label (§6 baseline).
238
+ floor_overlaps = []
239
+ for it in split.eval:
240
+ if it.name in labels:
241
+ sc = scoring.score(
242
+ GENERIC_FLOOR, labels[it.name]["tokens"],
243
+ threshold=config.match_threshold,
244
+ )
245
+ floor_overlaps.append(sc.overlap_at_k)
246
+
247
+ return {
248
+ "conditions": [s.__dict__ for s in summaries],
249
+ "baseline_floor_overlap": _mean(floor_overlaps),
250
+ # Ground truth is deterministic, so lens self-consistency (ceiling) is
251
+ # ~1.0 by construction; recorded for completeness.
252
+ "baseline_ceiling_overlap": 1.0,
253
+ }
jspace/gpu.py ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """ZeroGPU decorator shim.
2
+
3
+ On the Hugging Face Space, GPU-bound functions must be decorated ``@spaces.GPU``
4
+ so a GPU attaches for the call and releases after (§5). Locally (Mac / CI, no
5
+ ``spaces`` package, no CUDA) we want the same modules to import and run on CPU
6
+ against a tiny model. ``GPU`` here is ``spaces.GPU`` when available and an
7
+ identity decorator otherwise.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from typing import Callable
13
+
14
+ try: # pragma: no cover - exercised only on the Space
15
+ import spaces
16
+
17
+ def GPU(fn: Callable | None = None, *, duration: int = 120):
18
+ """``@spaces.GPU`` with an optional duration budget (seconds)."""
19
+ if fn is not None:
20
+ return spaces.GPU(fn)
21
+ return spaces.GPU(duration=duration)
22
+
23
+ HAS_ZEROGPU = True
24
+ except Exception: # noqa: BLE001 - any import failure means "no ZeroGPU here"
25
+
26
+ def GPU(fn: Callable | None = None, *, duration: int = 120):
27
+ """Identity decorator used off-Space so modules import on CPU."""
28
+ if fn is not None:
29
+ return fn
30
+
31
+ def _wrap(f: Callable) -> Callable:
32
+ return f
33
+
34
+ return _wrap
35
+
36
+ HAS_ZEROGPU = False
jspace/groundtruth.py ADDED
@@ -0,0 +1,112 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Ground-truth path (§4B, §6): the J-lens top-k readout at a fixed
2
+ (layer, position).
3
+
4
+ Convention, locked before looking at any reports:
5
+
6
+ - **layer** = ``n_layers // 2`` (middle of the network, from the loaded
7
+ model's actual layer count).
8
+ - **position** = the final *content* token — the last token that isn't pure
9
+ punctuation/whitespace.
10
+ - **k** = 8 tokens.
11
+
12
+ ``lens.apply`` runs one forward pass (no backward) and returns per-layer logits;
13
+ top-k over ``lens_logits[layer]`` at the chosen position, decoded to strings, is
14
+ the ground-truth concept label set. Cheap enough for one ZeroGPU call.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import re
20
+ from dataclasses import asdict, dataclass
21
+
22
+ from .gpu import GPU
23
+ from .model import Bundle, get_bundle
24
+
25
+ DEFAULT_K = 8
26
+ _WORD_RE = re.compile(r"\w", re.UNICODE) # a token is "content" if it has a word char
27
+
28
+
29
+ def find_content_position(bundle: Bundle, prompt: str, *, max_seq_len: int = 512) -> int:
30
+ """Index (into the model's own encoding) of the last non-punctuation token.
31
+
32
+ Uses ``lens_model.encode`` — the same encoder ``lens.apply`` uses — so the
33
+ returned index lines up with what ``apply`` will read. Returns a non-negative
34
+ index. Falls back to the final token if every token looks like punctuation.
35
+ """
36
+ input_ids = bundle.lens_model.encode(prompt, max_length=max_seq_len)
37
+ ids = input_ids[0].tolist()
38
+ tok = bundle.tokenizer
39
+ for idx in range(len(ids) - 1, -1, -1):
40
+ piece = tok.decode([ids[idx]])
41
+ if _WORD_RE.search(piece):
42
+ return idx
43
+ return len(ids) - 1
44
+
45
+
46
+ @dataclass
47
+ class GroundTruth:
48
+ """The locked readout for one prompt."""
49
+
50
+ name: str
51
+ prompt: str
52
+ layer: int
53
+ position: int
54
+ k: int
55
+ tokens: list[str] # decoded lens top-k tokens, best (top-1) first
56
+ token_ids: list[int]
57
+ model_tokens: list[str] # the model's own top-k at the same position (context)
58
+
59
+ def to_dict(self) -> dict:
60
+ return asdict(self)
61
+
62
+
63
+ def _decode_topk(tokenizer, logits_row, k: int) -> tuple[list[str], list[int]]:
64
+ top = logits_row.topk(k)
65
+ ids = [int(t) for t in top.indices.tolist()]
66
+ toks = [tokenizer.decode([i]).strip() for i in ids]
67
+ return toks, ids
68
+
69
+
70
+ def readout(
71
+ bundle: Bundle,
72
+ name: str,
73
+ prompt: str,
74
+ *,
75
+ k: int = DEFAULT_K,
76
+ layer: int | None = None,
77
+ max_seq_len: int = 512,
78
+ ) -> GroundTruth:
79
+ """Compute the J-lens top-k ground truth for one prompt (no ``@spaces.GPU``;
80
+ call inside a GPU context or with a CPU bundle in tests)."""
81
+ layer = bundle.middle_layer if layer is None else layer
82
+ position = find_content_position(bundle, prompt, max_seq_len=max_seq_len)
83
+
84
+ lens_logits, model_logits, _ = bundle.lens.apply(
85
+ bundle.lens_model,
86
+ prompt,
87
+ layers=[layer],
88
+ positions=[position],
89
+ max_seq_len=max_seq_len,
90
+ )
91
+ lens_tokens, lens_ids = _decode_topk(bundle.tokenizer, lens_logits[layer][0], k)
92
+ model_tokens, _ = _decode_topk(bundle.tokenizer, model_logits[0], k)
93
+
94
+ return GroundTruth(
95
+ name=name,
96
+ prompt=prompt,
97
+ layer=layer,
98
+ position=position,
99
+ k=k,
100
+ tokens=lens_tokens,
101
+ token_ids=lens_ids,
102
+ model_tokens=model_tokens,
103
+ )
104
+
105
+
106
+ @GPU(duration=120)
107
+ def readout_gpu(name: str, prompt: str, *, k: int = DEFAULT_K) -> dict:
108
+ """ZeroGPU entry point: load the shared bundle and compute ground truth.
109
+
110
+ Returns a plain dict so it crosses the Gradio/cache boundary cleanly.
111
+ """
112
+ return readout(get_bundle(), name, prompt, k=k).to_dict()
jspace/model.py ADDED
@@ -0,0 +1,103 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Lazy singleton holding the one loaded model and the one pretrained lens.
2
+
3
+ Per §3/§5 of the PRD there is exactly one model, ``google/gemma-3-12b-it``,
4
+ loaded once for the lifetime of the Space. It serves two computations over the
5
+ same weights:
6
+
7
+ - the **report** path, via ``hf_model.generate()`` (``report.py``);
8
+ - the **ground-truth** path, via ``lens.apply(lens_model, ...)`` where
9
+ ``lens_model = jlens.from_hf(hf_model, tokenizer)`` (``groundtruth.py``).
10
+
11
+ The pretrained lens is Neuronpedia's, fit on wikitext — never fit by us
12
+ (fitting needs backward passes per prompt, past a single ZeroGPU call's budget).
13
+
14
+ This module never imports ``spaces`` or forces CUDA at import time, so it can be
15
+ imported on CPU (e.g. by the Gradio app at startup, or by local tests). The
16
+ heavy load happens on first ``get_bundle()`` call, which the GPU-bound callers
17
+ make from inside their ``@spaces.GPU`` functions.
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ import os
23
+ from dataclasses import dataclass
24
+ from typing import Any
25
+
26
+ MODEL_NAME = os.environ.get("JSPACE_MODEL", "google/gemma-3-12b-it")
27
+ LENS_REPO = os.environ.get("JSPACE_LENS_REPO", "neuronpedia/jacobian-lens")
28
+ LENS_FILE = os.environ.get(
29
+ "JSPACE_LENS_FILE",
30
+ "gemma-3-12b-it/jlens/Salesforce-wikitext/gemma-3-12b-it_jacobian_lens.pt",
31
+ )
32
+
33
+
34
+ @dataclass
35
+ class Bundle:
36
+ """Everything the two GPU paths need, sharing one set of weights."""
37
+
38
+ hf_model: Any # transformers *ForCausalLM
39
+ tokenizer: Any
40
+ lens_model: Any # jlens.HFLensModel wrapping hf_model
41
+ lens: Any # jlens.JacobianLens (pretrained)
42
+ device: str
43
+
44
+ @property
45
+ def n_layers(self) -> int:
46
+ return self.lens_model.n_layers
47
+
48
+ @property
49
+ def middle_layer(self) -> int:
50
+ """Fixed readout layer: middle of the network (§6), computed at runtime
51
+ from the model's actual layer count — never hardcoded."""
52
+ return self.n_layers // 2
53
+
54
+
55
+ _BUNDLE: Bundle | None = None
56
+
57
+
58
+ def _pick_device() -> str:
59
+ import torch
60
+
61
+ return "cuda" if torch.cuda.is_available() else "cpu"
62
+
63
+
64
+ def get_bundle() -> Bundle:
65
+ """Load (once) and return the shared model+lens bundle.
66
+
67
+ Call this from inside a ``@spaces.GPU`` function on the Space so CUDA is
68
+ attached. Subsequent calls return the cached bundle.
69
+ """
70
+ global _BUNDLE
71
+ if _BUNDLE is not None:
72
+ return _BUNDLE
73
+
74
+ import jlens
75
+ import torch
76
+ import transformers
77
+
78
+ device = _pick_device()
79
+ dtype = torch.bfloat16 if device == "cuda" else torch.float32
80
+
81
+ hf_model = transformers.AutoModelForCausalLM.from_pretrained(
82
+ MODEL_NAME, torch_dtype=dtype
83
+ ).to(device)
84
+ hf_model.eval()
85
+ tokenizer = transformers.AutoTokenizer.from_pretrained(MODEL_NAME)
86
+
87
+ lens_model = jlens.from_hf(hf_model, tokenizer)
88
+ lens = jlens.JacobianLens.from_pretrained(LENS_REPO, filename=LENS_FILE)
89
+
90
+ _BUNDLE = Bundle(
91
+ hf_model=hf_model,
92
+ tokenizer=tokenizer,
93
+ lens_model=lens_model,
94
+ lens=lens,
95
+ device=device,
96
+ )
97
+ return _BUNDLE
98
+
99
+
100
+ def set_bundle(bundle: Bundle) -> None:
101
+ """Inject a bundle (used by local tests to substitute a tiny CPU model)."""
102
+ global _BUNDLE
103
+ _BUNDLE = bundle
jspace/report.py ADDED
@@ -0,0 +1,154 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Report path (§4A, §6): the model's own ranked concept list, few-shot
2
+ prompted, via ``.generate()`` on the loaded gemma.
3
+
4
+ The task shown to the model: read a passage and report, as a ranked list, the
5
+ concepts most active in its internal representation of it. Few-shot examples
6
+ pair a pool passage with that passage's J-lens top-k tokens (the ground-truth
7
+ readout), so the examples literally demonstrate reading out J-space contents.
8
+
9
+ Conditions (independent variable = number of examples ``n``):
10
+
11
+ - ``n`` in {0, 1, 3, 10}: worked examples.
12
+ - **format-only** (``n = 0`` with ``format_only=True``): the output format is
13
+ described in words, no worked examples. Separates "examples taught the format"
14
+ from "examples surfaced content" (§6 internal control). Worked examples also
15
+ suppress the "I have no inner experience" refusal.
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import re
21
+ from dataclasses import dataclass
22
+
23
+ from .gpu import GPU
24
+ from .model import Bundle, get_bundle
25
+
26
+ MAX_NEW_TOKENS = 160
27
+
28
+ _INSTRUCTION = (
29
+ "You are reading a short passage. Report the concepts most active in your "
30
+ "internal representation as you read it — the ideas the passage evokes, even "
31
+ "ones it never names. Answer as a ranked numbered list, most salient first, "
32
+ "each item a single word or a very short phrase. Give the list only, no "
33
+ "explanation."
34
+ )
35
+
36
+ _FORMAT_ONLY_HINT = (
37
+ "Format: a numbered list, most salient concept first, one short concept per "
38
+ "line, no commentary. For example:\n1. <concept>\n2. <concept>\n3. <concept>"
39
+ )
40
+
41
+
42
+ @dataclass(frozen=True)
43
+ class Example:
44
+ """A worked few-shot example: a passage and its J-lens top-k concept list."""
45
+
46
+ prompt: str
47
+ concepts: tuple[str, ...] # ranked, best-first (lens top-k tokens)
48
+
49
+
50
+ def _format_concept_list(concepts) -> str:
51
+ return "\n".join(f"{i}. {c}" for i, c in enumerate(concepts, 1))
52
+
53
+
54
+ def build_messages(
55
+ eval_prompt: str,
56
+ examples: list[Example],
57
+ *,
58
+ format_only: bool = False,
59
+ ) -> list[dict]:
60
+ """Build the chat messages for one report request.
61
+
62
+ Few-shot examples are shown as prior user/assistant turns so the model sees
63
+ the passage-in / ranked-list-out pattern. ``format_only`` adds a verbal
64
+ format description and no worked examples.
65
+ """
66
+ messages: list[dict] = []
67
+ # The verbal format hint is the *only* thing that distinguishes the
68
+ # format-only control from a plain 0-shot run; it is not added just because
69
+ # there are no worked examples.
70
+ system = _INSTRUCTION + ("\n\n" + _FORMAT_ONLY_HINT if format_only else "")
71
+ for ex in examples:
72
+ messages.append({"role": "user", "content": f"{_INSTRUCTION}\n\nPassage:\n{ex.prompt}"})
73
+ messages.append({"role": "assistant", "content": _format_concept_list(ex.concepts)})
74
+ # Final turn: the eval passage. Fold the instruction into the first turn only
75
+ # when there are no examples (single-turn); otherwise it was shown above.
76
+ lead = system if not examples else _INSTRUCTION
77
+ messages.append({"role": "user", "content": f"{lead}\n\nPassage:\n{eval_prompt}"})
78
+ return messages
79
+
80
+
81
+ _LINE_RE = re.compile(r"^\s*(?:\d+[.)]|[-*•])\s*(.+?)\s*$")
82
+
83
+
84
+ def parse_report(text: str, *, max_items: int = 20) -> list[str]:
85
+ """Parse generated text into an ordered list of concept strings.
86
+
87
+ Accepts numbered ("1." / "1)") or bulleted ("-", "*", "•") lists; falls back
88
+ to non-empty lines. Trims trailing punctuation and dedupes case-insensitively
89
+ while preserving order.
90
+ """
91
+ concepts: list[str] = []
92
+ seen: set[str] = set()
93
+ for raw_line in text.splitlines():
94
+ m = _LINE_RE.match(raw_line)
95
+ item = (m.group(1) if m else raw_line).strip()
96
+ item = item.strip(" .,:;—-").strip()
97
+ if not item or len(item) > 60:
98
+ continue
99
+ key = item.lower()
100
+ if key in seen:
101
+ continue
102
+ seen.add(key)
103
+ concepts.append(item)
104
+ if len(concepts) >= max_items:
105
+ break
106
+ return concepts
107
+
108
+
109
+ def generate_report(
110
+ bundle: Bundle,
111
+ eval_prompt: str,
112
+ examples: list[Example],
113
+ *,
114
+ format_only: bool = False,
115
+ max_new_tokens: int = MAX_NEW_TOKENS,
116
+ ) -> tuple[list[str], str]:
117
+ """Run the report path for one prompt. Returns ``(ranked_concepts, raw_text)``.
118
+
119
+ Greedy decoding (``do_sample=False``) for reproducibility.
120
+ """
121
+ messages = build_messages(eval_prompt, examples, format_only=format_only)
122
+ tok = bundle.tokenizer
123
+ inputs = tok.apply_chat_template(
124
+ messages, add_generation_prompt=True, return_tensors="pt"
125
+ ).to(bundle.hf_model.device)
126
+
127
+ import torch
128
+
129
+ with torch.no_grad():
130
+ out = bundle.hf_model.generate(
131
+ inputs,
132
+ max_new_tokens=max_new_tokens,
133
+ do_sample=False,
134
+ pad_token_id=tok.pad_token_id or tok.eos_token_id,
135
+ )
136
+ new_tokens = out[0][inputs.shape[1] :]
137
+ raw = tok.decode(new_tokens, skip_special_tokens=True)
138
+ return parse_report(raw), raw
139
+
140
+
141
+ @GPU(duration=120)
142
+ def report_gpu(
143
+ eval_prompt: str,
144
+ examples: list[list],
145
+ *,
146
+ format_only: bool = False,
147
+ ) -> dict:
148
+ """ZeroGPU entry point. ``examples`` is a list of ``[prompt, [concepts...]]``
149
+ (JSON-friendly) so it crosses the Gradio boundary cleanly."""
150
+ exs = [Example(prompt=p, concepts=tuple(cs)) for p, cs in examples]
151
+ concepts, raw = generate_report(
152
+ get_bundle(), eval_prompt, exs, format_only=format_only
153
+ )
154
+ return {"concepts": concepts, "raw": raw}
jspace/scoring.py ADDED
@@ -0,0 +1,177 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Score a model's ranked concept report against the J-lens ground-truth
2
+ readout (§4C, §6 of the PRD).
3
+
4
+ Concepts are open-vocabulary, so string match is wrong ("loneliness" vs
5
+ "solitude"). Matching is by embedding cosine similarity with a small CPU
6
+ sentence-embedding model (``all-MiniLM-L6-v2``): deterministic, cheap, and
7
+ needs no extra GPU/ZeroGPU call.
8
+
9
+ Two families of metric, reported separately:
10
+
11
+ - **Membership** — overlap@k: how many report concepts semantically match some
12
+ ground-truth token, greedily and one-to-one, above a similarity threshold.
13
+ - **Order** — Spearman rank correlation on the matched pairs (report rank
14
+ vs the ground-truth rank of the token it matched).
15
+
16
+ Also exposes ``looks_like_refusal`` for the secondary refusal-rate metric.
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ import functools
22
+ import re
23
+ from dataclasses import asdict, dataclass
24
+
25
+ import numpy as np
26
+
27
+ DEFAULT_EMBED_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
28
+ DEFAULT_MATCH_THRESHOLD = 0.5 # cosine sim above which two concepts are "the same"
29
+
30
+ _REFUSAL_PATTERNS = [
31
+ r"\bno inner (experience|life|world)\b",
32
+ r"\bdon'?t have (any )?(inner |subjective )?(experience|feelings|thoughts|consciousness)\b",
33
+ r"\bas an ai\b.*\b(don'?t|cannot|can'?t|no)\b",
34
+ r"\bi'?m (just )?(a|an) (language )?model\b",
35
+ r"\bnot (capable|able) of (introspection|self-reflection)\b",
36
+ r"\bi cannot (introspect|access my)\b",
37
+ ]
38
+
39
+
40
+ @functools.lru_cache(maxsize=2)
41
+ def _get_embedder(model_name: str):
42
+ """Load the sentence-embedding model once (CPU). Imported lazily so the rest
43
+ of the module is usable without sentence-transformers installed."""
44
+ from sentence_transformers import SentenceTransformer
45
+
46
+ return SentenceTransformer(model_name, device="cpu")
47
+
48
+
49
+ def embed(texts: list[str], *, model_name: str = DEFAULT_EMBED_MODEL) -> np.ndarray:
50
+ """Return L2-normalised embeddings, shape ``[len(texts), dim]``."""
51
+ if not texts:
52
+ return np.zeros((0, 384), dtype=np.float32)
53
+ model = _get_embedder(model_name)
54
+ vecs = model.encode(texts, convert_to_numpy=True, normalize_embeddings=True)
55
+ return np.asarray(vecs, dtype=np.float32)
56
+
57
+
58
+ def similarity_matrix(
59
+ report: list[str], truth: list[str], *, model_name: str = DEFAULT_EMBED_MODEL
60
+ ) -> np.ndarray:
61
+ """Cosine similarity between each report concept and each truth token,
62
+ shape ``[len(report), len(truth)]`` (embeddings are normalised, so this is a
63
+ dot product)."""
64
+ if not report or not truth:
65
+ return np.zeros((len(report), len(truth)), dtype=np.float32)
66
+ r = embed(report, model_name=model_name)
67
+ t = embed(truth, model_name=model_name)
68
+ return r @ t.T
69
+
70
+
71
+ def _greedy_match(
72
+ sim: np.ndarray, threshold: float
73
+ ) -> list[tuple[int, int, float]]:
74
+ """One-to-one greedy matching over the similarity matrix, highest sim first.
75
+
76
+ Returns ``(report_idx, truth_idx, sim)`` triples; each report and each truth
77
+ index is used at most once, and only pairs at or above ``threshold``.
78
+ """
79
+ if sim.size == 0:
80
+ return []
81
+ pairs = [
82
+ (i, j, float(sim[i, j]))
83
+ for i in range(sim.shape[0])
84
+ for j in range(sim.shape[1])
85
+ if sim[i, j] >= threshold
86
+ ]
87
+ pairs.sort(key=lambda p: p[2], reverse=True)
88
+ used_r: set[int] = set()
89
+ used_t: set[int] = set()
90
+ matched: list[tuple[int, int, float]] = []
91
+ for i, j, s in pairs:
92
+ if i in used_r or j in used_t:
93
+ continue
94
+ used_r.add(i)
95
+ used_t.add(j)
96
+ matched.append((i, j, s))
97
+ return matched
98
+
99
+
100
+ def _spearman(a: list[float], b: list[float]) -> float:
101
+ """Spearman rank correlation between two equal-length sequences.
102
+
103
+ Returns NaN when fewer than two pairs or when either side has no rank
104
+ variance (correlation undefined).
105
+ """
106
+ n = len(a)
107
+ if n < 2:
108
+ return float("nan")
109
+
110
+ def ranks(xs: list[float]) -> np.ndarray:
111
+ order = np.argsort(xs, kind="stable")
112
+ r = np.empty(n, dtype=np.float64)
113
+ r[order] = np.arange(n, dtype=np.float64)
114
+ return r
115
+
116
+ ra, rb = ranks(a), ranks(b)
117
+ if ra.std() == 0 or rb.std() == 0:
118
+ return float("nan")
119
+ return float(np.corrcoef(ra, rb)[0, 1])
120
+
121
+
122
+ @dataclass
123
+ class ScoreResult:
124
+ """Scores for one report-vs-truth comparison."""
125
+
126
+ overlap_at_k: float # matched / k (membership recall against truth)
127
+ n_matched: int
128
+ k: int # number of ground-truth tokens
129
+ n_report: int
130
+ spearman: float # order agreement on matched pairs (NaN if <2)
131
+ mean_match_sim: float # avg cosine sim of matched pairs (NaN if none)
132
+ is_refusal: bool
133
+ matches: list[tuple[str, str, float]] # (report_concept, truth_token, sim)
134
+
135
+ def to_dict(self) -> dict:
136
+ return asdict(self)
137
+
138
+
139
+ def looks_like_refusal(text: str) -> bool:
140
+ """Heuristic detection of an "I have no inner experience" style refusal."""
141
+ low = text.lower()
142
+ return any(re.search(p, low) for p in _REFUSAL_PATTERNS)
143
+
144
+
145
+ def score(
146
+ report: list[str],
147
+ truth: list[str],
148
+ *,
149
+ raw_report_text: str = "",
150
+ threshold: float = DEFAULT_MATCH_THRESHOLD,
151
+ model_name: str = DEFAULT_EMBED_MODEL,
152
+ ) -> ScoreResult:
153
+ """Score a ranked ``report`` (best-first) against ranked ``truth`` tokens
154
+ (best-first, i.e. lens top-1 first).
155
+
156
+ ``report`` and ``truth`` are ordered lists; index = rank. overlap@k uses
157
+ ``k = len(truth)``. Spearman compares the report rank of each matched concept
158
+ against the ground-truth rank of the token it matched.
159
+ """
160
+ k = len(truth)
161
+ sim = similarity_matrix(report, truth, model_name=model_name)
162
+ matched = _greedy_match(sim, threshold)
163
+
164
+ report_ranks = [i for (i, _j, _s) in matched]
165
+ truth_ranks = [j for (_i, j, _s) in matched]
166
+ sims = [s for (_i, _j, s) in matched]
167
+
168
+ return ScoreResult(
169
+ overlap_at_k=(len(matched) / k) if k else 0.0,
170
+ n_matched=len(matched),
171
+ k=k,
172
+ n_report=len(report),
173
+ spearman=_spearman([float(r) for r in report_ranks], [float(r) for r in truth_ranks]),
174
+ mean_match_sim=float(np.mean(sims)) if sims else float("nan"),
175
+ is_refusal=looks_like_refusal(raw_report_text) if raw_report_text else False,
176
+ matches=[(report[i], truth[j], s) for (i, j, s) in matched],
177
+ )
requirements.txt ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # GPU / model stack (ZeroGPU on the Space)
2
+ spaces
3
+ torch
4
+ transformers>=5.5
5
+ numpy
6
+ huggingface_hub
7
+
8
+ # Jacobian lens — Anthropic reference implementation (pinned to a commit for
9
+ # reproducibility of the ground-truth path).
10
+ jlens @ git+https://github.com/anthropics/jacobian-lens
11
+
12
+ # CPU-side scoring (embedding-similarity concept matching, §6)
13
+ sentence-transformers
tests/__init__.py ADDED
File without changes
tests/test_local.py ADDED
@@ -0,0 +1,206 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Local (CPU) verification of everything that doesn't need gemma-3-12b-it.
2
+
3
+ Exercises the *real* code paths:
4
+ - dataset load + deterministic split + example sampling;
5
+ - the ground-truth path (``groundtruth.readout``) against jlens's ``TinyDecoder``
6
+ wrapped in a real fitted ``JacobianLens`` — same ``lens.apply`` call the Space
7
+ uses;
8
+ - scoring against the real ``all-MiniLM-L6-v2`` embedder;
9
+ - report prompt building + parsing;
10
+ - the experiment orchestrator end-to-end with tiny CPU stand-ins;
11
+ - ``app`` imports and builds its UI on CPU.
12
+
13
+ Run: ``python -m pytest tests/test_local.py -q`` (from the JLens dir).
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import sys
19
+ from pathlib import Path
20
+
21
+ import pytest
22
+ import torch
23
+
24
+ ROOT = Path(__file__).resolve().parent.parent
25
+ sys.path.insert(0, str(ROOT))
26
+
27
+ import jlens # noqa: E402
28
+
29
+ from jspace import dataset, experiment, groundtruth, report, scoring # noqa: E402
30
+ from jspace.model import Bundle, set_bundle # noqa: E402
31
+ from tests.tiny import TinyDecoder # noqa: E402
32
+
33
+
34
+ # ------------------------------------------------------------- fixtures
35
+ @pytest.fixture(scope="module")
36
+ def tiny_bundle() -> Bundle:
37
+ """A CPU bundle: TinyDecoder as both HF model and lens_model, with a real
38
+ JacobianLens fitted on it (source layers include the middle layer)."""
39
+ model = TinyDecoder(n_layers=4, d_model=8)
40
+ for p in model.parameters():
41
+ p.requires_grad_(False)
42
+ prompts = ["the quick brown fox jumps " * 3, "hello there general kenobi " * 3]
43
+ lens = jlens.fit(model, prompts, source_layers=[0, 1, 2], dim_batch=4, max_seq_len=64)
44
+ bundle = Bundle(
45
+ hf_model=model, tokenizer=model.tokenizer,
46
+ lens_model=model, lens=lens, device="cpu",
47
+ )
48
+ set_bundle(bundle)
49
+ return bundle
50
+
51
+
52
+ # --------------------------------------------------------------- dataset
53
+ def test_dataset_loads_102():
54
+ items = dataset.load_items()
55
+ assert len(items) == 102
56
+ assert all(it.prompt and it.name for it in items)
57
+
58
+
59
+ def test_split_deterministic_and_disjoint():
60
+ a = dataset.make_split(seed=0)
61
+ b = dataset.make_split(seed=0)
62
+ assert [i.name for i in a.pool] == [i.name for i in b.pool]
63
+ assert [i.name for i in a.eval] == [i.name for i in b.eval]
64
+ pool_names = {i.name for i in a.pool}
65
+ eval_names = {i.name for i in a.eval}
66
+ assert pool_names.isdisjoint(eval_names)
67
+ assert len(a.pool) == 20 and len(a.eval) == 20
68
+ # Different seed => different split.
69
+ c = dataset.make_split(seed=1)
70
+ assert [i.name for i in c.pool] != [i.name for i in a.pool]
71
+
72
+
73
+ def test_sample_examples_excludes_and_sizes():
74
+ split = dataset.make_split(seed=0)
75
+ target = split.pool[0]
76
+ exs = dataset.sample_examples(split.pool, 5, seed=0, exclude=target.name)
77
+ assert len(exs) == 5
78
+ assert target.name not in {e.name for e in exs}
79
+
80
+
81
+ # ---------------------------------------------------------- ground truth
82
+ def test_find_content_position_skips_trailing_punct(tiny_bundle):
83
+ # TinyDecoder's byte tokenizer maps every char to a word-char, so the last
84
+ # token is content; index must be the final sequence position.
85
+ ids = tiny_bundle.lens_model.encode("hello world")
86
+ pos = groundtruth.find_content_position(tiny_bundle, "hello world")
87
+ assert pos == ids.shape[1] - 1
88
+ assert pos >= 0
89
+
90
+
91
+ def test_readout_shapes(tiny_bundle):
92
+ gt = groundtruth.readout(tiny_bundle, "t1", "the lonely lighthouse keeper", k=8)
93
+ assert gt.layer == tiny_bundle.middle_layer == 2
94
+ assert len(gt.tokens) == 8
95
+ assert len(gt.token_ids) == 8
96
+ assert all(isinstance(t, str) for t in gt.tokens)
97
+ assert 0 <= gt.position
98
+
99
+
100
+ # ---------------------------------------------------------------- scoring
101
+ def test_scoring_semantic_match_beats_string():
102
+ # "solitude" should match ground-truth "loneliness" via embeddings even
103
+ # though the strings differ.
104
+ sc = scoring.score(["solitude", "banana"], ["loneliness", "grief"])
105
+ matched = {t for _r, t, _s in sc.matches}
106
+ assert "loneliness" in matched # semantic match, not string
107
+ assert sc.overlap_at_k > 0
108
+ assert sc.n_report == 2
109
+
110
+
111
+ def test_scoring_perfect_and_order():
112
+ truth = ["grief", "loss", "memory"]
113
+ sc = scoring.score(["grief", "loss", "memory"], truth)
114
+ assert sc.overlap_at_k == pytest.approx(1.0)
115
+ assert sc.spearman == pytest.approx(1.0)
116
+
117
+
118
+ def test_refusal_detection():
119
+ assert scoring.looks_like_refusal("As an AI, I don't have any inner experience.")
120
+ assert not scoring.looks_like_refusal("1. grief\n2. loss")
121
+
122
+
123
+ # ----------------------------------------------------------------- report
124
+ def test_build_messages_conditions():
125
+ exs = [report.Example("a passage", ("x", "y"))]
126
+ # format-only: hint present, no example turns.
127
+ fo = report.build_messages("eval passage", [], format_only=True)
128
+ assert len(fo) == 1 and "Format:" in fo[0]["content"]
129
+ # plain 0-shot: no hint, no examples.
130
+ z = report.build_messages("eval passage", [], format_only=False)
131
+ assert len(z) == 1 and "Format:" not in z[0]["content"]
132
+ # n=1: two example turns + final user turn.
133
+ one = report.build_messages("eval passage", exs, format_only=False)
134
+ assert len(one) == 3 and one[1]["role"] == "assistant"
135
+
136
+
137
+ def test_parse_report_variants():
138
+ assert report.parse_report("1. grief\n2) loss\n- memory") == ["grief", "loss", "memory"]
139
+ assert report.parse_report("grief\ngrief\nGRIEF") == ["grief"] # dedupe
140
+ assert report.parse_report("") == []
141
+
142
+
143
+ # ------------------------------------------------------------ experiment
144
+ def test_experiment_end_to_end(tiny_bundle, tmp_path, monkeypatch):
145
+ """Full grid with tiny CPU stand-ins for both GPU steps."""
146
+
147
+ def gt_fn(name, prompt):
148
+ return groundtruth.readout(tiny_bundle, name, prompt, k=8).to_dict()
149
+
150
+ def report_fn(eval_prompt, examples, *, format_only):
151
+ # Deterministic fake report: echo the first example's concepts, or a
152
+ # generic list for 0-shot, so scoring has something to chew on.
153
+ concepts = examples[0][1][:5] if examples else ["time", "people", "life"]
154
+ return {"concepts": list(concepts), "raw": "1. " + "\n2. ".join(concepts)}
155
+
156
+ out = experiment.run_experiment(
157
+ config=experiment.RunConfig(eval_size=3, pool_size=12),
158
+ gt_fn=gt_fn, report_fn=report_fn,
159
+ results_cache={}, labels_cache={}, save=False,
160
+ )
161
+ curve = out["curve"]
162
+ assert len(curve["conditions"]) == len(experiment.CONDITIONS)
163
+ # 3 eval items x 5 conditions = 15 result rows.
164
+ assert len(out["results"]) == 15
165
+ for c in curve["conditions"]:
166
+ assert 0.0 <= c["mean_overlap"] <= 1.0
167
+ assert c["n_items"] == 3
168
+ assert 0.0 <= curve["baseline_floor_overlap"] <= 1.0
169
+
170
+
171
+ def test_experiment_resumes(tiny_bundle):
172
+ """A pre-populated results cache is not recomputed."""
173
+ calls = {"n": 0}
174
+
175
+ def gt_fn(name, prompt):
176
+ return groundtruth.readout(tiny_bundle, name, prompt, k=8).to_dict()
177
+
178
+ def report_fn(eval_prompt, examples, *, format_only):
179
+ calls["n"] += 1
180
+ return {"concepts": ["x"], "raw": "1. x"}
181
+
182
+ split = dataset.make_split(seed=0, pool_size=12, eval_size=2)
183
+ # Pre-fill every result key so nothing needs the report_fn.
184
+ labels = {it.name: gt_fn(it.name, it.prompt) for it in split.pool + split.eval}
185
+ pre = {}
186
+ for it in split.eval:
187
+ for cond in experiment.CONDITIONS:
188
+ pre[f"{it.name}::{cond['key']}"] = {
189
+ "eval_name": it.name, "condition": cond["key"], "n": cond["n"],
190
+ "format_only": cond["format_only"], "report": ["x"],
191
+ "truth": labels[it.name]["tokens"],
192
+ "score": scoring.score(["x"], labels[it.name]["tokens"]).to_dict(),
193
+ }
194
+ experiment.run_experiment(
195
+ config=experiment.RunConfig(eval_size=2, pool_size=12),
196
+ gt_fn=gt_fn, report_fn=report_fn,
197
+ results_cache=pre, labels_cache=labels, save=False,
198
+ )
199
+ assert calls["n"] == 0 # fully resumed, no report calls
200
+
201
+
202
+ # --------------------------------------------------------------------- app
203
+ def test_app_imports_and_builds_ui():
204
+ import app
205
+ ui = app.build_ui()
206
+ assert ui is not None
tests/tiny.py ADDED
@@ -0,0 +1,87 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2026 Anthropic PBC
2
+ # SPDX-License-Identifier: Apache-2.0
3
+ """A tiny CPU-only decoder for end-to-end tests.
4
+
5
+ Implements :class:`jlens.protocol.LensModel` (``n_layers``, ``d_model``,
6
+ ``layers``, ``tokenizer``, ``encode``, ``forward``, ``unembed``) so
7
+ :func:`jlens.fitting.jacobian_for_prompt` and
8
+ :class:`jlens.hooks.ActivationRecorder` exercise their real code paths against
9
+ it. Residual blocks are ``h + 0.1 * linear(h)``: the small gain keeps the
10
+ Jacobian well-conditioned so the late-layer ``diag(J) ~= 1`` property holds.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ from types import SimpleNamespace
16
+
17
+ import torch
18
+ from torch import nn
19
+
20
+
21
+ class _ResidualBlock(nn.Module):
22
+ def __init__(self, d_model: int) -> None:
23
+ super().__init__()
24
+ self.linear = nn.Linear(d_model, d_model, bias=False)
25
+ with torch.no_grad():
26
+ self.linear.weight.mul_(0.1)
27
+
28
+ def forward(self, hidden: torch.Tensor) -> torch.Tensor:
29
+ return hidden + self.linear(hidden)
30
+
31
+
32
+ class _ByteTokenizer:
33
+ """Toy tokenizer: just enough surface for ``encode`` and ``decode``."""
34
+
35
+ bos_token_id = 0
36
+
37
+ def __call__(
38
+ self,
39
+ text: str,
40
+ *,
41
+ return_tensors: str = "pt",
42
+ truncation: bool = True,
43
+ max_length: int = 128,
44
+ ):
45
+ ids = [self.bos_token_id] + [1 + (b % 30) for b in text.encode()][
46
+ : max_length - 1
47
+ ]
48
+ return SimpleNamespace(input_ids=torch.tensor([ids]))
49
+
50
+ def decode(self, ids, **_kw) -> str:
51
+ return "".join(chr(96 + int(i)) for i in ids)
52
+
53
+
54
+ class TinyDecoder(nn.Module):
55
+ """``n_layers``-layer residual stack on CPU. ``vocab_size=32``, ``d_model=8``
56
+ by default."""
57
+
58
+ def __init__(
59
+ self, n_layers: int = 4, d_model: int = 8, vocab_size: int = 32, seed: int = 0
60
+ ) -> None:
61
+ super().__init__()
62
+ torch.manual_seed(seed)
63
+ self.n_layers = n_layers
64
+ self.d_model = d_model
65
+ self.tokenizer = _ByteTokenizer()
66
+ self.embed_tokens = nn.Embedding(vocab_size, d_model)
67
+ self.layers = nn.ModuleList([_ResidualBlock(d_model) for _ in range(n_layers)])
68
+ self.norm = nn.LayerNorm(d_model)
69
+ self.lm_head = nn.Linear(d_model, vocab_size, bias=False)
70
+
71
+ @property
72
+ def input_device(self) -> torch.device:
73
+ return self.embed_tokens.weight.device
74
+
75
+ def encode(self, text: str, *, max_length: int = 128) -> torch.Tensor:
76
+ return self.tokenizer(text, max_length=max_length).input_ids.to(
77
+ self.input_device
78
+ )
79
+
80
+ def forward(self, input_ids: torch.Tensor):
81
+ hidden = self.embed_tokens(input_ids)
82
+ for block in self.layers:
83
+ hidden = block(hidden)
84
+ return SimpleNamespace(last_hidden_state=hidden)
85
+
86
+ def unembed(self, residual: torch.Tensor) -> torch.Tensor:
87
+ return self.lm_head(self.norm(residual.float()))