Add OraRL training and evaluation code

.gitattributes

@@ -1,2 +1,43 @@
 *.safetensors filter=lfs diff=lfs merge=lfs -text
 tokenizer.json filter=lfs diff=lfs merge=lfs -text
+code/assets/orarl-data-scaling.gif filter=lfs diff=lfs merge=lfs -text
+code/assets/orarl-hero.gif filter=lfs diff=lfs merge=lfs -text
+code/assets/orarl-method.gif filter=lfs diff=lfs merge=lfs -text
+code/assets/orarl-model-scaling.gif filter=lfs diff=lfs merge=lfs -text
+code/assets/orarl-teaser.mp4 filter=lfs diff=lfs merge=lfs -text
+code/assets/paper-data-scaling.png filter=lfs diff=lfs merge=lfs -text
+code/assets/paper-framework.png filter=lfs diff=lfs merge=lfs -text
+code/assets/paper-model-scaling.png filter=lfs diff=lfs merge=lfs -text
+code/assets/paper-results.png filter=lfs diff=lfs merge=lfs -text
+code/runs/smoke-training-9b-20260825_175949/grpo/global_step_1/actor/model_world_size_8_rank_0.pt filter=lfs diff=lfs merge=lfs -text
+code/runs/smoke-training-9b-20260825_175949/grpo/global_step_1/actor/model_world_size_8_rank_1.pt filter=lfs diff=lfs merge=lfs -text
+code/runs/smoke-training-9b-20260825_175949/grpo/global_step_1/actor/model_world_size_8_rank_2.pt filter=lfs diff=lfs merge=lfs -text
+code/runs/smoke-training-9b-20260825_175949/grpo/global_step_1/actor/model_world_size_8_rank_3.pt filter=lfs diff=lfs merge=lfs -text
+code/runs/smoke-training-9b-20260825_175949/grpo/global_step_1/actor/model_world_size_8_rank_4.pt filter=lfs diff=lfs merge=lfs -text
+code/runs/smoke-training-9b-20260825_175949/grpo/global_step_1/actor/model_world_size_8_rank_5.pt filter=lfs diff=lfs merge=lfs -text
+code/runs/smoke-training-9b-20260825_175949/grpo/global_step_1/actor/model_world_size_8_rank_6.pt filter=lfs diff=lfs merge=lfs -text
+code/runs/smoke-training-9b-20260825_175949/grpo/global_step_1/actor/model_world_size_8_rank_7.pt filter=lfs diff=lfs merge=lfs -text
+code/runs/smoke-training-9b-20260825_175949/grpo/global_step_1/actor/optim_world_size_8_rank_0.pt filter=lfs diff=lfs merge=lfs -text
+code/runs/smoke-training-9b-20260825_175949/grpo/global_step_1/actor/optim_world_size_8_rank_1.pt filter=lfs diff=lfs merge=lfs -text
+code/runs/smoke-training-9b-20260825_175949/grpo/global_step_1/actor/optim_world_size_8_rank_2.pt filter=lfs diff=lfs merge=lfs -text
+code/runs/smoke-training-9b-20260825_175949/grpo/global_step_1/actor/optim_world_size_8_rank_3.pt filter=lfs diff=lfs merge=lfs -text
+code/runs/smoke-training-9b-20260825_175949/grpo/global_step_1/actor/optim_world_size_8_rank_4.pt filter=lfs diff=lfs merge=lfs -text
+code/runs/smoke-training-9b-20260825_175949/grpo/global_step_1/actor/optim_world_size_8_rank_5.pt filter=lfs diff=lfs merge=lfs -text
+code/runs/smoke-training-9b-20260825_175949/grpo/global_step_1/actor/optim_world_size_8_rank_6.pt filter=lfs diff=lfs merge=lfs -text
+code/runs/smoke-training-9b-20260825_175949/grpo/global_step_1/actor/optim_world_size_8_rank_7.pt filter=lfs diff=lfs merge=lfs -text
+code/runs/smoke-training-9b-20260825_175949/orarl/global_step_1/actor/model_world_size_8_rank_0.pt filter=lfs diff=lfs merge=lfs -text
+code/runs/smoke-training-9b-20260825_175949/orarl/global_step_1/actor/model_world_size_8_rank_1.pt filter=lfs diff=lfs merge=lfs -text
+code/runs/smoke-training-9b-20260825_175949/orarl/global_step_1/actor/model_world_size_8_rank_2.pt filter=lfs diff=lfs merge=lfs -text
+code/runs/smoke-training-9b-20260825_175949/orarl/global_step_1/actor/model_world_size_8_rank_3.pt filter=lfs diff=lfs merge=lfs -text
+code/runs/smoke-training-9b-20260825_175949/orarl/global_step_1/actor/model_world_size_8_rank_4.pt filter=lfs diff=lfs merge=lfs -text
+code/runs/smoke-training-9b-20260825_175949/orarl/global_step_1/actor/model_world_size_8_rank_5.pt filter=lfs diff=lfs merge=lfs -text
+code/runs/smoke-training-9b-20260825_175949/orarl/global_step_1/actor/model_world_size_8_rank_6.pt filter=lfs diff=lfs merge=lfs -text
+code/runs/smoke-training-9b-20260825_175949/orarl/global_step_1/actor/model_world_size_8_rank_7.pt filter=lfs diff=lfs merge=lfs -text
+code/runs/smoke-training-9b-20260825_175949/orarl/global_step_1/actor/optim_world_size_8_rank_0.pt filter=lfs diff=lfs merge=lfs -text
+code/runs/smoke-training-9b-20260825_175949/orarl/global_step_1/actor/optim_world_size_8_rank_1.pt filter=lfs diff=lfs merge=lfs -text
+code/runs/smoke-training-9b-20260825_175949/orarl/global_step_1/actor/optim_world_size_8_rank_2.pt filter=lfs diff=lfs merge=lfs -text
+code/runs/smoke-training-9b-20260825_175949/orarl/global_step_1/actor/optim_world_size_8_rank_3.pt filter=lfs diff=lfs merge=lfs -text
+code/runs/smoke-training-9b-20260825_175949/orarl/global_step_1/actor/optim_world_size_8_rank_4.pt filter=lfs diff=lfs merge=lfs -text
+code/runs/smoke-training-9b-20260825_175949/orarl/global_step_1/actor/optim_world_size_8_rank_5.pt filter=lfs diff=lfs merge=lfs -text
+code/runs/smoke-training-9b-20260825_175949/orarl/global_step_1/actor/optim_world_size_8_rank_6.pt filter=lfs diff=lfs merge=lfs -text
+code/runs/smoke-training-9b-20260825_175949/orarl/global_step_1/actor/optim_world_size_8_rank_7.pt filter=lfs diff=lfs merge=lfs -text

code/.gitignore

@@ -0,0 +1,80 @@
+# Python and test state
+__pycache__/
+*.py[cod]
+*.egg-info/
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+.nox/
+.tox/
+.coverage
+htmlcov/
+build/
+dist/
+.venv/
+venv/
+
+# Local inputs and generated runs
+/artifacts/
+/cache/
+/checkpoint/
+/checkpoints/
+/data/
+/local_data/
+/logs/
+/models/
+/outputs/
+/prepared/
+/ray_results/
+/runs/
+/wandb/
+/orarl-eval-summary.json
+nohup.out
+core.*
+
+# Model, media, and generated dataset payloads
+*.arrow
+*.avi
+*.bin
+*.ckpt
+*.jsonl
+*.mkv
+*.mov
+*.mp4
+*.npy
+*.npz
+*.parquet
+*.pt
+*.pth
+*.safetensors
+*.webm
+*.log
+
+# Only the small evaluation profile manifest is release source. Canonical
+# annotations, media, subtitles, and processed artifacts live on Hugging Face.
+!/data/
+/data/*
+!/data/eval/
+/data/eval/*
+!/data/eval/datasets.jsonl
+!/data/eval/README.md
+
+# README media shipped with the repository
+!/assets/orarl-teaser.mp4
+
+# Editor and operating-system state
+.DS_Store
+.idea/
+.ipynb_checkpoints/
+.vscode/
+*~
+
+# Local configuration
+.env
+.env.*
+*.local.yaml
+*.local.yml
+*.key
+*.pem
+credentials.json
+token

code/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2026 The OraRL Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

code/MANIFEST.in

@@ -0,0 +1,20 @@
+include LICENSE
+include NOTICE
+include README.md
+include environment.yml
+include pyproject.toml
+include requirements-cu129.txt
+recursive-include configs *.yaml
+recursive-include data/eval *.jsonl *.md
+recursive-include docs *.md
+recursive-include eval *.md *.py *.sh
+recursive-include scripts *.py *.sh
+recursive-include tests *.py
+recursive-include verl *.py
+prune build
+prune dist
+prune .pytest_cache
+prune .ruff_cache
+prune checkpoint
+prune checkpoints
+global-exclude __pycache__ *.py[cod] .DS_Store

code/NOTICE

@@ -0,0 +1,7 @@
+OraRL
+Copyright 2026 The OraRL Authors
+
+Portions of this software are derived from veRL and other Apache-2.0 upstream
+projects. The derived portions retain their original copyright notices,
+including Copyright 2024 ByteDance Ltd. and/or its affiliates, and are provided
+under the Apache License, Version 2.0.

code/README.md

@@ -0,0 +1,201 @@
+<p align="right"><a href="README_zh.md">简体中文</a></p>
+
+<div align="center">
+
+# OraRL
+
+### Annotations as Rollouts
+
+**Efficient and scalable reinforcement learning for unified video MLLMs**
+
+Yunheng Li · Guohong Mu · Hao Li · Shengsheng Qian · Dingwen Zhang ·
+Qibin Hou · Ming-Ming Cheng
+
+<p>
+  <a href="https://arxiv.org/abs/2608.20492">📄 Paper</a>
+  &nbsp;&nbsp;·&nbsp;&nbsp;
+  <a href="https://orarl.github.io/">🌐 Project Page</a>
+  &nbsp;&nbsp;·&nbsp;&nbsp;
+  <a href="#models">🤗 Models (4B / 9B)</a>
+</p>
+<p>
+  <a href="docs/environment.md">⚙️ Environment</a>
+  &nbsp;&nbsp;·&nbsp;&nbsp;
+  <a href="docs/training.md">🚀 Training</a>
+  &nbsp;&nbsp;·&nbsp;&nbsp;
+  <a href="docs/evaluation.md">📊 Evaluation</a>
+  &nbsp;&nbsp;·&nbsp;&nbsp;
+  <a href="LICENSE">⚖️ License</a>
+</p>
+
+<a href="https://orarl.github.io/assets/orarl-teaser.mp4">
+  <img src="assets/orarl-hero.gif"
+       alt="Animated OraRL method preview" width="92%">
+</a>
+
+**▶ Click the image to watch the 1:38 project overview.**
+
+</div>
+
+## Why OraRL
+
+- **Annotation-as-rollout:** annotations become reliable positive rollouts while
+  policy samples retain an on-policy baseline.
+- **Seven task families:** one update rule covers temporal and spatial grounding,
+  segmentation, tracking, spatial-temporal grounding, video QA, and spatial
+  intelligence.
+- **Efficient training (4B):** sign-balanced pruning delivers **1.48× faster
+  updates** (**92.5 → 62.4 s/step**) while reducing peak per-GPU memory from
+  **62.4 to 50.9 GB**.
+- **Efficient inference:** on one H20 with vLLM in BF16, weight loading occupies
+  **8.6 GiB (4B)** and **17.6 GiB (9B)**. On ten-minute, 2-fps videos,
+  answer-only decoding cuts median post-TTFT latency from **4.78 s to 130 ms**
+  and total latency from **29.03 to 24.30 s**.
+- **Multimodal veRL infrastructure:** a unified video contract carries cached
+  artifacts, raw paths, or inline frame tensors through vLLM rollouts and FSDP
+  updates, with decode-once frame reuse, temporal metadata, task-grouped
+  batching, asynchronous Ray rewards, and safe hybrid-engine cache handling.
+
+## OraRL in One Update
+
+<p align="center">
+  <img src="assets/orarl-method.gif"
+       alt="Animated OraRL framework" width="96%">
+</p>
+
+An OraRL update separates reliable annotation guidance from on-policy
+normalization:
+
+1. **Build the group:** append one serialized annotation rollout to the policy
+   samples generated for the same prompt.
+2. **Keep the baseline on-policy:** estimate the group baseline from policy
+   rewards only.
+3. **Guide and select:** convert the annotation-policy reward gap into a
+   correction, then retain a sign-balanced subset for the update.
+
+This design uses task-native annotations directly and requires no
+chain-of-thought supervision or decoding.
+
+## Video-ORA Results
+
+<p align="center">
+  <img src="assets/paper-results.png"
+       alt="Video-ORA-9B results across seven task families" width="100%">
+</p>
+
+### Dataset-Level Results
+
+<picture>
+  <source media="(prefers-color-scheme: dark)"
+          srcset="assets/video_ora_benchmark_matrix_dark.svg">
+  <source media="(prefers-color-scheme: light)"
+          srcset="assets/video_ora_benchmark_matrix_light.svg">
+  <img src="assets/video_ora_benchmark_matrix_light.svg"
+       alt="Dataset-level benchmark matrix comparing Video-ORA with multimodal baselines"
+       width="100%">
+</picture>
+
+Video-ORA-9B leads the matched seven-family comparison without CoT decoding.
+Best and second-best values are highlighted per row; `†` denotes an
+original-report value whose frame, prompt, split, or decoding settings may
+differ. Averages require complete family coverage.
+
+<!-- <details>
+<summary>Benchmark sources</summary>
+
+Unmarked values come from Tables 1–8 and Appendix Table 20 of the latest
+[OraRL paper](https://arxiv.org/abs/2608.20492). External entries follow the original
+[LLaVA-OneVision-2](https://arxiv.org/abs/2605.25979),
+[VideoChat3](https://github.com/MCG-NJU/VideoChat3), and
+[OneThinker](https://arxiv.org/abs/2512.03043) reports. OneThinker is cited only
+as the source of public Qwen3-VL scores. ReVSI uses each model's reported frame
+setting; the paper's three-benchmark spatial-intelligence average excludes it.
+
+</details> -->
+
+### Model Scaling
+
+<p align="center">
+  <img src="assets/orarl-model-scaling.gif"
+       alt="Animated Video-ORA model scaling from 0.8B to 9B" width="100%">
+</p>
+
+### Data Scaling
+
+<p align="center">
+  <img src="assets/orarl-data-scaling.gif"
+       alt="Animated OraRL data scaling and reward dynamics" width="100%">
+</p>
+
+## Models
+
+| Model | Backbone | Released recipe | Weights |
+| --- | --- | --- | --- |
+| **Video-ORA-9B** | Qwen3.5-9B | `orarl_9b.yaml` | [Hugging Face](https://huggingface.co/OraRL/Video-ORA-9B) |
+| **Video-ORA-4B** | Qwen3.5-4B | `orarl_4b.yaml` | Hugging Face (coming soon) |
+
+### vLLM Serving
+
+Both Video-ORA checkpoints load directly with **vLLM 0.19.1** for
+OpenAI-compatible serving:
+
+```bash
+MODEL=OraRL/Video-ORA-9B
+
+vllm serve "$MODEL" \
+  --served-model-name Video-ORA-9B \
+  --trust-remote-code \
+  --dtype bfloat16 \
+  --tensor-parallel-size 1 \
+  --max-model-len 131072 \
+  --limit-mm-per-prompt '{"image": 1, "video": 1}'
+```
+
+Set `--tensor-parallel-size` to the GPU count for multi-GPU deployment and
+lower `--max-model-len` on smaller-memory devices. Use
+`enable_thinking=false` in the chat template for answer-only inference.
+
+## Use OraRL
+
+The release is organized around three user-facing workflows:
+
+1. **[Environment](docs/environment.md):** install the pinned CUDA stack that
+   covers both the bundled trainer and the evaluators.
+2. **[Training](docs/training.md):** prepare licensed local training data and
+   launch GRPO or OraRL on one or multiple nodes.
+3. **[Evaluation](docs/evaluation.md):** download Video-ORA and OraRL-Data,
+   then run a smoke test or the complete paper suite.
+
+Training and evaluation are dry runs by default; inspect the resolved command
+before adding `--run`. Checkpoints and evaluation media are hosted under the
+[OraRL Hugging Face organization](https://huggingface.co/OraRL).
+
+## Acknowledgements
+
+OraRL is built on [veRL](https://github.com/volcengine/verl) — a
+high-performance RL framework with HybridEngine. We thank its authors and
+contributors for open-sourcing the training infrastructure.
+
+## License
+
+OraRL source is released under [Apache-2.0](LICENSE). Datasets, models,
+benchmarks, and optional dependencies retain their original licenses; see
+[NOTICE](NOTICE).
+
+## Citation
+
+If you find OraRL useful, please consider giving this repository a ⭐ and
+citing our [paper](https://arxiv.org/abs/2608.20492).
+
+```bibtex
+@article{li2026orarl,
+  title   = {Annotations as Rollouts: Efficient and Scalable
+             Reinforcement Learning for Video MLLMs},
+  author  = {Li, Yunheng and Mu, Guohong and Li, Hao and
+             Qian, Shengsheng and Zhang, Dingwen and Hou, Qibin
+             and Cheng, Ming-Ming},
+  journal = {arXiv preprint arXiv:2608.20492},
+  year    = {2026},
+  url     = {https://arxiv.org/abs/2608.20492}
+}
+```

code/README_zh.md

@@ -0,0 +1,193 @@
+<p align="right"><a href="README.md">English</a></p>
+
+<div align="center">
+
+# OraRL
+
+### 将标注作为 Rollout
+
+**面向统一视频多模态大模型的高效、可扩展强化学习**
+
+Yunheng Li · Guohong Mu · Hao Li · Shengsheng Qian · Dingwen Zhang ·
+Qibin Hou · Ming-Ming Cheng
+
+<p>
+  <a href="https://arxiv.org/abs/2608.20492">📄 论文</a>
+  &nbsp;&nbsp;·&nbsp;&nbsp;
+  <a href="https://orarl.github.io/">🌐 项目主页</a>
+  &nbsp;&nbsp;·&nbsp;&nbsp;
+  <a href="#models">🤗 模型（4B / 9B）</a>
+</p>
+<p>
+  <a href="docs/environment_zh.md">⚙️ 环境</a>
+  &nbsp;&nbsp;·&nbsp;&nbsp;
+  <a href="docs/training_zh.md">🚀 训练</a>
+  &nbsp;&nbsp;·&nbsp;&nbsp;
+  <a href="docs/evaluation_zh.md">📊 评测</a>
+  &nbsp;&nbsp;·&nbsp;&nbsp;
+  <a href="LICENSE">⚖️ 许可证</a>
+</p>
+
+<a href="https://orarl.github.io/assets/orarl-teaser.mp4">
+  <img src="assets/orarl-hero.gif"
+       alt="OraRL 方法动态预览" width="92%">
+</a>
+
+**▶ 点击图片观看 1 分 38 秒的项目介绍视频。**
+
+</div>
+
+## 为什么选择 OraRL
+
+- **将标注作为 Rollout：** 将标注转化为可靠的正样本 rollout，同时策略样本
+  仍使用纯 on-policy 基线。
+- **七类任务统一训练：** 同一套更新规则覆盖时序定位、空间定位、分割、跟踪、
+  时空定位、视频问答和空间智能。
+- **高效训练（4B）：** 符号均衡剪枝带来 **1.48× 的更新加速**
+  （**92.5 → 62.4 秒/步**），同时将单卡峰值显存从 **62.4 GB 降至
+  50.9 GB**。
+- **高效推理：** 在单张 H20 上使用 vLLM 和 BF16 时，权重加载占用分别为
+  **8.6 GiB（4B）**和 **17.6 GiB（9B）**。对于 2 fps 采样的十分钟视频，
+  仅生成答案将首 token 后的中位延迟从 **4.78 秒降至 130 毫秒**，并将总
+  中位延迟从 **29.03 秒降至 24.30 秒**。
+- **多模态 veRL 基础设施：** 统一的视频数据契约支持缓存产物、原始路径和内联
+  帧张量，并贯通 vLLM rollout 与 FSDP 更新；同时加入单次解码帧复用、时序
+  元数据、按任务分组的批处理、异步 Ray 奖励和安全的混合引擎缓存处理。
+
+## 一次 OraRL 更新
+
+<p align="center">
+  <img src="assets/orarl-method.gif"
+       alt="OraRL 方法框架动画" width="96%">
+</p>
+
+一次 OraRL 更新将可靠的标注引导与 on-policy 归一化明确分开：
+
+1. **构造样本组：** 在同一 prompt 的策略样本后加入一个序列化标注 rollout。
+2. **保持 on-policy 基线：** 仅使用策略样本的奖励估计组内基线。
+3. **引导并筛选：** 将标注与策略样本的奖励差转化为修正量，再选择符号均衡的
+   子集执行更新。
+
+该设计直接使用任务原生标注，不需要思维链监督或解码。
+
+## Video-ORA 结果
+
+<p align="center">
+  <img src="assets/paper-results.png"
+       alt="Video-ORA-9B 在七类任务上的结果" width="100%">
+</p>
+
+### 数据集级结果
+
+<picture>
+  <source media="(prefers-color-scheme: dark)"
+          srcset="assets/video_ora_benchmark_matrix_dark.svg">
+  <source media="(prefers-color-scheme: light)"
+          srcset="assets/video_ora_benchmark_matrix_light.svg">
+  <img src="assets/video_ora_benchmark_matrix_light.svg"
+       alt="Video-ORA 与多模态基线的数据集级对比矩阵"
+       width="100%">
+</picture>
+
+Video-ORA-9B 在不使用思维链解码的情况下，在统一的七类任务对比中取得领先。
+每行的最佳和次佳结果均已突出显示；`†` 表示该数值来自原始报告，其帧数、提示词、
+数据划分或解码设置可能不同。仅在完整覆盖该任务类别时才计算类别平均值。
+
+<!-- <details>
+<summary>基准来源</summary>
+
+未标注的数值来自最新版 [OraRL 论文](https://arxiv.org/abs/2608.20492)的表 1–8 和附录表 20。
+外部数值来自
+[LLaVA-OneVision-2](https://arxiv.org/abs/2605.25979)、
+[VideoChat3](https://github.com/MCG-NJU/VideoChat3) 和
+[OneThinker](https://arxiv.org/abs/2512.03043) 的原始报告。
+此处仅将 OneThinker 作为公开 Qwen3-VL 分数的来源。ReVSI 使用各模型报告的
+帧设置；论文中的三基准空间智能平均值不包含 ReVSI。
+
+</details> -->
+
+### 模型规模扩展
+
+<p align="center">
+  <img src="assets/orarl-model-scaling.gif"
+       alt="Video-ORA 从 0.8B 到 9B 的模型规模扩展动画" width="100%">
+</p>
+
+### 数据规模扩展
+
+<p align="center">
+  <img src="assets/orarl-data-scaling.gif"
+       alt="OraRL 数据规模扩展与奖励变化动画" width="100%">
+</p>
+
+<a id="models"></a>
+
+## 模型
+
+| 模型 | 基座模型 | 发布配置 | 权重 |
+| --- | --- | --- | --- |
+| **Video-ORA-9B** | Qwen3.5-9B | `orarl_9b.yaml` | [Hugging Face](https://huggingface.co/OraRL/Video-ORA-9B) |
+| **Video-ORA-4B** | Qwen3.5-4B | `orarl_4b.yaml` | Hugging Face（即将发布） |
+
+### vLLM 部署
+
+两个 Video-ORA 检查点均可通过 **vLLM 0.19.1** 直接提供��容 OpenAI 的
+推理服务：
+
+```bash
+MODEL=OraRL/Video-ORA-9B
+
+vllm serve "$MODEL" \
+  --served-model-name Video-ORA-9B \
+  --trust-remote-code \
+  --dtype bfloat16 \
+  --tensor-parallel-size 1 \
+  --max-model-len 131072 \
+  --limit-mm-per-prompt '{"image": 1, "video": 1}'
+```
+
+多卡部署时，将 `--tensor-parallel-size` 设为 GPU 数量；显存较小时可降低
+`--max-model-len`。如需仅生成答案，请在聊天模板中设置
+`enable_thinking=false`。
+
+## 使用 OraRL
+
+本仓库围绕三个面向用户的工作流组织：
+
+1. **[环境](docs/environment_zh.md)：** 安装同时覆盖自带训练运行时与评测运行时的
+   固定 CUDA 软件栈。
+2. **[训练](docs/training_zh.md)：** 准备已获许可的本地训练数据，并启动单节点或
+   多节点 GRPO/OraRL。
+3. **[评测](docs/evaluation_zh.md)：** 下载 Video-ORA 和 OraRL-Data，运行
+   smoke test 或完整论文评测。
+
+训练和评测默认仅执行 dry run；检查解析后的命令后再添加 `--run`。Checkpoint
+和评测媒体托管在 [OraRL Hugging Face 组织](https://huggingface.co/OraRL)下。
+
+## 致谢
+
+OraRL 基于 [veRL](https://github.com/volcengine/verl) 构建——这是一个采用
+HybridEngine 的高性能强化学习框架。感谢其作者和贡献者开源训练基础设施。
+
+## 许可证
+
+OraRL 源代码采用 [Apache-2.0](LICENSE) 许可证发布。数据集、模型、基准和可选
+依赖仍遵循各自的原始许可证，详情请参阅 [NOTICE](NOTICE)。
+
+## 引用
+
+如果 OraRL 对您有所帮助，欢迎为本仓库点亮 ⭐，并引用我们的
+[论文](https://arxiv.org/abs/2608.20492)。
+
+```bibtex
+@article{li2026orarl,
+  title   = {Annotations as Rollouts: Efficient and Scalable
+             Reinforcement Learning for Video MLLMs},
+  author  = {Li, Yunheng and Mu, Guohong and Li, Hao and
+             Qian, Shengsheng and Zhang, Dingwen and Hou, Qibin
+             and Cheng, Ming-Ming},
+  journal = {arXiv preprint arXiv:2608.20492},
+  year    = {2026},
+  url     = {https://arxiv.org/abs/2608.20492}
+}
+```

code/assets/orarl-teaser.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7915acbb2207376ef20467792c3e5b2b86b7a7a292d7f10c1bf3f44817c28e0b
+size 5901910

code/configs/data_sources.example.yaml

@@ -0,0 +1,90 @@
+# Copy this file, then replace every ../local_data placeholder.
+#
+# OraRL never downloads data or media. You must obtain each dataset under its
+# license, keep the required attribution, and point input/media_root at your
+# own local, licensed copy. The optional license and url values are audit
+# metadata only and are never opened by the builder.
+#
+# The seven quotas reproduce the final paper mixture: 100,032 train prompts
+# (1,563 batches of 64), with 62,656 structured prompts and 37,376 answer-only
+# prompts. The canary is
+# additional held-out data and is not subtracted from the train target.
+version: 1
+seed: 42
+target: 100032
+canary_size: 512
+max_prompts_per_media: 2
+require_media: true
+allow_shortfall: false
+
+# Supply JSON/JSONL benchmark records before a release build. Any candidate
+# sharing a normalized prompt identity or media identity is removed.
+benchmark_excludes:
+  - ../local_data/exclusions/public_benchmarks.jsonl
+
+sources:
+  - name: temporal_grounding_train
+    input: ../local_data/annotations/temporal_grounding.jsonl
+    task: temporal grounding
+    family: temporal
+    quota: 20096
+    media_root: ../local_data/media/temporal
+    license: REPLACE_WITH_DATASET_LICENSE
+    url: REPLACE_WITH_DATASET_HOME_PAGE
+
+  - name: tracking_train
+    input: ../local_data/annotations/tracking.jsonl
+    task: tracking
+    family: tracking
+    quota: 13952
+    media_root: ../local_data/media/tracking
+    license: REPLACE_WITH_DATASET_LICENSE
+    url: REPLACE_WITH_DATASET_HOME_PAGE
+
+  - name: segmentation_train
+    input: ../local_data/annotations/segmentation.jsonl
+    task: segmentation
+    family: segmentation
+    quota: 12032
+    media_root: ../local_data/media/segmentation
+    license: REPLACE_WITH_DATASET_LICENSE
+    url: REPLACE_WITH_DATASET_HOME_PAGE
+
+  - name: spatial_grounding_train
+    input: ../local_data/annotations/spatial_grounding.jsonl
+    task: spatial grounding
+    family: spatial
+    quota: 7040
+    media_root: ../local_data/media/spatial
+    license: REPLACE_WITH_DATASET_LICENSE
+    url: REPLACE_WITH_DATASET_HOME_PAGE
+
+  - name: spatial_temporal_grounding_train
+    input: ../local_data/annotations/spatial_temporal_grounding.jsonl
+    task: spatial-temporal grounding
+    family: stvg
+    quota: 9536
+    media_root: ../local_data/media/spatial_temporal
+    license: REPLACE_WITH_DATASET_LICENSE
+    url: REPLACE_WITH_DATASET_HOME_PAGE
+
+  - name: video_qa_train
+    input: ../local_data/annotations/video_qa.jsonl
+    task: video_qa_mc
+    family: video_qa
+    quota: 20288
+    media_root: ../local_data/media/video_qa
+    license: REPLACE_WITH_DATASET_LICENSE
+    url: REPLACE_WITH_DATASET_HOME_PAGE
+
+  - name: spatial_intelligence_train
+    input: ../local_data/annotations/spatial_intelligence.jsonl
+    task: spatial intelligence
+    family: spatial_intelligence
+    # Keep subtype labels such as object_counting and object_rel_direction;
+    # the reward adapter uses them to select the official scoring rule.
+    preserve_problem_type: true
+    quota: 17088
+    media_root: ../local_data/media/spatial_intelligence
+    license: REPLACE_WITH_DATASET_LICENSE
+    url: REPLACE_WITH_DATASET_HOME_PAGE

code/configs/grpo_4b.yaml

@@ -0,0 +1,139 @@
+# Complete 4B GRPO baseline. Required paths are supplied by the CLI or environment.
+data:
+  train_files: ${oc.env:ORARL_TRAIN_DATA}
+  val_files: ${oc.env:ORARL_VAL_DATA}
+  prompt_key: problem
+  answer_key: answer
+  image_key: images
+  video_key: videos
+  image_dir: ${oc.env:ORARL_MEDIA_ROOT,null}
+  dataloader_num_workers: ${oc.env:ORARL_DATALOADER_WORKERS,4}
+  video_max_pixels: ${oc.env:ORARL_VIDEO_MAX_PIXELS,786432}
+  video_min_pixels: ${oc.env:ORARL_VIDEO_MIN_PIXELS,4096}
+  video_max_frames: ${oc.env:ORARL_VIDEO_MAX_FRAMES,128}
+  video_fps: ${oc.env:ORARL_VIDEO_FPS,2}
+  video_total_pixels: ${oc.env:ORARL_VIDEO_TOTAL_PIXELS,8388608}
+  min_pixels: ${oc.env:ORARL_IMAGE_MIN_PIXELS,4096}
+  image_max_pixels: ${oc.env:ORARL_IMAGE_MAX_PIXELS,1048576}
+  use_preprocessed_videos: false
+  video_source_mode: ${oc.env:ORARL_VIDEO_SOURCE_MODE,realtime_only}
+  preprocessed_video_dir: ${oc.env:ORARL_PREPROCESSED_DIR,null}
+  inline_video_tensors: ${oc.env:ORARL_INLINE_VIDEO_TENSORS,false}
+  max_prompt_length: ${oc.env:ORARL_MAX_PROMPT_LENGTH,16384}
+  max_response_length: ${oc.env:ORARL_MAX_RESPONSE_LENGTH,4096}
+  rollout_batch_size: ${oc.env:ORARL_ROLLOUT_BATCH_SIZE,64}
+  mini_rollout_batch_size: null
+  val_batch_size: ${oc.env:ORARL_VAL_BATCH_SIZE,16}
+  format_prompt: null
+  override_chat_template: null
+  enable_thinking: ${oc.env:ORARL_ENABLE_THINKING,false}
+  response_prefix: ""
+  shuffle: true
+  seed: ${oc.env:ORARL_SEED,42}
+  group_by_task: true
+  group_by_task_key: problem_type
+  filter_overlong_prompts: false
+
+algorithm:
+  name: grpo
+  adv_estimator: grpo
+  scale_rewards: true
+  disable_kl: true
+  use_kl_loss: false
+  kl_penalty: low_var_kl
+  kl_coef: 0.0
+
+worker:
+  actor:
+    global_batch_size: ${oc.env:ORARL_GLOBAL_BATCH_SIZE,64}
+    micro_batch_size_per_device_for_update: ${oc.env:ORARL_UPDATE_MICRO_BATCH,2}
+    micro_batch_size_per_device_for_experience: ${oc.env:ORARL_EXPERIENCE_MICRO_BATCH,1}
+    ppo_epochs: 1
+    max_grad_norm: 1.0
+    clip_ratio_low: 0.2
+    clip_ratio_high: 0.2
+    loss_avg_mode: token
+    padding_free: true
+    dynamic_batching: true
+    max_token_len_per_gpu: ${oc.env:ORARL_MAX_TOKEN_LEN_PER_GPU,24576}
+    ulysses_size: 1
+    use_torch_compile: false
+    model:
+      model_path: ${oc.env:ORARL_MODEL_PATH}
+      tokenizer_path: ${oc.env:ORARL_TOKENIZER_PATH,${oc.env:ORARL_MODEL_PATH}}
+      enable_gradient_checkpointing: true
+      trust_remote_code: false
+      freeze_vision_tower: true
+      train_vision_merger: false
+    optim:
+      lr: ${oc.env:ORARL_LEARNING_RATE,2.0e-6}
+      weight_decay: 0.0
+      strategy: adamw
+      lr_warmup_ratio: 0.0
+      lr_scheduler_type: constant
+    fsdp:
+      torch_dtype: bf16
+      enable_full_shard: true
+      enable_cpu_offload: false
+      enable_rank0_init: true
+      use_orig_params: false
+    offload:
+      offload_params: false
+      offload_optimizer: false
+
+  rollout:
+    name: ${oc.env:ORARL_ROLLOUT_BACKEND,vllm}
+    n: 8
+    seed: ${oc.env:ORARL_SEED,42}
+    temperature: 1.0
+    top_p: 0.85
+    limit_images: 10
+    gpu_memory_utilization: 0.5
+    enforce_eager: false
+    enable_chunked_prefill: true
+    tensor_parallel_size: 1
+    max_num_batched_tokens: 32768
+    disable_tqdm: true
+    calculate_log_probs: false
+    collect_seq_logprob_for_filter: false
+    val_override_config:
+      temperature: 0.7
+      top_p: 0.95
+      n: 1
+
+  ref:
+    fsdp:
+      torch_dtype: bf16
+      enable_full_shard: true
+      enable_cpu_offload: false
+      enable_rank0_init: true
+    offload:
+      offload_params: false
+
+  reward:
+    reward_function: orarl.rewards:compute_score
+    reward_function_kwargs: {}
+
+trainer:
+  total_epochs: 1
+  max_steps: null
+  project_name: OraRL
+  experiment_name: grpo-4b
+  logger: ["console"]
+  nnodes: 1
+  n_gpus_per_node: 8
+  val_freq: -1
+  val_before_train: false
+  val_only: false
+  val_generations_to_log: 0
+  save_freq: 50
+  save_limit: 3
+  save_model_only: false
+  keep_optim_only_latest: true
+  save_checkpoint_path: ${oc.env:ORARL_OUTPUT_DIR}
+  load_checkpoint_path: null
+  find_last_checkpoint: false
+  keep_best_train_ckpt: false
+  best_train_metric_key: reward/overall
+  best_train_smooth_window: 5
+  best_train_min_step: 10

code/configs/grpo_9b.yaml

@@ -0,0 +1,139 @@
+# Complete 9B GRPO baseline. Required paths are supplied by the CLI or environment.
+data:
+  train_files: ${oc.env:ORARL_TRAIN_DATA}
+  val_files: ${oc.env:ORARL_VAL_DATA}
+  prompt_key: problem
+  answer_key: answer
+  image_key: images
+  video_key: videos
+  image_dir: ${oc.env:ORARL_MEDIA_ROOT,null}
+  dataloader_num_workers: ${oc.env:ORARL_DATALOADER_WORKERS,4}
+  video_max_pixels: ${oc.env:ORARL_VIDEO_MAX_PIXELS,786432}
+  video_min_pixels: ${oc.env:ORARL_VIDEO_MIN_PIXELS,4096}
+  video_max_frames: ${oc.env:ORARL_VIDEO_MAX_FRAMES,128}
+  video_fps: ${oc.env:ORARL_VIDEO_FPS,2}
+  video_total_pixels: ${oc.env:ORARL_VIDEO_TOTAL_PIXELS,8388608}
+  min_pixels: ${oc.env:ORARL_IMAGE_MIN_PIXELS,4096}
+  image_max_pixels: ${oc.env:ORARL_IMAGE_MAX_PIXELS,1048576}
+  use_preprocessed_videos: false
+  video_source_mode: ${oc.env:ORARL_VIDEO_SOURCE_MODE,realtime_only}
+  preprocessed_video_dir: ${oc.env:ORARL_PREPROCESSED_DIR,null}
+  inline_video_tensors: ${oc.env:ORARL_INLINE_VIDEO_TENSORS,false}
+  max_prompt_length: ${oc.env:ORARL_MAX_PROMPT_LENGTH,24576}
+  max_response_length: ${oc.env:ORARL_MAX_RESPONSE_LENGTH,4096}
+  rollout_batch_size: ${oc.env:ORARL_ROLLOUT_BATCH_SIZE,64}
+  mini_rollout_batch_size: null
+  val_batch_size: ${oc.env:ORARL_VAL_BATCH_SIZE,16}
+  format_prompt: null
+  override_chat_template: null
+  enable_thinking: ${oc.env:ORARL_ENABLE_THINKING,false}
+  response_prefix: ""
+  shuffle: true
+  seed: ${oc.env:ORARL_SEED,42}
+  group_by_task: true
+  group_by_task_key: problem_type
+  filter_overlong_prompts: false
+
+algorithm:
+  name: grpo
+  adv_estimator: grpo
+  scale_rewards: true
+  disable_kl: true
+  use_kl_loss: false
+  kl_penalty: low_var_kl
+  kl_coef: 0.0
+
+worker:
+  actor:
+    global_batch_size: ${oc.env:ORARL_GLOBAL_BATCH_SIZE,64}
+    micro_batch_size_per_device_for_update: ${oc.env:ORARL_UPDATE_MICRO_BATCH,1}
+    micro_batch_size_per_device_for_experience: ${oc.env:ORARL_EXPERIENCE_MICRO_BATCH,1}
+    ppo_epochs: 1
+    max_grad_norm: 1.0
+    clip_ratio_low: 0.2
+    clip_ratio_high: 0.2
+    loss_avg_mode: token
+    padding_free: true
+    dynamic_batching: true
+    max_token_len_per_gpu: ${oc.env:ORARL_MAX_TOKEN_LEN_PER_GPU,32768}
+    ulysses_size: 1
+    use_torch_compile: false
+    model:
+      model_path: ${oc.env:ORARL_MODEL_PATH}
+      tokenizer_path: ${oc.env:ORARL_TOKENIZER_PATH,${oc.env:ORARL_MODEL_PATH}}
+      enable_gradient_checkpointing: true
+      trust_remote_code: false
+      freeze_vision_tower: true
+      train_vision_merger: false
+    optim:
+      lr: ${oc.env:ORARL_LEARNING_RATE,1.0e-6}
+      weight_decay: 0.0
+      strategy: adamw
+      lr_warmup_ratio: 0.0
+      lr_scheduler_type: constant
+    fsdp:
+      torch_dtype: bf16
+      enable_full_shard: true
+      enable_cpu_offload: false
+      enable_rank0_init: true
+      use_orig_params: false
+    offload:
+      offload_params: false
+      offload_optimizer: false
+
+  rollout:
+    name: ${oc.env:ORARL_ROLLOUT_BACKEND,vllm}
+    n: 8
+    seed: ${oc.env:ORARL_SEED,42}
+    temperature: 1.0
+    top_p: 0.85
+    limit_images: 10
+    gpu_memory_utilization: 0.45
+    enforce_eager: false
+    enable_chunked_prefill: true
+    tensor_parallel_size: 1
+    max_num_batched_tokens: 32768
+    disable_tqdm: true
+    calculate_log_probs: false
+    collect_seq_logprob_for_filter: false
+    val_override_config:
+      temperature: 0.7
+      top_p: 0.95
+      n: 1
+
+  ref:
+    fsdp:
+      torch_dtype: bf16
+      enable_full_shard: true
+      enable_cpu_offload: false
+      enable_rank0_init: true
+    offload:
+      offload_params: false
+
+  reward:
+    reward_function: orarl.rewards:compute_score
+    reward_function_kwargs: {}
+
+trainer:
+  total_epochs: 1
+  max_steps: null
+  project_name: OraRL
+  experiment_name: grpo-9b
+  logger: ["console"]
+  nnodes: 1
+  n_gpus_per_node: 8
+  val_freq: -1
+  val_before_train: false
+  val_only: false
+  val_generations_to_log: 0
+  save_freq: 50
+  save_limit: 3
+  save_model_only: false
+  keep_optim_only_latest: true
+  save_checkpoint_path: ${oc.env:ORARL_OUTPUT_DIR}
+  load_checkpoint_path: null
+  find_last_checkpoint: false
+  keep_best_train_ckpt: false
+  best_train_metric_key: reward/overall
+  best_train_smooth_window: 5
+  best_train_min_step: 10

code/configs/orarl_4b.yaml

@@ -0,0 +1,162 @@
+# Complete 4B OraRL recipe. Required paths are supplied by the CLI or environment.
+data:
+  train_files: ${oc.env:ORARL_TRAIN_DATA}
+  val_files: ${oc.env:ORARL_VAL_DATA}
+  prompt_key: problem
+  answer_key: answer
+  image_key: images
+  video_key: videos
+  image_dir: ${oc.env:ORARL_MEDIA_ROOT,null}
+  dataloader_num_workers: ${oc.env:ORARL_DATALOADER_WORKERS,4}
+  video_max_pixels: ${oc.env:ORARL_VIDEO_MAX_PIXELS,786432}
+  video_min_pixels: ${oc.env:ORARL_VIDEO_MIN_PIXELS,4096}
+  video_max_frames: ${oc.env:ORARL_VIDEO_MAX_FRAMES,128}
+  video_fps: ${oc.env:ORARL_VIDEO_FPS,2}
+  video_total_pixels: ${oc.env:ORARL_VIDEO_TOTAL_PIXELS,8388608}
+  min_pixels: ${oc.env:ORARL_IMAGE_MIN_PIXELS,4096}
+  image_max_pixels: ${oc.env:ORARL_IMAGE_MAX_PIXELS,1048576}
+  use_preprocessed_videos: false
+  video_source_mode: ${oc.env:ORARL_VIDEO_SOURCE_MODE,realtime_only}
+  preprocessed_video_dir: ${oc.env:ORARL_PREPROCESSED_DIR,null}
+  inline_video_tensors: ${oc.env:ORARL_INLINE_VIDEO_TENSORS,false}
+  max_prompt_length: ${oc.env:ORARL_MAX_PROMPT_LENGTH,16384}
+  max_response_length: ${oc.env:ORARL_MAX_RESPONSE_LENGTH,4096}
+  rollout_batch_size: ${oc.env:ORARL_ROLLOUT_BATCH_SIZE,64}
+  mini_rollout_batch_size: null
+  val_batch_size: ${oc.env:ORARL_VAL_BATCH_SIZE,16}
+  format_prompt: null
+  override_chat_template: null
+  enable_thinking: ${oc.env:ORARL_ENABLE_THINKING,false}
+  response_prefix: ""
+  shuffle: true
+  seed: ${oc.env:ORARL_SEED,42}
+  group_by_task: true
+  group_by_task_key: problem_type
+  filter_overlong_prompts: false
+
+algorithm:
+  name: orarl
+  adv_estimator: grpo
+  scale_rewards: false
+  oracle_injection: true
+  oracle_injection_mode: append
+  oracle_builder: orarl.rewards:build_oracle_response_from_ground_truth
+  oracle_log_exclude: true
+  directional_gain: true
+  directional_gain_gamma: 0.25
+  directional_gain_positive_only: true
+  directional_gain_recenter: true
+  detached_oracle_advantage: true
+  detached_oracle_advantage_scale: 2.0
+  detached_oracle_use_directional_gain: false
+  detached_oracle_match_best_ratio: 1.2
+  detached_oracle_match_best_min: 0.05
+  detached_oracle_match_best_max: 1.0
+  oracle_reward_gate_beta: 2.0
+  selection_prune_ratio: 0.5
+  selection_keep_oracle: true
+  selection_positive_quota: 1
+  selection_negative_quota: 2
+  selection_strict_sign_balance: true
+  post_selection_recenter: true
+  post_selection_rms_match: true
+  post_selection_rms_min_scale: 0.25
+  disable_kl: true
+  use_kl_loss: false
+  kl_penalty: low_var_kl
+  kl_coef: 0.0
+
+worker:
+  actor:
+    global_batch_size: ${oc.env:ORARL_GLOBAL_BATCH_SIZE,64}
+    micro_batch_size_per_device_for_update: ${oc.env:ORARL_UPDATE_MICRO_BATCH,2}
+    micro_batch_size_per_device_for_experience: ${oc.env:ORARL_EXPERIENCE_MICRO_BATCH,1}
+    ppo_epochs: 1
+    max_grad_norm: 1.0
+    clip_ratio_low: 0.2
+    clip_ratio_high: 0.2
+    loss_avg_mode: token
+    padding_free: true
+    dynamic_batching: true
+    max_token_len_per_gpu: ${oc.env:ORARL_MAX_TOKEN_LEN_PER_GPU,24576}
+    ulysses_size: 1
+    use_torch_compile: false
+    model:
+      model_path: ${oc.env:ORARL_MODEL_PATH}
+      tokenizer_path: ${oc.env:ORARL_TOKENIZER_PATH,${oc.env:ORARL_MODEL_PATH}}
+      enable_gradient_checkpointing: true
+      trust_remote_code: false
+      freeze_vision_tower: true
+      train_vision_merger: false
+    optim:
+      lr: ${oc.env:ORARL_LEARNING_RATE,2.0e-6}
+      weight_decay: 0.0
+      strategy: adamw
+      lr_warmup_ratio: 0.0
+      lr_scheduler_type: constant
+    fsdp:
+      torch_dtype: bf16
+      enable_full_shard: true
+      enable_cpu_offload: false
+      enable_rank0_init: true
+      use_orig_params: false
+    offload:
+      offload_params: false
+      offload_optimizer: false
+
+  rollout:
+    name: ${oc.env:ORARL_ROLLOUT_BACKEND,vllm}
+    n: 8
+    seed: ${oc.env:ORARL_SEED,42}
+    temperature: 1.0
+    top_p: 0.85
+    limit_images: 10
+    gpu_memory_utilization: 0.5
+    enforce_eager: false
+    enable_chunked_prefill: true
+    tensor_parallel_size: 1
+    max_num_batched_tokens: 32768
+    disable_tqdm: true
+    calculate_log_probs: false
+    collect_seq_logprob_for_filter: false
+    val_override_config:
+      temperature: 0.7
+      top_p: 0.95
+      n: 1
+
+  ref:
+    fsdp:
+      torch_dtype: bf16
+      enable_full_shard: true
+      enable_cpu_offload: false
+      enable_rank0_init: true
+    offload:
+      offload_params: false
+
+  reward:
+    reward_function: orarl.rewards:compute_score
+    reward_function_kwargs: {}
+
+trainer:
+  total_epochs: 1
+  max_steps: null
+  project_name: OraRL
+  experiment_name: orarl-4b
+  logger: ["console"]
+  nnodes: 1
+  n_gpus_per_node: 8
+  val_freq: -1
+  val_before_train: false
+  val_only: false
+  val_generations_to_log: 0
+  save_freq: 50
+  save_limit: 3
+  save_model_only: false
+  keep_optim_only_latest: true
+  save_checkpoint_path: ${oc.env:ORARL_OUTPUT_DIR}
+  load_checkpoint_path: null
+  find_last_checkpoint: false
+  keep_best_train_ckpt: false
+  best_train_metric_key: reward/overall
+  best_train_smooth_window: 5
+  best_train_min_step: 10

code/configs/orarl_9b.yaml

@@ -0,0 +1,162 @@
+# Complete 9B OraRL recipe. Required paths are supplied by the CLI or environment.
+data:
+  train_files: ${oc.env:ORARL_TRAIN_DATA}
+  val_files: ${oc.env:ORARL_VAL_DATA}
+  prompt_key: problem
+  answer_key: answer
+  image_key: images
+  video_key: videos
+  image_dir: ${oc.env:ORARL_MEDIA_ROOT,null}
+  dataloader_num_workers: ${oc.env:ORARL_DATALOADER_WORKERS,4}
+  video_max_pixels: ${oc.env:ORARL_VIDEO_MAX_PIXELS,786432}
+  video_min_pixels: ${oc.env:ORARL_VIDEO_MIN_PIXELS,4096}
+  video_max_frames: ${oc.env:ORARL_VIDEO_MAX_FRAMES,128}
+  video_fps: ${oc.env:ORARL_VIDEO_FPS,2}
+  video_total_pixels: ${oc.env:ORARL_VIDEO_TOTAL_PIXELS,8388608}
+  min_pixels: ${oc.env:ORARL_IMAGE_MIN_PIXELS,4096}
+  image_max_pixels: ${oc.env:ORARL_IMAGE_MAX_PIXELS,1048576}
+  use_preprocessed_videos: false
+  video_source_mode: ${oc.env:ORARL_VIDEO_SOURCE_MODE,realtime_only}
+  preprocessed_video_dir: ${oc.env:ORARL_PREPROCESSED_DIR,null}
+  inline_video_tensors: ${oc.env:ORARL_INLINE_VIDEO_TENSORS,false}
+  max_prompt_length: ${oc.env:ORARL_MAX_PROMPT_LENGTH,24576}
+  max_response_length: ${oc.env:ORARL_MAX_RESPONSE_LENGTH,4096}
+  rollout_batch_size: ${oc.env:ORARL_ROLLOUT_BATCH_SIZE,64}
+  mini_rollout_batch_size: null
+  val_batch_size: ${oc.env:ORARL_VAL_BATCH_SIZE,16}
+  format_prompt: null
+  override_chat_template: null
+  enable_thinking: ${oc.env:ORARL_ENABLE_THINKING,false}
+  response_prefix: ""
+  shuffle: true
+  seed: ${oc.env:ORARL_SEED,42}
+  group_by_task: true
+  group_by_task_key: problem_type
+  filter_overlong_prompts: false
+
+algorithm:
+  name: orarl
+  adv_estimator: grpo
+  scale_rewards: false
+  oracle_injection: true
+  oracle_injection_mode: append
+  oracle_builder: orarl.rewards:build_oracle_response_from_ground_truth
+  oracle_log_exclude: true
+  directional_gain: true
+  directional_gain_gamma: 0.25
+  directional_gain_positive_only: true
+  directional_gain_recenter: true
+  detached_oracle_advantage: true
+  detached_oracle_advantage_scale: 2.0
+  detached_oracle_use_directional_gain: false
+  detached_oracle_match_best_ratio: 1.2
+  detached_oracle_match_best_min: 0.05
+  detached_oracle_match_best_max: 1.0
+  oracle_reward_gate_beta: 2.0
+  selection_prune_ratio: 0.5
+  selection_keep_oracle: true
+  selection_positive_quota: 1
+  selection_negative_quota: 2
+  selection_strict_sign_balance: true
+  post_selection_recenter: true
+  post_selection_rms_match: true
+  post_selection_rms_min_scale: 0.25
+  disable_kl: true
+  use_kl_loss: false
+  kl_penalty: low_var_kl
+  kl_coef: 0.0
+
+worker:
+  actor:
+    global_batch_size: ${oc.env:ORARL_GLOBAL_BATCH_SIZE,64}
+    micro_batch_size_per_device_for_update: ${oc.env:ORARL_UPDATE_MICRO_BATCH,1}
+    micro_batch_size_per_device_for_experience: ${oc.env:ORARL_EXPERIENCE_MICRO_BATCH,1}
+    ppo_epochs: 1
+    max_grad_norm: 1.0
+    clip_ratio_low: 0.2
+    clip_ratio_high: 0.2
+    loss_avg_mode: token
+    padding_free: true
+    dynamic_batching: true
+    max_token_len_per_gpu: ${oc.env:ORARL_MAX_TOKEN_LEN_PER_GPU,32768}
+    ulysses_size: 1
+    use_torch_compile: false
+    model:
+      model_path: ${oc.env:ORARL_MODEL_PATH}
+      tokenizer_path: ${oc.env:ORARL_TOKENIZER_PATH,${oc.env:ORARL_MODEL_PATH}}
+      enable_gradient_checkpointing: true
+      trust_remote_code: false
+      freeze_vision_tower: true
+      train_vision_merger: false
+    optim:
+      lr: ${oc.env:ORARL_LEARNING_RATE,1.0e-6}
+      weight_decay: 0.0
+      strategy: adamw
+      lr_warmup_ratio: 0.0
+      lr_scheduler_type: constant
+    fsdp:
+      torch_dtype: bf16
+      enable_full_shard: true
+      enable_cpu_offload: false
+      enable_rank0_init: true
+      use_orig_params: false
+    offload:
+      offload_params: false
+      offload_optimizer: false
+
+  rollout:
+    name: ${oc.env:ORARL_ROLLOUT_BACKEND,vllm}
+    n: 8
+    seed: ${oc.env:ORARL_SEED,42}
+    temperature: 1.0
+    top_p: 0.85
+    limit_images: 10
+    gpu_memory_utilization: 0.45
+    enforce_eager: false
+    enable_chunked_prefill: true
+    tensor_parallel_size: 1
+    max_num_batched_tokens: 32768
+    disable_tqdm: true
+    calculate_log_probs: false
+    collect_seq_logprob_for_filter: false
+    val_override_config:
+      temperature: 0.7
+      top_p: 0.95
+      n: 1
+
+  ref:
+    fsdp:
+      torch_dtype: bf16
+      enable_full_shard: true
+      enable_cpu_offload: false
+      enable_rank0_init: true
+    offload:
+      offload_params: false
+
+  reward:
+    reward_function: orarl.rewards:compute_score
+    reward_function_kwargs: {}
+
+trainer:
+  total_epochs: 1
+  max_steps: null
+  project_name: OraRL
+  experiment_name: orarl-9b
+  logger: ["console"]
+  nnodes: 1
+  n_gpus_per_node: 8
+  val_freq: -1
+  val_before_train: false
+  val_only: false
+  val_generations_to_log: 0
+  save_freq: 50
+  save_limit: 3
+  save_model_only: false
+  keep_optim_only_latest: true
+  save_checkpoint_path: ${oc.env:ORARL_OUTPUT_DIR}
+  load_checkpoint_path: null
+  find_last_checkpoint: false
+  keep_best_train_ckpt: false
+  best_train_metric_key: reward/overall
+  best_train_smooth_window: 5
+  best_train_min_step: 10

code/data/eval/README.md

@@ -0,0 +1,25 @@
+# Evaluation profiles
+
+This directory contains only the small, reviewable `datasets.jsonl` manifest
+that pins the paper-suite splits, prompts, preprocessing settings, parsers, and
+metrics.
+
+Evaluation annotations and raw media are hosted separately at
+[`OraRL/OraRL-Data`](https://huggingface.co/datasets/OraRL/OraRL-Data).
+Model checkpoints are hosted under the
+[`OraRL`](https://huggingface.co/OraRL) organization. They are intentionally not
+duplicated in this Git repository.
+
+Use the Hugging Face dataset ID or a complete local snapshot when evaluating:
+
+```bash
+orarl-eval \
+  --model /path/to/Video-ORA-9B \
+  --tasks paper \
+  --dataset OraRL/OraRL-Data \
+  --summary ./evaluation.json \
+  --run
+```
+
+The manifest in this directory is release metadata, not a standalone dataset
+root; it does not include `assets.jsonl` or media payloads.

code/data/eval/datasets.jsonl

@@ -0,0 +1,29 @@
+{"annotation_path":"annotations/video_qa/longvideobench/val.jsonl","artifact_paths":[],"benchmark":"longvideobench","evaluation":{"metric_profile":"accuracy","parser_profile":"multiple_choice","prompt_profile":"longvideobench_default"},"expected_count":1337,"family":"video_qa","legacy_environment":{"LONGVIDEOBENCH_BATCH_SIZE":1,"LONGVIDEOBENCH_ENABLE_THINKING":false,"LONGVIDEOBENCH_FPS":2,"LONGVIDEOBENCH_GPU_MEMORY_UTILIZATION":0.9,"LONGVIDEOBENCH_MAX_FRAMES":384,"LONGVIDEOBENCH_MAX_MODEL_LEN":65536,"LONGVIDEOBENCH_MAX_NEW_TOKENS":128,"LONGVIDEOBENCH_MAX_NUM_BATCHED_TOKENS":65536,"LONGVIDEOBENCH_MIN_P":0.0,"LONGVIDEOBENCH_PRESENCE_PENALTY":0.0,"LONGVIDEOBENCH_PROMPT_MODE":"default","LONGVIDEOBENCH_TEMPERATURE":0.0,"LONGVIDEOBENCH_TOP_K":-1,"LONGVIDEOBENCH_TOP_P":1.0,"LONGVIDEOBENCH_VIDEO_MAX_PIXELS":262144,"LONGVIDEOBENCH_VIDEO_MIN_PIXELS":4096,"LONGVIDEOBENCH_VIDEO_TOTAL_PIXELS":0},"license":"See the upstream benchmark terms","media_paths":["media/video_qa/longvideobench/subtitles","media/video_qa/longvideobench/subtitles/val","media/video_qa/longvideobench/videos","media/video_qa/longvideobench/videos/val"],"preprocessing":{"batch_size":1,"enable_thinking":false,"fps":2,"gpu_memory_utilization":0.9,"max_frames":384,"max_model_len":65536,"max_new_tokens":128,"max_num_batched_tokens":65536,"min_p":0.0,"presence_penalty":0.0,"prompt_mode":"default","temperature":0.0,"top_k":-1,"top_p":1.0,"video_max_pixels":262144,"video_min_pixels":4096,"video_total_pixels":0},"redistribution_authorized":true,"schema_version":1,"source_url":"https://github.com/longvideobench/LongVideoBench","split":"val","task":"longvideobench"}
+{"annotation_path":"annotations/spatial_intelligence/mindcube/test.jsonl","artifact_paths":[],"benchmark":"mindcube","evaluation":{"aggregation":"micro","expected_group_counts":{"among":600,"around":250,"rotation":200},"metric_profile":"micro_accuracy","parser_profile":"multiple_choice","prompt_profile":"mindcube_official"},"expected_count":1050,"family":"spatial_intelligence","legacy_environment":{"MINDCUBE_BATCH_SIZE":16,"MINDCUBE_MAX_MODEL_LEN":32768,"MINDCUBE_MAX_NEW_TOKENS":1024},"license":"See the upstream benchmark terms","media_paths":["media/spatial_intelligence/mindcube/images","media/spatial_intelligence/mindcube/images/test"],"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/oscarqjh/MindCube_lmmseval/tree/7dd2725d9bd4149f2aad00a9843f72a3824da003","split":"test","task":"mindcube"}
+{"annotation_path":"annotations/video_qa/mlvu/dev.jsonl","artifact_paths":[],"benchmark":"mlvu","evaluation":{"metric_profile":"macro_accuracy","parser_profile":"multiple_choice","prompt_profile":"mlvu_default"},"expected_count":2174,"family":"video_qa","legacy_environment":{"MLVU_BATCH_SIZE":1,"MLVU_ENABLE_THINKING":false,"MLVU_FPS":2,"MLVU_GPU_MEMORY_UTILIZATION":0.9,"MLVU_MAX_FRAMES":384,"MLVU_MAX_MODEL_LEN":65536,"MLVU_MAX_NEW_TOKENS":128,"MLVU_MAX_NUM_BATCHED_TOKENS":65536,"MLVU_MIN_P":0.0,"MLVU_PRESENCE_PENALTY":0.0,"MLVU_PROMPT_MODE":"default","MLVU_TEMPERATURE":0.0,"MLVU_TOP_K":-1,"MLVU_TOP_P":1.0,"MLVU_VIDEO_MAX_PIXELS":262144,"MLVU_VIDEO_MIN_PIXELS":4096,"MLVU_VIDEO_TOTAL_PIXELS":0},"license":"See the upstream benchmark terms","media_paths":["media/video_qa/mlvu/videos","media/video_qa/mlvu/videos/dev"],"preprocessing":{"batch_size":1,"enable_thinking":false,"fps":2,"gpu_memory_utilization":0.9,"max_frames":384,"max_model_len":65536,"max_new_tokens":128,"max_num_batched_tokens":65536,"min_p":0.0,"presence_penalty":0.0,"prompt_mode":"default","temperature":0.0,"top_k":-1,"top_p":1.0,"video_max_pixels":262144,"video_min_pixels":4096,"video_total_pixels":0},"redistribution_authorized":true,"schema_version":1,"source_url":"https://github.com/JUNJIE99/MLVU","split":"dev","task":"mlvu"}
+{"annotation_path":"annotations/spatial_intelligence/mmsi/test.jsonl","artifact_paths":[],"benchmark":"mmsi","evaluation":{"metric_profile":"accuracy","parser_profile":"multiple_choice","prompt_profile":"mmsi_default"},"expected_count":1000,"family":"spatial_intelligence","legacy_environment":{"MMSI_BACKEND":"transformers","MMSI_ENABLE_THINKING":false,"MMSI_IMAGE_MAX_PIXELS":262144,"MMSI_IMAGE_MIN_PIXELS":4096},"license":"See the upstream benchmark terms","media_paths":["media/spatial_intelligence/mmsi/images/test"],"preprocessing":{"image_max_pixels":262144,"image_min_pixels":4096},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/RunsenXu/MMSI-Bench","split":"test","task":"mmsi"}
+{"annotation_path":"annotations/video_qa/mmvu/test.jsonl","artifact_paths":[],"benchmark":"mmvu","evaluation":{"metric_profile":"accuracy","parser_profile":"multiple_choice","prompt_profile":"mmvu_default"},"expected_count":625,"family":"video_qa","legacy_environment":{"MMVU_BATCH_SIZE":1,"MMVU_ENABLE_THINKING":false,"MMVU_FPS":2,"MMVU_GPU_MEMORY_UTILIZATION":0.9,"MMVU_MAX_FRAMES":384,"MMVU_MAX_MODEL_LEN":65536,"MMVU_MAX_NEW_TOKENS":128,"MMVU_MAX_NUM_BATCHED_TOKENS":65536,"MMVU_MIN_P":0.0,"MMVU_PRESENCE_PENALTY":0.0,"MMVU_PROMPT_MODE":"default","MMVU_TEMPERATURE":0.0,"MMVU_TOP_K":-1,"MMVU_TOP_P":1.0,"MMVU_VIDEO_MAX_PIXELS":262144,"MMVU_VIDEO_MIN_PIXELS":4096,"MMVU_VIDEO_TOTAL_PIXELS":0},"license":"See the upstream benchmark terms","media_paths":["media/video_qa/mmvu/videos","media/video_qa/mmvu/videos/test"],"preprocessing":{"batch_size":1,"enable_thinking":false,"fps":2,"gpu_memory_utilization":0.9,"max_frames":384,"max_model_len":65536,"max_new_tokens":128,"max_num_batched_tokens":65536,"min_p":0.0,"presence_penalty":0.0,"prompt_mode":"default","temperature":0.0,"top_k":-1,"top_p":1.0,"video_max_pixels":262144,"video_min_pixels":4096,"video_total_pixels":0},"redistribution_authorized":true,"schema_version":1,"source_url":"https://github.com/yale-nlp/MMVU","split":"test","task":"mmvu"}
+{"annotation_path":"annotations/video_qa/mvbench/test.jsonl","artifact_paths":[],"benchmark":"mvbench","evaluation":{"metric_profile":"accuracy","parser_profile":"multiple_choice","prompt_profile":"mvbench_default"},"expected_count":3586,"family":"video_qa","legacy_environment":{"MVBENCH_BATCH_SIZE":1,"MVBENCH_ENABLE_THINKING":false,"MVBENCH_FPS":2,"MVBENCH_GPU_MEMORY_UTILIZATION":0.9,"MVBENCH_MAX_FRAMES":384,"MVBENCH_MAX_MODEL_LEN":65536,"MVBENCH_MAX_NEW_TOKENS":128,"MVBENCH_MAX_NUM_BATCHED_TOKENS":65536,"MVBENCH_MIN_P":0.0,"MVBENCH_PRESENCE_PENALTY":0.0,"MVBENCH_PROMPT_MODE":"default","MVBENCH_TEMPERATURE":0.0,"MVBENCH_TOP_K":-1,"MVBENCH_TOP_P":1.0,"MVBENCH_VIDEO_MAX_PIXELS":262144,"MVBENCH_VIDEO_MIN_PIXELS":4096,"MVBENCH_VIDEO_TOTAL_PIXELS":0},"license":"See the upstream benchmark terms","media_paths":["media/video_qa/mvbench/videos","media/video_qa/mvbench/videos/test"],"preprocessing":{"batch_size":1,"enable_thinking":false,"fps":2,"gpu_memory_utilization":0.9,"max_frames":384,"max_model_len":65536,"max_new_tokens":128,"max_num_batched_tokens":65536,"min_p":0.0,"presence_penalty":0.0,"prompt_mode":"default","temperature":0.0,"top_k":-1,"top_p":1.0,"video_max_pixels":262144,"video_min_pixels":4096,"video_total_pixels":0},"redistribution_authorized":true,"schema_version":1,"source_url":"https://github.com/OpenGVLab/Ask-Anything","split":"test","task":"mvbench"}
+{"annotation_path":"annotations/spatial_intelligence/revsi/test.jsonl","artifact_paths":[],"benchmark":"revsi","evaluation":{"aggregation":"macro_by_question_type","frame_protocol":"native_all_frame","metric_profile":"revsi_macro_average","parser_profile":"revsi_answer","prompt_profile":"revsi_official"},"expected_count":6808,"family":"spatial_intelligence","legacy_environment":{"REVSI_BATCH_SIZE":16,"REVSI_ENABLE_THINKING":false,"REVSI_EXACT_NFRAMES":true,"REVSI_EXPECTED_SAMPLES":6808,"REVSI_FPS":2,"REVSI_FRAME_BUDGET":"all","REVSI_GPU_MEMORY_UTILIZATION":0.9,"REVSI_MAX_FRAMES":128,"REVSI_MAX_MODEL_LEN":32768,"REVSI_MAX_NEW_TOKENS":64,"REVSI_SETTING":"native-all-f128-exacttrue-fps2-min65536-maxnone-total16777216","REVSI_VIDEO_MIN_PIXELS":65536,"REVSI_VIDEO_TOTAL_PIXELS":16777216},"license":"See the upstream benchmark terms","media_paths":["media/spatial_intelligence/revsi/videos","media/spatial_intelligence/revsi/videos/test"],"preprocessing":{"exact_nframes":true,"fps":2,"max_frames":128,"video_min_pixels":65536,"video_total_pixels":16777216},"redistribution_authorized":true,"schema_version":1,"source_url":"https://arxiv.org/abs/2605.25979","split":"test","task":"revsi"}
+{"annotation_path":"annotations/segmentation/mevis.jsonl","artifact_paths":[],"benchmark":"segmentation","evaluation":{"metric_profile":"segmentation_official","oracle_location":"task_payload.segmentation_output","oracle_profile":"segmentation_rle","parser_profile":"sam2_prompt","prompt_profile":"train_seg"},"expected_count":424,"family":"segmentation","legacy_environment":{"SEGMENTATION_BATCH_SIZE":16,"SEGMENTATION_DATASETS":"refcoco,refcocop,refcocog,mevis,reasonvos","SEGMENTATION_DATA_TYPE":"all","SEGMENTATION_ENABLE_THINKING":false,"SEGMENTATION_FPS":2,"SEGMENTATION_GPU_MEM_UTIL":0.85,"SEGMENTATION_MAX_FRAMES":128,"SEGMENTATION_MAX_MODEL_LEN":32768,"SEGMENTATION_MAX_NEW_TOKENS":1024,"SEGMENTATION_PROMPT_MODE":"train_seg","SEGMENTATION_RUN_SAM2":false,"SEGMENTATION_SEED":42,"SEGMENTATION_SETTING":"segmentation-refcoco_refcocop_refcocog_mevis_reasonvos-train_seg-f128-fps2-min4096-max262144-total16777216-readerdecord-new1024","SEGMENTATION_VIDEO_MAX_PIXELS":262144,"SEGMENTATION_VIDEO_MIN_PIXELS":4096,"SEGMENTATION_VIDEO_READER":"decord","SEGMENTATION_VIDEO_TOTAL_PIXELS":16777216},"license":"See the upstream benchmark terms","media_paths":["media/segmentation","media/segmentation/videos/mevis"],"preprocessing":{"fps":2,"max_frames":128,"video_max_pixels":262144,"video_min_pixels":4096,"video_reader":"decord","video_total_pixels":16777216},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/OneThink/OneThinker-eval","split":"mevis","task":"segmentation"}
+{"annotation_path":"annotations/segmentation/reasonvos.jsonl","artifact_paths":[],"benchmark":"segmentation","evaluation":{"metric_profile":"segmentation_official","oracle_location":"task_payload.segmentation_output","oracle_profile":"segmentation_rle","parser_profile":"sam2_prompt","prompt_profile":"train_seg"},"expected_count":458,"family":"segmentation","legacy_environment":{"SEGMENTATION_BATCH_SIZE":16,"SEGMENTATION_DATASETS":"refcoco,refcocop,refcocog,mevis,reasonvos","SEGMENTATION_DATA_TYPE":"all","SEGMENTATION_ENABLE_THINKING":false,"SEGMENTATION_FPS":2,"SEGMENTATION_GPU_MEM_UTIL":0.85,"SEGMENTATION_MAX_FRAMES":128,"SEGMENTATION_MAX_MODEL_LEN":32768,"SEGMENTATION_MAX_NEW_TOKENS":1024,"SEGMENTATION_PROMPT_MODE":"train_seg","SEGMENTATION_RUN_SAM2":false,"SEGMENTATION_SEED":42,"SEGMENTATION_SETTING":"segmentation-refcoco_refcocop_refcocog_mevis_reasonvos-train_seg-f128-fps2-min4096-max262144-total16777216-readerdecord-new1024","SEGMENTATION_VIDEO_MAX_PIXELS":262144,"SEGMENTATION_VIDEO_MIN_PIXELS":4096,"SEGMENTATION_VIDEO_READER":"decord","SEGMENTATION_VIDEO_TOTAL_PIXELS":16777216},"license":"See the upstream benchmark terms","media_paths":["media/segmentation","media/segmentation/videos/reasonvos"],"preprocessing":{"fps":2,"max_frames":128,"video_max_pixels":262144,"video_min_pixels":4096,"video_reader":"decord","video_total_pixels":16777216},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/OneThink/OneThinker-eval","split":"reasonvos","task":"segmentation"}
+{"annotation_path":"annotations/segmentation/refcoco.jsonl","artifact_paths":[],"benchmark":"segmentation","evaluation":{"metric_profile":"segmentation_official","oracle_location":"task_payload.segmentation_output","oracle_profile":"segmentation_rle","parser_profile":"sam2_prompt","prompt_profile":"train_seg"},"expected_count":3811,"family":"segmentation","legacy_environment":{"SEGMENTATION_BATCH_SIZE":16,"SEGMENTATION_DATASETS":"refcoco,refcocop,refcocog,mevis,reasonvos","SEGMENTATION_DATA_TYPE":"all","SEGMENTATION_ENABLE_THINKING":false,"SEGMENTATION_FPS":2,"SEGMENTATION_GPU_MEM_UTIL":0.85,"SEGMENTATION_MAX_FRAMES":128,"SEGMENTATION_MAX_MODEL_LEN":32768,"SEGMENTATION_MAX_NEW_TOKENS":1024,"SEGMENTATION_PROMPT_MODE":"train_seg","SEGMENTATION_RUN_SAM2":false,"SEGMENTATION_SEED":42,"SEGMENTATION_SETTING":"segmentation-refcoco_refcocop_refcocog_mevis_reasonvos-train_seg-f128-fps2-min4096-max262144-total16777216-readerdecord-new1024","SEGMENTATION_VIDEO_MAX_PIXELS":262144,"SEGMENTATION_VIDEO_MIN_PIXELS":4096,"SEGMENTATION_VIDEO_READER":"decord","SEGMENTATION_VIDEO_TOTAL_PIXELS":16777216},"license":"See the upstream benchmark terms","media_paths":["media/segmentation","media/segmentation/images/refcoco"],"preprocessing":{"fps":2,"max_frames":128,"video_max_pixels":262144,"video_min_pixels":4096,"video_reader":"decord","video_total_pixels":16777216},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/OneThink/OneThinker-eval","split":"refcoco","task":"segmentation"}
+{"annotation_path":"annotations/segmentation/refcocog.jsonl","artifact_paths":[],"benchmark":"segmentation","evaluation":{"metric_profile":"segmentation_official","oracle_location":"task_payload.segmentation_output","oracle_profile":"segmentation_rle","parser_profile":"sam2_prompt","prompt_profile":"train_seg"},"expected_count":2537,"family":"segmentation","legacy_environment":{"SEGMENTATION_BATCH_SIZE":16,"SEGMENTATION_DATASETS":"refcoco,refcocop,refcocog,mevis,reasonvos","SEGMENTATION_DATA_TYPE":"all","SEGMENTATION_ENABLE_THINKING":false,"SEGMENTATION_FPS":2,"SEGMENTATION_GPU_MEM_UTIL":0.85,"SEGMENTATION_MAX_FRAMES":128,"SEGMENTATION_MAX_MODEL_LEN":32768,"SEGMENTATION_MAX_NEW_TOKENS":1024,"SEGMENTATION_PROMPT_MODE":"train_seg","SEGMENTATION_RUN_SAM2":false,"SEGMENTATION_SEED":42,"SEGMENTATION_SETTING":"segmentation-refcoco_refcocop_refcocog_mevis_reasonvos-train_seg-f128-fps2-min4096-max262144-total16777216-readerdecord-new1024","SEGMENTATION_VIDEO_MAX_PIXELS":262144,"SEGMENTATION_VIDEO_MIN_PIXELS":4096,"SEGMENTATION_VIDEO_READER":"decord","SEGMENTATION_VIDEO_TOTAL_PIXELS":16777216},"license":"See the upstream benchmark terms","media_paths":["media/segmentation","media/segmentation/images/refcocog"],"preprocessing":{"fps":2,"max_frames":128,"video_max_pixels":262144,"video_min_pixels":4096,"video_reader":"decord","video_total_pixels":16777216},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/OneThink/OneThinker-eval","split":"refcocog","task":"segmentation"}
+{"annotation_path":"annotations/segmentation/refcocop.jsonl","artifact_paths":[],"benchmark":"segmentation","evaluation":{"metric_profile":"segmentation_official","oracle_location":"task_payload.segmentation_output","oracle_profile":"segmentation_rle","parser_profile":"sam2_prompt","prompt_profile":"train_seg"},"expected_count":3805,"family":"segmentation","legacy_environment":{"SEGMENTATION_BATCH_SIZE":16,"SEGMENTATION_DATASETS":"refcoco,refcocop,refcocog,mevis,reasonvos","SEGMENTATION_DATA_TYPE":"all","SEGMENTATION_ENABLE_THINKING":false,"SEGMENTATION_FPS":2,"SEGMENTATION_GPU_MEM_UTIL":0.85,"SEGMENTATION_MAX_FRAMES":128,"SEGMENTATION_MAX_MODEL_LEN":32768,"SEGMENTATION_MAX_NEW_TOKENS":1024,"SEGMENTATION_PROMPT_MODE":"train_seg","SEGMENTATION_RUN_SAM2":false,"SEGMENTATION_SEED":42,"SEGMENTATION_SETTING":"segmentation-refcoco_refcocop_refcocog_mevis_reasonvos-train_seg-f128-fps2-min4096-max262144-total16777216-readerdecord-new1024","SEGMENTATION_VIDEO_MAX_PIXELS":262144,"SEGMENTATION_VIDEO_MIN_PIXELS":4096,"SEGMENTATION_VIDEO_READER":"decord","SEGMENTATION_VIDEO_TOTAL_PIXELS":16777216},"license":"See the upstream benchmark terms","media_paths":["media/segmentation","media/segmentation/images/refcocop"],"preprocessing":{"fps":2,"max_frames":128,"video_max_pixels":262144,"video_min_pixels":4096,"video_reader":"decord","video_total_pixels":16777216},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/OneThink/OneThinker-eval","split":"refcocop","task":"segmentation"}
+{"annotation_path":"annotations/spatial_grounding/refcoco_test_a.jsonl","artifact_paths":[],"benchmark":"spatial_grounding","evaluation":{"metric_profile":"refcoco_iou","parser_profile":"norm1000_bbox","prompt_profile":"qwen_native"},"expected_count":5657,"family":"spatial_grounding","legacy_environment":{"SPATIAL_GROUNDING_BBOX_SELECT":"first","SPATIAL_GROUNDING_COORD_SYSTEM":"norm1000","SPATIAL_GROUNDING_DATASETS":"refcoco-val,refcoco-testA,refcoco-testB,refcoco+-val,refcoco+-testA,refcoco+-testB,refcocog-val,refcocog-test","SPATIAL_GROUNDING_MAX_NEW_TOKENS":1024,"SPATIAL_GROUNDING_MIN_TOKENS":64,"SPATIAL_GROUNDING_PROMPT_STYLE":"qwen_native","SPATIAL_GROUNDING_TOTAL_TOKENS":1024},"license":"See the upstream benchmark terms","media_paths":["media/spatial_grounding/images","media/spatial_grounding/images/refcoco_test_a"],"preprocessing":{"coordinate_system":"norm1000"},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/OneThink/OneThinker-eval","split":"refcoco_test_a","task":"spatial_grounding"}
+{"annotation_path":"annotations/spatial_grounding/refcoco_test_b.jsonl","artifact_paths":[],"benchmark":"spatial_grounding","evaluation":{"metric_profile":"refcoco_iou","parser_profile":"norm1000_bbox","prompt_profile":"qwen_native"},"expected_count":5095,"family":"spatial_grounding","legacy_environment":{"SPATIAL_GROUNDING_BBOX_SELECT":"first","SPATIAL_GROUNDING_COORD_SYSTEM":"norm1000","SPATIAL_GROUNDING_DATASETS":"refcoco-val,refcoco-testA,refcoco-testB,refcoco+-val,refcoco+-testA,refcoco+-testB,refcocog-val,refcocog-test","SPATIAL_GROUNDING_MAX_NEW_TOKENS":1024,"SPATIAL_GROUNDING_MIN_TOKENS":64,"SPATIAL_GROUNDING_PROMPT_STYLE":"qwen_native","SPATIAL_GROUNDING_TOTAL_TOKENS":1024},"license":"See the upstream benchmark terms","media_paths":["media/spatial_grounding/images","media/spatial_grounding/images/refcoco_test_b"],"preprocessing":{"coordinate_system":"norm1000"},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/OneThink/OneThinker-eval","split":"refcoco_test_b","task":"spatial_grounding"}
+{"annotation_path":"annotations/spatial_grounding/refcoco_val.jsonl","artifact_paths":[],"benchmark":"spatial_grounding","evaluation":{"metric_profile":"refcoco_iou","parser_profile":"norm1000_bbox","prompt_profile":"qwen_native"},"expected_count":10834,"family":"spatial_grounding","legacy_environment":{"SPATIAL_GROUNDING_BBOX_SELECT":"first","SPATIAL_GROUNDING_COORD_SYSTEM":"norm1000","SPATIAL_GROUNDING_DATASETS":"refcoco-val,refcoco-testA,refcoco-testB,refcoco+-val,refcoco+-testA,refcoco+-testB,refcocog-val,refcocog-test","SPATIAL_GROUNDING_MAX_NEW_TOKENS":1024,"SPATIAL_GROUNDING_MIN_TOKENS":64,"SPATIAL_GROUNDING_PROMPT_STYLE":"qwen_native","SPATIAL_GROUNDING_TOTAL_TOKENS":1024},"license":"See the upstream benchmark terms","media_paths":["media/spatial_grounding/images","media/spatial_grounding/images/refcoco_val"],"preprocessing":{"coordinate_system":"norm1000"},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/OneThink/OneThinker-eval","split":"refcoco_val","task":"spatial_grounding"}
+{"annotation_path":"annotations/spatial_grounding/refcocog_test.jsonl","artifact_paths":[],"benchmark":"spatial_grounding","evaluation":{"metric_profile":"refcoco_iou","parser_profile":"norm1000_bbox","prompt_profile":"qwen_native"},"expected_count":9602,"family":"spatial_grounding","legacy_environment":{"SPATIAL_GROUNDING_BBOX_SELECT":"first","SPATIAL_GROUNDING_COORD_SYSTEM":"norm1000","SPATIAL_GROUNDING_DATASETS":"refcoco-val,refcoco-testA,refcoco-testB,refcoco+-val,refcoco+-testA,refcoco+-testB,refcocog-val,refcocog-test","SPATIAL_GROUNDING_MAX_NEW_TOKENS":1024,"SPATIAL_GROUNDING_MIN_TOKENS":64,"SPATIAL_GROUNDING_PROMPT_STYLE":"qwen_native","SPATIAL_GROUNDING_TOTAL_TOKENS":1024},"license":"See the upstream benchmark terms","media_paths":["media/spatial_grounding/images","media/spatial_grounding/images/refcocog_test"],"preprocessing":{"coordinate_system":"norm1000"},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/OneThink/OneThinker-eval","split":"refcocog_test","task":"spatial_grounding"}
+{"annotation_path":"annotations/spatial_grounding/refcocog_val.jsonl","artifact_paths":[],"benchmark":"spatial_grounding","evaluation":{"metric_profile":"refcoco_iou","parser_profile":"norm1000_bbox","prompt_profile":"qwen_native"},"expected_count":4896,"family":"spatial_grounding","legacy_environment":{"SPATIAL_GROUNDING_BBOX_SELECT":"first","SPATIAL_GROUNDING_COORD_SYSTEM":"norm1000","SPATIAL_GROUNDING_DATASETS":"refcoco-val,refcoco-testA,refcoco-testB,refcoco+-val,refcoco+-testA,refcoco+-testB,refcocog-val,refcocog-test","SPATIAL_GROUNDING_MAX_NEW_TOKENS":1024,"SPATIAL_GROUNDING_MIN_TOKENS":64,"SPATIAL_GROUNDING_PROMPT_STYLE":"qwen_native","SPATIAL_GROUNDING_TOTAL_TOKENS":1024},"license":"See the upstream benchmark terms","media_paths":["media/spatial_grounding/images","media/spatial_grounding/images/refcocog_val"],"preprocessing":{"coordinate_system":"norm1000"},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/OneThink/OneThinker-eval","split":"refcocog_val","task":"spatial_grounding"}
+{"annotation_path":"annotations/spatial_grounding/refcocop_test_a.jsonl","artifact_paths":[],"benchmark":"spatial_grounding","evaluation":{"metric_profile":"refcoco_iou","parser_profile":"norm1000_bbox","prompt_profile":"qwen_native"},"expected_count":5726,"family":"spatial_grounding","legacy_environment":{"SPATIAL_GROUNDING_BBOX_SELECT":"first","SPATIAL_GROUNDING_COORD_SYSTEM":"norm1000","SPATIAL_GROUNDING_DATASETS":"refcoco-val,refcoco-testA,refcoco-testB,refcoco+-val,refcoco+-testA,refcoco+-testB,refcocog-val,refcocog-test","SPATIAL_GROUNDING_MAX_NEW_TOKENS":1024,"SPATIAL_GROUNDING_MIN_TOKENS":64,"SPATIAL_GROUNDING_PROMPT_STYLE":"qwen_native","SPATIAL_GROUNDING_TOTAL_TOKENS":1024},"license":"See the upstream benchmark terms","media_paths":["media/spatial_grounding/images","media/spatial_grounding/images/refcocop_test_a"],"preprocessing":{"coordinate_system":"norm1000"},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/OneThink/OneThinker-eval","split":"refcocop_test_a","task":"spatial_grounding"}
+{"annotation_path":"annotations/spatial_grounding/refcocop_test_b.jsonl","artifact_paths":[],"benchmark":"spatial_grounding","evaluation":{"metric_profile":"refcoco_iou","parser_profile":"norm1000_bbox","prompt_profile":"qwen_native"},"expected_count":4889,"family":"spatial_grounding","legacy_environment":{"SPATIAL_GROUNDING_BBOX_SELECT":"first","SPATIAL_GROUNDING_COORD_SYSTEM":"norm1000","SPATIAL_GROUNDING_DATASETS":"refcoco-val,refcoco-testA,refcoco-testB,refcoco+-val,refcoco+-testA,refcoco+-testB,refcocog-val,refcocog-test","SPATIAL_GROUNDING_MAX_NEW_TOKENS":1024,"SPATIAL_GROUNDING_MIN_TOKENS":64,"SPATIAL_GROUNDING_PROMPT_STYLE":"qwen_native","SPATIAL_GROUNDING_TOTAL_TOKENS":1024},"license":"See the upstream benchmark terms","media_paths":["media/spatial_grounding/images","media/spatial_grounding/images/refcocop_test_b"],"preprocessing":{"coordinate_system":"norm1000"},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/OneThink/OneThinker-eval","split":"refcocop_test_b","task":"spatial_grounding"}
+{"annotation_path":"annotations/spatial_grounding/refcocop_val.jsonl","artifact_paths":[],"benchmark":"spatial_grounding","evaluation":{"metric_profile":"refcoco_iou","parser_profile":"norm1000_bbox","prompt_profile":"qwen_native"},"expected_count":10758,"family":"spatial_grounding","legacy_environment":{"SPATIAL_GROUNDING_BBOX_SELECT":"first","SPATIAL_GROUNDING_COORD_SYSTEM":"norm1000","SPATIAL_GROUNDING_DATASETS":"refcoco-val,refcoco-testA,refcoco-testB,refcoco+-val,refcoco+-testA,refcoco+-testB,refcocog-val,refcocog-test","SPATIAL_GROUNDING_MAX_NEW_TOKENS":1024,"SPATIAL_GROUNDING_MIN_TOKENS":64,"SPATIAL_GROUNDING_PROMPT_STYLE":"qwen_native","SPATIAL_GROUNDING_TOTAL_TOKENS":1024},"license":"See the upstream benchmark terms","media_paths":["media/spatial_grounding/images","media/spatial_grounding/images/refcocop_val"],"preprocessing":{"coordinate_system":"norm1000"},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/OneThink/OneThinker-eval","split":"refcocop_val","task":"spatial_grounding"}
+{"annotation_path":"annotations/stvg/stvg.jsonl","artifact_paths":[],"benchmark":"stvg","evaluation":{"metric_profile":"stvg_official","parser_profile":"temporal_spatial_boxes","prompt_profile":"train_stvg"},"expected_count":1851,"family":"spatial_temporal_grounding","legacy_environment":{"STVG_DATASETS":"eval_stvg","STVG_FPS":2,"STVG_MAX_FRAMES":128,"STVG_MAX_NEW_TOKENS":2048,"STVG_PROMPT_MODE":"train_stvg","STVG_VIDEO_MAX_PIXELS":393216,"STVG_VIDEO_MIN_PIXELS":65536,"STVG_VIDEO_TOTAL_PIXELS":10485760},"license":"See the upstream benchmark terms","media_paths":["media/stvg","media/stvg/videos/stvg"],"preprocessing":{"stvg_fps":2,"stvg_max_frames":128,"stvg_video_max_pixels":393216,"stvg_video_min_pixels":65536,"stvg_video_total_pixels":10485760},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/OneThink/OneThinker-eval","split":"stvg","task":"stvg"}
+{"annotation_path":"annotations/temporal_grounding/activitynet_timelens.jsonl","artifact_paths":[],"benchmark":"temporal_grounding","checksums":{"annotations/temporal_grounding/activitynet_timelens.jsonl":"3091f03a20b2bf4548d4079d95bb8dc564da53d4e6b2c32470862b15622c7a77"},"evaluation":{"metric_profile":"temporal_iou","parser_profile":"temporal_spans","prompt_profile":"timelens_same"},"expected_count":4500,"family":"temporal_grounding","legacy_environment":{"TIMELENS_DATASETS":"charades_timelens,activitynet_timelens,qvhighlights_timelens","TIMELENS_ENABLE_THINKING":false,"TIMELENS_FPS":4,"TIMELENS_MAX_FRAMES":2048,"TIMELENS_MAX_NEW_TOKENS":128,"TIMELENS_MAX_PIXELS":409600,"TIMELENS_MIN_TOKENS":1,"TIMELENS_NUM_WORKERS":2,"TIMELENS_PROMPT_MODE":"same","TIMELENS_STOP_AFTER_ANSWER":true,"TIMELENS_TOTAL_TOKENS":128000},"license":"See the upstream benchmark terms","media_paths":["media/temporal_grounding/videos"],"preprocessing":{"fps":4,"max_frames":2048,"max_pixels":409600,"min_tokens":1,"total_tokens":128000},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/TencentARC/TimeLens-Bench","split":"activitynet_timelens","task":"temporal_grounding"}
+{"annotation_path":"annotations/temporal_grounding/charades_timelens.jsonl","artifact_paths":[],"benchmark":"temporal_grounding","checksums":{"annotations/temporal_grounding/charades_timelens.jsonl":"af289d9be3e8c56f98b12c39bda37758a34205c1aa75e2c03e3174dbbe6f2794"},"evaluation":{"metric_profile":"temporal_iou","parser_profile":"temporal_spans","prompt_profile":"timelens_same"},"expected_count":3363,"family":"temporal_grounding","legacy_environment":{"TIMELENS_DATASETS":"charades_timelens,activitynet_timelens,qvhighlights_timelens","TIMELENS_ENABLE_THINKING":false,"TIMELENS_FPS":4,"TIMELENS_MAX_FRAMES":2048,"TIMELENS_MAX_NEW_TOKENS":128,"TIMELENS_MAX_PIXELS":409600,"TIMELENS_MIN_TOKENS":1,"TIMELENS_NUM_WORKERS":2,"TIMELENS_PROMPT_MODE":"same","TIMELENS_STOP_AFTER_ANSWER":true,"TIMELENS_TOTAL_TOKENS":128000},"license":"See the upstream benchmark terms","media_paths":["media/temporal_grounding/videos"],"preprocessing":{"fps":4,"max_frames":2048,"max_pixels":409600,"min_tokens":1,"total_tokens":128000},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/TencentARC/TimeLens-Bench","split":"charades_timelens","task":"temporal_grounding"}
+{"annotation_path":"annotations/temporal_grounding/qvhighlights_timelens.jsonl","artifact_paths":[],"benchmark":"temporal_grounding","checksums":{"annotations/temporal_grounding/qvhighlights_timelens.jsonl":"922a736a73ff9fb3d3996e0b75ee1343283bee23d7e60b368d6840f8612e619b"},"evaluation":{"metric_profile":"temporal_iou","parser_profile":"temporal_spans","prompt_profile":"timelens_same"},"expected_count":1541,"family":"temporal_grounding","legacy_environment":{"TIMELENS_DATASETS":"charades_timelens,activitynet_timelens,qvhighlights_timelens","TIMELENS_ENABLE_THINKING":false,"TIMELENS_FPS":4,"TIMELENS_MAX_FRAMES":2048,"TIMELENS_MAX_NEW_TOKENS":128,"TIMELENS_MAX_PIXELS":409600,"TIMELENS_MIN_TOKENS":1,"TIMELENS_NUM_WORKERS":2,"TIMELENS_PROMPT_MODE":"same","TIMELENS_STOP_AFTER_ANSWER":true,"TIMELENS_TOTAL_TOKENS":128000},"license":"See the upstream benchmark terms","media_paths":["media/temporal_grounding/videos"],"preprocessing":{"fps":4,"max_frames":2048,"max_pixels":409600,"min_tokens":1,"total_tokens":128000},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/TencentARC/TimeLens-Bench","split":"qvhighlights_timelens","task":"temporal_grounding"}
+{"annotation_path":"annotations/tracking/got10k.jsonl","artifact_paths":[],"benchmark":"tracking","checksums":{"annotations/tracking/got10k.jsonl":"1206da3c1fd75360ccea074a9e211965fe2d8cfd514903564b037641a00fb12b"},"evaluation":{"metric_profile":"got10k_ao","parser_profile":"tracking_boxes","prompt_profile":"tracking_default"},"expected_count":180,"family":"tracking","legacy_environment":{"TRACKING_DATASETS":"eval_got10k","TRACKING_FPS":1,"TRACKING_MAX_FRAMES":32,"TRACKING_MAX_NEW_TOKENS":8192,"TRACKING_PROMPT_MODE":"default","TRACKING_VIDEO_MAX_PIXELS":786432,"TRACKING_VIDEO_MIN_PIXELS":4096,"TRACKING_VIDEO_TOTAL_PIXELS":8388608},"license":"See the upstream benchmark terms","media_paths":["media/tracking","media/tracking/videos"],"preprocessing":{"tracking_fps":1,"tracking_max_frames":32,"tracking_video_max_pixels":786432,"tracking_video_min_pixels":4096,"tracking_video_total_pixels":8388608},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/OneThink/OneThinker-eval","split":"got10k","task":"tracking"}
+{"annotation_path":"annotations/video_qa/videoholmes/test.jsonl","artifact_paths":[],"benchmark":"videoholmes","evaluation":{"metric_profile":"accuracy","parser_profile":"multiple_choice","prompt_profile":"videoholmes_default"},"expected_count":1837,"family":"video_qa","legacy_environment":{"VIDEOHOLMES_BATCH_SIZE":1,"VIDEOHOLMES_ENABLE_THINKING":false,"VIDEOHOLMES_FPS":2,"VIDEOHOLMES_GPU_MEMORY_UTILIZATION":0.9,"VIDEOHOLMES_MAX_FRAMES":384,"VIDEOHOLMES_MAX_MODEL_LEN":65536,"VIDEOHOLMES_MAX_NEW_TOKENS":128,"VIDEOHOLMES_MAX_NUM_BATCHED_TOKENS":65536,"VIDEOHOLMES_MIN_P":0.0,"VIDEOHOLMES_PRESENCE_PENALTY":0.0,"VIDEOHOLMES_PROMPT_MODE":"default","VIDEOHOLMES_TEMPERATURE":0.0,"VIDEOHOLMES_TOP_K":-1,"VIDEOHOLMES_TOP_P":1.0,"VIDEOHOLMES_VIDEO_MAX_PIXELS":262144,"VIDEOHOLMES_VIDEO_MIN_PIXELS":4096,"VIDEOHOLMES_VIDEO_TOTAL_PIXELS":0},"license":"See the upstream benchmark terms","media_paths":["media/video_qa/videoholmes/videos","media/video_qa/videoholmes/videos/test"],"preprocessing":{"batch_size":1,"enable_thinking":false,"fps":2,"gpu_memory_utilization":0.9,"max_frames":384,"max_model_len":65536,"max_new_tokens":128,"max_num_batched_tokens":65536,"min_p":0.0,"presence_penalty":0.0,"prompt_mode":"default","temperature":0.0,"top_k":-1,"top_p":1.0,"video_max_pixels":262144,"video_min_pixels":4096,"video_total_pixels":0},"redistribution_authorized":true,"schema_version":1,"source_url":"https://github.com/TencentARC/Video-Holmes","split":"test","task":"videoholmes"}
+{"annotation_path":"annotations/video_qa/videomme/test.jsonl","artifact_paths":["artifacts/video_qa/videomme"],"benchmark":"videomme","evaluation":{"metric_profile":"accuracy","parser_profile":"multiple_choice","prompt_profile":"videomme_default"},"expected_count":2700,"family":"video_qa","legacy_environment":{"VIDEOMME_BATCH_SIZE":1,"VIDEOMME_ENABLE_THINKING":false,"VIDEOMME_FPS":2,"VIDEOMME_GPU_MEMORY_UTILIZATION":0.9,"VIDEOMME_MAX_FRAMES":384,"VIDEOMME_MAX_MODEL_LEN":65536,"VIDEOMME_MAX_NEW_TOKENS":128,"VIDEOMME_MAX_NUM_BATCHED_TOKENS":65536,"VIDEOMME_MIN_P":0.0,"VIDEOMME_PRESENCE_PENALTY":0.0,"VIDEOMME_PROMPT_MODE":"default","VIDEOMME_SETTING":"all-qwen3_vl-sub0-f384-fps2-min4096-max262144-total0-videomme_preprocessed_384f_262k_total0","VIDEOMME_TEMPERATURE":0.0,"VIDEOMME_TOP_K":-1,"VIDEOMME_TOP_P":1.0,"VIDEOMME_VIDEO_MAX_PIXELS":262144,"VIDEOMME_VIDEO_MIN_PIXELS":4096,"VIDEOMME_VIDEO_TOTAL_PIXELS":0},"license":"See the upstream benchmark terms","media_paths":["media/video_qa/videomme/videos","media/video_qa/videomme/videos/test"],"preprocessing":{"batch_size":1,"enable_thinking":false,"fps":2,"gpu_memory_utilization":0.9,"max_frames":384,"max_model_len":65536,"max_new_tokens":128,"max_num_batched_tokens":65536,"min_p":0.0,"presence_penalty":0.0,"prompt_mode":"default","temperature":0.0,"top_k":-1,"top_p":1.0,"video_max_pixels":262144,"video_min_pixels":4096,"video_total_pixels":0},"redistribution_authorized":true,"schema_version":1,"source_url":"https://github.com/BradyFU/Video-MME","split":"test","task":"videomme"}
+{"annotation_path":"annotations/video_qa/videommev2/test.jsonl","artifact_paths":["artifacts/video_qa/videommev2"],"benchmark":"videommev2","evaluation":{"metric_profile":"accuracy","parser_profile":"multiple_choice","prompt_profile":"videommev2_default"},"expected_count":3200,"family":"video_qa","legacy_environment":{"VIDEOMMEV2_ANSWER_FILTER":"","VIDEOMMEV2_BATCH_SIZE":1,"VIDEOMMEV2_ENABLE_THINKING":false,"VIDEOMMEV2_FPS":2,"VIDEOMMEV2_GPU_MEMORY_UTILIZATION":0.9,"VIDEOMMEV2_MAX_FRAMES":384,"VIDEOMMEV2_MAX_MODEL_LEN":65536,"VIDEOMMEV2_MAX_NEW_TOKENS":128,"VIDEOMMEV2_MAX_NUM_BATCHED_TOKENS":65536,"VIDEOMMEV2_MIN_P":0.0,"VIDEOMMEV2_PRESENCE_PENALTY":0.0,"VIDEOMMEV2_PROMPT_MODE":"default","VIDEOMMEV2_TEMPERATURE":0.0,"VIDEOMMEV2_TOP_K":-1,"VIDEOMMEV2_TOP_P":1.0,"VIDEOMMEV2_VIDEO_MAX_PIXELS":262144,"VIDEOMMEV2_VIDEO_MIN_PIXELS":4096,"VIDEOMMEV2_VIDEO_TOTAL_PIXELS":0},"license":"See the upstream benchmark terms","media_paths":["media/video_qa/videommev2/videos","media/video_qa/videommev2/videos/test"],"preprocessing":{"batch_size":1,"enable_thinking":false,"fps":2,"gpu_memory_utilization":0.9,"max_frames":384,"max_model_len":65536,"max_new_tokens":128,"max_num_batched_tokens":65536,"min_p":0.0,"presence_penalty":0.0,"prompt_mode":"default","temperature":0.0,"top_k":-1,"top_p":1.0,"video_max_pixels":262144,"video_min_pixels":4096,"video_total_pixels":0},"redistribution_authorized":true,"schema_version":1,"source_url":"https://github.com/BradyFU/Video-MME","split":"test","task":"videommev2"}
+{"annotation_path":"annotations/spatial_intelligence/vsi/test.jsonl","artifact_paths":["artifacts/spatial_intelligence/vsi"],"benchmark":"vsi","evaluation":{"metric_profile":"vsi_official","parser_profile":"vsi_answer","prompt_profile":"vsi_default"},"expected_count":5130,"family":"spatial_intelligence","legacy_environment":{"VSI_BATCH_SIZE":16,"VSI_MAX_MODEL_LEN":32768,"VSI_MAX_NEW_TOKENS":1024,"VSI_SETTING":"video128-16M-video-f128-fps2-min65536-maxnone-total16777216-vsibench_preprocessed_128f_16M"},"license":"See the upstream benchmark terms","media_paths":["media/spatial_intelligence/vsi/videos","media/spatial_intelligence/vsi/videos/test"],"preprocessing":{"fps":2,"max_frames":128,"video_min_pixels":65536,"video_total_pixels":16777216},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/nyu-visionx/VSI-Bench","split":"test","task":"vsi"}

code/docs/environment.md

@@ -0,0 +1,60 @@
+<p align="right"><a href="environment_zh.md">简体中文</a></p>
+
+# Reproduce the OraRL Environment
+
+OraRL uses one reproducible software stack for policy optimization and
+task-native evaluation. The released configuration was validated on NVIDIA
+H20/Hopper GPUs with CUDA 12.9 and includes both runtimes in this repository.
+
+## Requirements
+
+- Linux with Conda available in `PATH`
+- NVIDIA GPU with a recent driver
+- CUDA 12.9-compatible hardware for the paper-matched environment
+
+CUDA 12.9 GA officially requires NVIDIA Linux driver 575.51.03 or newer.
+Other NVIDIA GPUs may work, but the published setup was validated on H20.
+
+## Install the pinned stack
+
+```bash
+bash scripts/create_conda_env.sh
+conda activate orarl
+
+python scripts/check_environment.py \
+  --require-gpu \
+  --model /path/to/local/model
+```
+
+The installer pins Python 3.11, PyTorch 2.10.0+cu129, Transformers 5.5.4,
+vLLM 0.19.1, FlashAttention 2.8.3, and the remaining packages in
+`requirements-cu129.txt`. The final editable install exposes the `orarl-*`
+commands and the bundled `verl` trainer package from this checkout; do not
+install an unrelated `verl` release from PyPI over it.
+
+## Validate an evaluation-only node
+
+If a node will only run evaluation, skip the training-side checks:
+
+```bash
+bash scripts/create_conda_env.sh --evaluation-only
+conda activate orarl
+
+python scripts/check_environment.py --evaluation-only
+```
+
+## Verify the release checkout
+
+Run the release checks before allocating a long job:
+
+```bash
+python scripts/check_release.py
+python -m pytest -q
+ruff check .
+```
+
+On a paper-matched H20 node, add `--require-gpu --require-h20` to
+`scripts/check_environment.py`. Use the same source revision and environment on
+every node of a distributed run.
+
+Continue with [Training](training.md) or [Evaluation](evaluation.md).

code/docs/environment_zh.md

@@ -0,0 +1,57 @@
+<p align="right"><a href="environment.md">English</a></p>
+
+# 复现 OraRL 环境
+
+OraRL 使用同一套可复现软件栈完成策略优化与任务原生评测。发布配置在 NVIDIA
+H20/Hopper 与 CUDA 12.9 上完成验证，本仓库同时包含两部分运行时。
+
+## 环境要求
+
+- `PATH` 中可用 Conda 的 Linux 系统
+- 驱动版本较新的 NVIDIA GPU
+- 复现论文环境时使用兼容 CUDA 12.9 的硬件
+
+CUDA 12.9 GA 官方要求 NVIDIA Linux 驱动不低于 575.51.03。其他 NVIDIA GPU
+也可能运行，但发布配置仅在 H20 上完成验证。
+
+## 安装固定版本软件栈
+
+```bash
+bash scripts/create_conda_env.sh
+conda activate orarl
+
+python scripts/check_environment.py \
+  --require-gpu \
+  --model /path/to/local/model
+```
+
+脚本会固定 Python 3.11、PyTorch 2.10.0+cu129、Transformers 5.5.4、
+vLLM 0.19.1、FlashAttention 2.8.3，以及 `requirements-cu129.txt` 中的其余依赖。
+最后一步的可编辑安装会同时提供 `orarl-*` 命令和本仓库自带的 `verl` 训练包；
+请勿再用 PyPI 上无关的 `verl` 版本覆盖它。
+
+## 验证仅用于评测的节点
+
+若某个节点只跑评测，可跳过训练侧检查：
+
+```bash
+bash scripts/create_conda_env.sh --evaluation-only
+conda activate orarl
+
+python scripts/check_environment.py --evaluation-only
+```
+
+## 验证发布源码
+
+分配长任务前先执行：
+
+```bash
+python scripts/check_release.py
+python -m pytest -q
+ruff check .
+```
+
+在与论文一致的 H20 节点上，可为 `scripts/check_environment.py` 增加
+`--require-gpu --require-h20`。多节点任务应在所有节点使用相同的源码版本和环境。
+
+随后参阅[训练](training_zh.md)或[评测](evaluation_zh.md)。

code/docs/evaluation.md

@@ -0,0 +1,121 @@
+<p align="right"><a href="evaluation_zh.md">简体中文</a></p>
+
+# Evaluate Video-ORA
+
+OraRL evaluates direct, task-native answers with canonical profiles for the
+seven released task families. The corresponding runtime lives under
+`eval/task/`; first
+[validate the evaluation node](environment.md#validate-an-evaluation-only-node).
+
+## 1. Materialize model and data
+
+Install the Hugging Face CLI and download the released assets:
+
+```bash
+python -m pip install -U huggingface_hub
+
+hf download OraRL/Video-ORA-9B \
+  --local-dir "$PWD/models/Video-ORA-9B"
+
+hf download OraRL/OraRL-Data \
+  --repo-type dataset \
+  --include "OraRL-eval-data/**" \
+  --local-dir "$PWD/OraRL-Data"
+```
+
+The complete evaluation release is large. Downloads are resumable, and
+`OraRL-eval-data/assets.jsonl` is the authoritative file inventory.
+
+The local dataset layout is:
+
+```text
+OraRL-Data/OraRL-eval-data/
+├── datasets.jsonl   # benchmark, prompt, parser, metric, and sampling profiles
+├── assets.jsonl     # released-file inventory
+├── annotations/     # canonical JSONL rows
+└── media/           # raw images, videos, and subtitles
+```
+
+Derived preprocessing caches are intentionally excluded. Evaluators decode the
+declared raw media when no compatible cache is present.
+
+## 2. Run the canonical paper profile
+
+Preview the resolved command first:
+
+```bash
+orarl-eval \
+  --model "$PWD/models/Video-ORA-9B" \
+  --tasks paper \
+  --dataset "$PWD/OraRL-Data/OraRL-eval-data" \
+  --summary "$PWD/outputs/Video-ORA-9B/evaluation.json"
+```
+
+`orarl-eval` is a dry run by default. After checking the model, dataset,
+evaluator, task profiles, and output paths, add `--run`.
+
+To validate the pipeline with a bounded smoke test:
+
+```bash
+orarl-eval \
+  --model "$PWD/models/Video-ORA-9B" \
+  --tasks videomme \
+  --dataset "$PWD/OraRL-Data/OraRL-eval-data" \
+  --max-samples 8 \
+  --summary "$PWD/outputs/Video-ORA-9B/videomme-smoke.json" \
+  --run
+```
+
+Smoke scores only validate execution and must not be reported as benchmark
+results.
+
+## 3. Compose a task suite
+
+`--tasks paper` selects all released tasks. `--tasks video_qa` selects the
+seven Video QA benchmarks. Individual task names may be comma-separated.
+
+| Family | Task names |
+| --- | --- |
+| Video QA | `videomme`, `videommev2`, `mvbench`, `mmvu`, `videoholmes`, `longvideobench`, `mlvu` |
+| Spatial intelligence | `vsi`, `mmsi`, `mindcube`, `revsi` |
+| Temporal grounding | `temporal_grounding` |
+| Spatial grounding | `spatial_grounding` |
+| Tracking | `tracking` |
+| Spatial-temporal grounding | `stvg` |
+| Segmentation | `segmentation` |
+
+The canonical profiles in `datasets.jsonl` pin frame sampling, resolution,
+prompts, parsers, and metrics. Changing them defines a different evaluation
+setting. ReVSI is reported separately from the three-benchmark
+spatial-intelligence average.
+
+## 4. Add segmentation post-processing
+
+Segmentation inference runs without SAM2 post-processing by default. Enabling
+`--segmentation-run-sam2` additionally requires:
+
+- SAM2 weights (`SEGMENTATION_SAM2_CKPT`)
+- the matching Hydra config (`SEGMENTATION_SAM2_CFG`)
+- the official OneThinker `seg_post_sam2.py`
+  (`SEGMENTATION_POSTPROCESSOR_PATH`)
+- the `sam2` Python package
+
+OraRL validates all three paths before launch.
+
+## 5. Preserve reportable outputs
+
+Each task evaluator writes its native summary, then `orarl-eval` creates the
+requested aggregate JSON with requested, completed, and missing tasks, the
+return code, and official metrics. Outputs are grouped by paper task family
+under `outputs/<model>/`.
+
+For a reportable run, retain:
+
+- the OraRL source revision
+- the model and data revisions
+- the exact command and aggregate summary
+- software versions and accelerator type/count
+- every non-default profile or CLI override
+
+See [`../eval/README.md`](../eval/README.md) for the evaluator map, video
+decoding backend, and checkpoint-format notes.

code/docs/evaluation_zh.md

@@ -0,0 +1,116 @@
+<p align="right"><a href="evaluation.md">English</a></p>
+
+# 评测 Video-ORA
+
+OraRL 使用七个发布任务族的规范配置评测任务原生答案，对应运行时位于
+`eval/task/`。运行前请先
+[验证评测节点](environment_zh.md#验证仅用于评测的节点)。
+
+## 1. 准备模型和数据
+
+安装 Hugging Face CLI 并下载发布文件：
+
+```bash
+python -m pip install -U huggingface_hub
+
+hf download OraRL/Video-ORA-9B \
+  --local-dir "$PWD/models/Video-ORA-9B"
+
+hf download OraRL/OraRL-Data \
+  --repo-type dataset \
+  --include "OraRL-eval-data/**" \
+  --local-dir "$PWD/OraRL-Data"
+```
+
+完整评测数据体积较大，下载支持断点续传；
+`OraRL-eval-data/assets.jsonl` 是权威文件清单。
+
+本地数据结构为：
+
+```text
+OraRL-Data/OraRL-eval-data/
+├── datasets.jsonl   # benchmark、prompt、parser、metric 与采样配置
+├── assets.jsonl     # 发布文件清单
+├── annotations/     # 规范化 JSONL 标注
+└── media/           # 原始图像、视频和字幕
+```
+
+发布数据有意排除衍生预处理缓存；不存在兼容缓存时，评测器会直接解码清单中声明的
+原始媒体。
+
+## 2. 运行论文规范配置
+
+先预览解析后的命令：
+
+```bash
+orarl-eval \
+  --model "$PWD/models/Video-ORA-9B" \
+  --tasks paper \
+  --dataset "$PWD/OraRL-Data/OraRL-eval-data" \
+  --summary "$PWD/outputs/Video-ORA-9B/evaluation.json"
+```
+
+`orarl-eval` 默认为 dry run。检查模型、数据、评测器、任务配置和输出路径后，
+增加 `--run`。
+
+可先运行限制样本数的 smoke test：
+
+```bash
+orarl-eval \
+  --model "$PWD/models/Video-ORA-9B" \
+  --tasks videomme \
+  --dataset "$PWD/OraRL-Data/OraRL-eval-data" \
+  --max-samples 8 \
+  --summary "$PWD/outputs/Video-ORA-9B/videomme-smoke.json" \
+  --run
+```
+
+Smoke test 分数仅用于验证流程，不应作为 benchmark 结果报告。
+
+## 3. 组合任务集
+
+`--tasks paper` 选择所有发布任务，`--tasks video_qa` 选择 7 个视频问答
+benchmark；多个单项任务可用逗号分隔。
+
+| 任务族 | 任务名 |
+| --- | --- |
+| 视频问答 | `videomme`, `videommev2`, `mvbench`, `mmvu`, `videoholmes`, `longvideobench`, `mlvu` |
+| 空间智能 | `vsi`, `mmsi`, `mindcube`, `revsi` |
+| 时序定位 | `temporal_grounding` |
+| 空间定位 | `spatial_grounding` |
+| 跟踪 | `tracking` |
+| 时空定位 | `stvg` |
+| 分割 | `segmentation` |
+
+`datasets.jsonl` 中的规范配置固定了帧采样、分辨率、prompt、parser 和 metric；
+修改这些值即代表不同的评测设置。ReVSI 单独报告，不计入三个 benchmark 的空间
+智能平均分。
+
+## 4. 添加分割后处理
+
+分割推理默认不执行 SAM2 后处理。启用 `--segmentation-run-sam2` 时还需要：
+
+- SAM2 权重（`SEGMENTATION_SAM2_CKPT`）
+- 匹配的 Hydra 配置（`SEGMENTATION_SAM2_CFG`）
+- OneThinker 官方 `seg_post_sam2.py`
+  （`SEGMENTATION_POSTPROCESSOR_PATH`）
+- `sam2` Python 包
+
+OraRL 会在启动前检查三个路径。
+
+## 5. 保存可报告的输出
+
+每个任务评测器先写入原生 summary，随后 `orarl-eval` 生成指定的聚合 JSON，
+其中包含请求、完成和缺失的任务、返回码以及官方 metric。输出按论文任务族组织在
+`outputs/<model>/` 下。
+
+正式报告结果时请保留：
+
+- OraRL 源码版本
+- 模型和数据版本
+- 完整命令与聚合 summary
+- 软件版本和加速卡类型/数量
+- 所有非默认配置或命令行覆盖项
+
+评测器映射、视频解码后端和 checkpoint 格式说明见
+[`../eval/README.md`](../eval/README.md)。

code/docs/training.md

@@ -0,0 +1,162 @@
+<p align="right"><a href="training_zh.md">简体中文</a></p>
+
+# Train with OraRL
+
+This workflow turns licensed source records into an auditable GRPO or OraRL
+run. Complete the
+[pinned environment installation](environment.md#install-the-pinned-stack)
+first.
+
+`orarl-train` is the public launch boundary: it validates paths and overrides,
+resolves one of the released recipes, and then starts the trainer bundled with
+this checkout. No second runtime repository is required.
+
+## 1. Build the training manifest
+
+The prepared training-data release will be uploaded separately and is not
+included in this Git repository yet. Until it is available, obtain each
+annotation and media source under its upstream license, then copy the example
+manifest:
+
+```bash
+cp configs/data_sources.example.yaml ./data_sources.local.yaml
+```
+
+Replace every `../local_data` placeholder with a licensed local path. Relative
+paths are resolved from the manifest location. Each source declares its input
+annotations, task, family, quota, media root, and optional license/source-page
+metadata.
+
+The public example reproduces the 100,032-row paper mixture:
+
+| Family | Rows |
+| --- | ---: |
+| Temporal grounding | 20,096 |
+| Tracking | 13,952 |
+| Segmentation | 12,032 |
+| Spatial grounding | 7,040 |
+| Spatial-temporal grounding | 9,536 |
+| Video QA | 20,288 |
+| Spatial intelligence | 17,088 |
+
+Build the deterministic train/canary split:
+
+```bash
+orarl-prepare \
+  --config ./data_sources.local.yaml \
+  --output ./prepared/train.jsonl \
+  --require-media
+```
+
+This writes:
+
+```text
+prepared/
+├── train.jsonl
+├── train.canary.jsonl
+└── train.manifest.json
+```
+
+The builder validates local media, normalizes task records, enforces source
+quotas and per-media caps, removes duplicate prompt identities, excludes
+supplied benchmark identities, and keeps train/canary media disjoint. The audit
+manifest records counts, shortfalls, source metadata, and SHA-256 checksums.
+
+## 2. Choose GRPO or OraRL
+
+| Recipe | Model scale | Method |
+| --- | --- | --- |
+| `grpo_4b.yaml` | 4B | GRPO baseline |
+| `grpo_9b.yaml` | 9B | GRPO baseline |
+| `orarl_4b.yaml` | 4B | OraRL |
+| `orarl_9b.yaml` | 9B | OraRL |
+
+The paper defaults use 64 prompts per rollout/update batch and eight policy
+samples per prompt. The 100,032-row mixture therefore runs for 1,563 steps in
+one epoch.
+
+## 3. Preview, then launch
+
+Set paths to a compatible local base model and the prepared data:
+
+```bash
+MODEL_DIR=/path/to/local/base-model
+OUTPUT_DIR="$PWD/runs/orarl-4b"
+
+orarl-train \
+  --config orarl_4b.yaml \
+  --model "$MODEL_DIR" \
+  --train-data "$PWD/prepared/train.jsonl" \
+  --val-data "$PWD/prepared/train.canary.jsonl" \
+  --output "$OUTPUT_DIR" \
+  --nodes 1 \
+  --gpus-per-node 8
+```
+
+The command is a dry run by default. Inspect the resolved invocation, then add
+`--run` to start training. Use `--set KEY=VALUE` for an explicit config
+override; retain all overrides with the run artifacts.
+
+For a one-update smoke test:
+
+```bash
+orarl-train \
+  --config orarl_4b.yaml \
+  --model "$MODEL_DIR" \
+  --train-data "$PWD/prepared/train.jsonl" \
+  --val-data "$PWD/prepared/train.canary.jsonl" \
+  --output "$OUTPUT_DIR" \
+  --nodes 1 \
+  --gpus-per-node 8 \
+  --set trainer.max_steps=1 \
+  --run
+```
+
+Repeat with `grpo_4b.yaml` and a different output directory to validate the
+baseline. Use the matching model and recipe for 9B runs.
+
+## 4. Scale across nodes
+
+All nodes must see the same source, model, data, and output paths:
+
+```bash
+HOSTS=node-a,node-b \
+  bash scripts/launch_multinode.sh \
+  --gpus-per-node 8 \
+  -- \
+  --config "$PWD/configs/orarl_4b.yaml" \
+  --model "$MODEL_DIR" \
+  --train-data "$PWD/prepared/train.jsonl" \
+  --val-data "$PWD/prepared/train.canary.jsonl" \
+  --output "$OUTPUT_DIR"
+```
+
+The launcher is also a dry run unless its own `--run` is supplied before the
+`--` separator. SSH host-key checking is strict by default.
+
+## 5. Accept a run
+
+`scripts/smoke_training.sh` runs one GRPO update and one OraRL update with small
+batches and saves a checkpoint for each:
+
+```bash
+bash scripts/smoke_training.sh \
+  --model "$MODEL_DIR" \
+  --train-data "$PWD/prepared/train.jsonl" \
+  --val-data "$PWD/prepared/train.canary.jsonl" \
+  --size 4b \
+  --gpus-per-node 8
+```
+
+Add `--dry-run` to inspect the resolved commands without allocating GPUs.
+
+Before a full experiment, verify that:
+
+- GRPO and OraRL each complete one update with finite rewards, losses, gradient
+  norms, and selection metrics.
+- A checkpoint can be saved, reloaded, and used for another update.
+- Multi-node runs form the expected Ray cluster and complete one update.
+- The source revision, config, command, data-manifest checksum, environment
+  versions, accelerator type, and all overrides are retained.
+
+Use [Evaluation](evaluation.md) to evaluate an exported checkpoint.

code/docs/training_zh.md

@@ -0,0 +1,151 @@
+<p align="right"><a href="training.md">English</a></p>
+
+# 使用 OraRL 训练
+
+本流程将已获许可的源数据整理为可审计的 GRPO 或 OraRL 训练任务。请先
+[安装固定版本软件栈](environment_zh.md#安装固定版本软件栈)。
+
+`orarl-train` 是公开的训练入口：它会检查路径与覆盖项、解析发布配置，再启动
+本仓库自带的训练器，无需准备第二份运行时源码。
+
+## 1. 构建训练清单
+
+整理后的训练数据将稍后单独上传，目前尚未包含在本 Git 仓库中。在发布完成前，
+请按上游许可证自行获取标注和媒体文件，然后复制示例清单：
+
+```bash
+cp configs/data_sources.example.yaml ./data_sources.local.yaml
+```
+
+将所有 `../local_data` 占位符替换为已获许可的本地路径。相对路径以清单文件所在
+目录为基准。每个数据源需要声明标注文件、任务、任务族、配额和媒体根目录，并可
+记录许可证与来源页面。
+
+公开示例对应论文使用的 100,032 条训练混合：
+
+| 任务族 | 条数 |
+| --- | ---: |
+| 时序定位 | 20,096 |
+| 跟踪 | 13,952 |
+| 分割 | 12,032 |
+| 空间定位 | 7,040 |
+| 时空定位 | 9,536 |
+| 视频问答 | 20,288 |
+| 空间智能 | 17,088 |
+
+构建确定性的训练集和 canary 集：
+
+```bash
+orarl-prepare \
+  --config ./data_sources.local.yaml \
+  --output ./prepared/train.jsonl \
+  --require-media
+```
+
+输出结构为：
+
+```text
+prepared/
+├── train.jsonl
+├── train.canary.jsonl
+└── train.manifest.json
+```
+
+构建器会检查本地媒体、规范化任务记录、执行数据源配额和单媒体上限、去除重复
+prompt、排除给定评测集中的样本，并保证训练集与 canary 集媒体互斥。审计清单会
+记录计数、缺额、来源信息和 SHA-256 校验值。
+
+## 2. 选择 GRPO 或 OraRL
+
+| 配置 | 模型规模 | 方法 |
+| --- | --- | --- |
+| `grpo_4b.yaml` | 4B | GRPO baseline |
+| `grpo_9b.yaml` | 9B | GRPO baseline |
+| `orarl_4b.yaml` | 4B | OraRL |
+| `orarl_9b.yaml` | 9B | OraRL |
+
+论文默认每个 rollout/update batch 使用 64 个 prompt，每个 prompt 采样 8 个策略
+回复，因此 100,032 条混合数据在一个 epoch 中对应 1,563 步。
+
+## 3. 预览并启动
+
+设置兼容的本地基础模型和已准备的数据路径：
+
+```bash
+MODEL_DIR=/path/to/local/base-model
+OUTPUT_DIR="$PWD/runs/orarl-4b"
+
+orarl-train \
+  --config orarl_4b.yaml \
+  --model "$MODEL_DIR" \
+  --train-data "$PWD/prepared/train.jsonl" \
+  --val-data "$PWD/prepared/train.canary.jsonl" \
+  --output "$OUTPUT_DIR" \
+  --nodes 1 \
+  --gpus-per-node 8
+```
+
+命令默认只执行 dry run。检查解析后的调用后，增加 `--run` 开始训练。可通过
+`--set KEY=VALUE` 显式覆盖配置，并应随运行产物保留全部覆盖项。
+
+单步 smoke test：
+
+```bash
+orarl-train \
+  --config orarl_4b.yaml \
+  --model "$MODEL_DIR" \
+  --train-data "$PWD/prepared/train.jsonl" \
+  --val-data "$PWD/prepared/train.canary.jsonl" \
+  --output "$OUTPUT_DIR" \
+  --nodes 1 \
+  --gpus-per-node 8 \
+  --set trainer.max_steps=1 \
+  --run
+```
+
+再使用 `grpo_4b.yaml` 和独立输出目录验证 baseline。9B 任务需使用匹配的模型与
+配置。
+
+## 4. 扩展到多节点
+
+所有节点必须看到相同的源码、模型、数据和输出路径：
+
+```bash
+HOSTS=node-a,node-b \
+  bash scripts/launch_multinode.sh \
+  --gpus-per-node 8 \
+  -- \
+  --config "$PWD/configs/orarl_4b.yaml" \
+  --model "$MODEL_DIR" \
+  --train-data "$PWD/prepared/train.jsonl" \
+  --val-data "$PWD/prepared/train.canary.jsonl" \
+  --output "$OUTPUT_DIR"
+```
+
+启动器同样默认为 dry run；需要在 `--` 分隔符前加入其自身的 `--run`。SSH
+主机密钥检查默认使用严格模式。
+
+## 5. 验收一次运行
+
+`scripts/smoke_training.sh` 会以较小的 batch 分别跑一步 GRPO 和一步 OraRL，
+并各保存一个 checkpoint：
+
+```bash
+bash scripts/smoke_training.sh \
+  --model "$MODEL_DIR" \
+  --train-data "$PWD/prepared/train.jsonl" \
+  --val-data "$PWD/prepared/train.canary.jsonl" \
+  --size 4b \
+  --gpus-per-node 8
+```
+
+加上 `--dry-run` 可在不占用 GPU 的情况下检查解析后的命令。
+
+正式实验前确认：
+
+- GRPO 和 OraRL 均可完成一步更新，reward、loss、梯度范数和选择指标均为有限值。
+- checkpoint 可保存、重新加载并继续完成一步更新。
+- 多节点任务能组成预期的 Ray 集群并完成一步更新。
+- 保存源码版本、配置、命令、数据清单校验值、环境版本、加速卡类型和所有覆盖项。
+
+使用[评测文档](evaluation_zh.md)评估导出的 checkpoint。

code/environment.yml

@@ -0,0 +1,18 @@
+# Base Conda layer. Use scripts/create_conda_env.sh so CUDA wheels and editable
+# packages are installed in the required order.
+name: orarl
+
+channels:
+  - nvidia
+  - conda-forge
+  - defaults
+
+dependencies:
+  - python=3.11
+  - pip
+  - setuptools>=77
+  - wheel
+  - packaging
+  - ninja
+  - ffmpeg=7
+  - cuda-toolkit=12.9

code/eval/README.md

@@ -0,0 +1,72 @@
+# Evaluation runtime
+
+`eval/task/` holds the evaluators that produce every number in the paper. They
+ship with this repository, so a single clone can reproduce the full suite.
+`orarl-eval` is the supported entry point: it resolves the canonical dataset
+layout into per-task environment variables and then runs `eval/task/eval.sh`.
+
+```bash
+orarl-eval \
+  --model /path/to/exported-model \
+  --tasks paper \
+  --dataset OraRL/OraRL-Data \
+  --summary ./evaluation.json \
+  --run
+```
+
+Without `--run` the command prints the resolved plan and exits, which is the
+fastest way to confirm data and model paths before allocating GPUs.
+`eval/task/eval.sh` can also be called directly when you want to bypass the
+canonical dataset resolution and set the per-task variables yourself.
+
+## Task map
+
+| Family | Benchmarks | Entry point |
+| --- | --- | --- |
+| Video QA | VideoMME, VideoMME-v2, MVBench, MMVU, Video-Holmes, LongVideoBench, MLVU | `eval_vllm.py` |
+| Spatial intelligence | VSI-Bench, MindCube, MMSI-Bench, ReVSI | `eval_vllm.py`, `mmsi/eval_mmsi_transformers.py`, `revsi/eval_revsi_vllm.py` |
+| Temporal grounding | TimeLens (Charades, ActivityNet, QVHighlights) | `temporal_grounding/eval_timelens_hf.py` |
+| Spatial grounding | RefCOCO, RefCOCO+, RefCOCOg | `spatial_grounding/eval_refcoco_vllm.py` |
+| Tracking | GOT-10k | `tracking/eval_tracking_vllm.py` |
+| Spatio-temporal grounding | STVG | `spatial_temporal_grounding/eval_stvg_vllm.py` |
+| Segmentation | RefCOCO series, MeViS, ReasonVOS | `segmentation/eval_seg_vllm.py` plus `segmentation/post_sam2.py` |
+
+`eval_prompt.py` is the single source of truth for prompts, and
+`canonical_data.py` adapts the canonical `ORARL_EVAL_*` layout for every
+evaluator. MMSI-Bench runs through Transformers rather than vLLM; the other
+families run through vLLM.
+
+## Assets you must obtain separately
+
+Annotations and media come from the `OraRL/OraRL-Data` dataset repository, and
+model weights come from the released checkpoints. Neither is vendored here.
+
+Segmentation additionally needs three inputs, all supplied by you under their
+upstream licenses:
+
+- SAM2 weights and the matching Hydra config, passed as
+  `SEGMENTATION_SAM2_CKPT` and `SEGMENTATION_SAM2_CFG`.
+- The official OneThinker `seg_post_sam2.py`, passed as
+  `SEGMENTATION_POSTPROCESSOR_PATH`. `segmentation/post_sam2.py` is a thin
+  wrapper that injects paths into it so mask metrics stay identical to the
+  upstream implementation.
+- The `sam2` Python package.
+
+`orarl-eval` refuses to start segmentation with `--segmentation-run-sam2`
+unless all three paths exist.
+
+## Video decoding
+
+Video evaluators pin the decord backend through `FORCE_QWENVL_VIDEO_READER`.
+`eval/task/qwenvl_decord_patch.py` replaces `qwen_vl_utils.fetch_video` so the
+backend is honoured across `qwen_vl_utils` releases that otherwise hard-code
+torchvision. The backend is not cosmetic: switching MeViS and ReasonVOS from
+torchcodec to decord moved MeViS J&F from 56.7 to 60.6. Set
+`SEGMENTATION_VIDEO_READER` to `torchcodec` or `torchvision` only when
+deliberately measuring that difference.
+
+## Checkpoint format
+
+Exported Hugging Face checkpoints run as-is. Pass `--force-merge` when the model
+path is a sharded FSDP actor directory; `eval/task/eval.sh` then merges it with
+`scripts/model_merger.py` before inference.

code/eval/task/canonical_data.py

@@ -0,0 +1,450 @@
+"""Dependency-light compatibility adapter for canonical OraRL evaluation rows."""
+
+from __future__ import annotations
+
+import copy
+import json
+import os
+import re
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+from typing import Any
+
+
+_CANONICAL_FIELDS = {
+    "schema_version",
+    "eval_task",
+    "sample_id",
+    "benchmark",
+    "split",
+    "problem",
+    "answer",
+    "images",
+    "videos",
+    "problem_type",
+    "source",
+}
+_MEDIA_PATH_KEYS = (
+    "path",
+    "video",
+    "image",
+    "video_path",
+    "image_path",
+    "file_name",
+)
+_PATH_KEY_RE = re.compile(
+    r"(?:^|_)(?:artifact|artifacts|cfg|checkpoint|ckpt|dir|directory|file|"
+    r"files|image|images|media|path|paths|prefix|root|subtitle|"
+    r"subtitles|tensor|tensors|video|videos)$"
+)
+_GROUND_TRUTH_KEYS = (
+    "bbox",
+    "bboxes",
+    "boxes",
+    "coordinates",
+    "duration",
+    "end",
+    "frame",
+    "frame_id",
+    "frame_ids",
+    "gt_bbox",
+    "mask",
+    "masks",
+    "negative_points",
+    "normalized_solution",
+    "object",
+    "positive_points",
+    "query",
+    "segment",
+    "segmentation",
+    "span",
+    "spans",
+    "start",
+    "target",
+    "time",
+    "timestamp",
+    "timestamps",
+    "trajectory",
+)
+_ASSET_DIRECTORIES = frozenset({"artifacts", "media"})
+
+
+class CanonicalDataError(ValueError):
+    """Raised when a canonical row cannot be safely adapted."""
+
+
+def is_canonical_row(record: Any) -> bool:
+    """Return whether a mapping uses the canonical evaluation-row contract."""
+
+    return (
+        isinstance(record, Mapping)
+        and record.get("schema_version") == 1
+        and _CANONICAL_FIELDS.issubset(record)
+    )
+
+
+def _configured_root() -> Path | None:
+    value = os.environ.get("ORARL_EVAL_DATA_ROOT", "").strip()
+    if value:
+        return Path(value).expanduser().resolve()
+    manifest = os.environ.get("ORARL_EVAL_DATASETS_JSONL", "").strip()
+    if manifest:
+        return Path(manifest).expanduser().resolve().parent
+    return None
+
+
+def _configured_asset_root() -> Path | None:
+    value = os.environ.get("ORARL_EVAL_ASSET_ROOT", "").strip()
+    return Path(value).expanduser().resolve() if value else None
+
+
+def _infer_root(input_path: Path) -> Path | None:
+    configured = _configured_root()
+    if configured is not None:
+        return configured
+    for candidate in (input_path.parent, *input_path.parents):
+        if (candidate / "datasets.jsonl").is_file():
+            return candidate.resolve()
+    return None
+
+
+def canonical_dataset_record(
+    eval_task: str,
+    split: str,
+    root: str | os.PathLike[str] | None = None,
+) -> dict[str, Any] | None:
+    """Return one canonical ``datasets.jsonl`` record for a task split."""
+
+    repository_root = (
+        Path(root).expanduser().resolve() if root is not None else _configured_root()
+    )
+    if repository_root is None:
+        return None
+    manifest = repository_root / "datasets.jsonl"
+    if not manifest.is_file():
+        return None
+
+    normalized_split = str(split).strip().casefold().replace("-", "_")
+    matches: list[dict[str, Any]] = []
+    try:
+        with manifest.open("r", encoding="utf-8") as handle:
+            for line_number, line in enumerate(handle, start=1):
+                if not line.strip():
+                    continue
+                try:
+                    record = json.loads(line)
+                except json.JSONDecodeError as error:
+                    raise CanonicalDataError(
+                        f"{manifest}:{line_number}: invalid JSON: {error}"
+                    ) from error
+                if not isinstance(record, Mapping):
+                    raise CanonicalDataError(
+                        f"{manifest}:{line_number}: dataset record must be an object"
+                    )
+                record_task = str(record.get("task") or "").casefold()
+                record_split = str(record.get("split") or "").casefold()
+                if record_task == str(eval_task).casefold() and record_split == normalized_split:
+                    matches.append(copy.deepcopy(dict(record)))
+    except OSError as error:
+        raise CanonicalDataError(f"cannot read canonical dataset manifest: {error}") from error
+    if len(matches) > 1:
+        raise CanonicalDataError(
+            f"duplicate canonical dataset profile for {eval_task}/{normalized_split}"
+        )
+    return matches[0] if matches else None
+
+
+def resolve_repository_path(
+    value: str,
+    root: str | os.PathLike[str],
+    asset_root: str | os.PathLike[str] | None = None,
+) -> str:
+    """Resolve one repository-relative canonical path without allowing escape."""
+
+    raw = str(value).strip()
+    if not raw:
+        raise CanonicalDataError("canonical repository path must be nonempty")
+    repository_root = Path(root).expanduser().resolve()
+    configured_asset_root = (
+        Path(asset_root).expanduser().resolve()
+        if asset_root is not None
+        else _configured_asset_root()
+    )
+    candidate = Path(raw).expanduser()
+    selected_root = repository_root
+    if (
+        not candidate.is_absolute()
+        and candidate.parts
+        and candidate.parts[0].casefold() in _ASSET_DIRECTORIES
+        and configured_asset_root is not None
+    ):
+        selected_root = configured_asset_root
+    resolved = (
+        candidate.resolve()
+        if candidate.is_absolute()
+        else (selected_root / candidate).resolve()
+    )
+    allowed_roots = [repository_root]
+    if configured_asset_root is not None:
+        allowed_roots.append(configured_asset_root)
+    try:
+        next(root for root in allowed_roots if resolved.is_relative_to(root))
+    except StopIteration as error:
+        raise CanonicalDataError(
+            f"canonical path resolves outside configured evaluation roots: {raw}"
+        ) from error
+    return os.path.normpath(str(resolved))
+
+
+def repository_relative_output_path(
+    value: str,
+    root: str | os.PathLike[str],
+) -> str:
+    """Return a ``./``-prefixed path when an asset is contained by ``root``."""
+
+    raw = str(value).strip()
+    if not raw or not str(root).strip():
+        return raw
+    candidate = Path(raw).expanduser()
+    if not candidate.is_absolute():
+        return raw
+    repository_root = Path(root).expanduser().resolve()
+    resolved = candidate.resolve()
+    try:
+        relative = resolved.relative_to(repository_root)
+    except ValueError:
+        return raw
+    return f"./{relative.as_posix()}"
+
+
+def _is_path_key(value: object) -> bool:
+    return isinstance(value, str) and bool(_PATH_KEY_RE.search(value.casefold()))
+
+
+def _resolve_media_entry(value: Any, root: Path, asset_root: Path | None) -> Any:
+    if isinstance(value, str):
+        return resolve_repository_path(value, root, asset_root)
+    if not isinstance(value, Mapping):
+        return copy.deepcopy(value)
+    resolved = copy.deepcopy(dict(value))
+    for key in _MEDIA_PATH_KEYS:
+        path = resolved.get(key)
+        if isinstance(path, str) and path.strip():
+            resolved[key] = resolve_repository_path(path, root, asset_root)
+    return resolved
+
+
+def _resolve_declared_paths(
+    value: Any,
+    root: Path,
+    asset_root: Path | None,
+    path_context: bool = False,
+) -> Any:
+    if isinstance(value, str):
+        return (
+            resolve_repository_path(value, root, asset_root)
+            if path_context
+            else value
+        )
+    if isinstance(value, Mapping):
+        return {
+            key: _resolve_declared_paths(
+                item,
+                root,
+                asset_root,
+                path_context=path_context or _is_path_key(key),
+            )
+            for key, item in value.items()
+        }
+    if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
+        return [
+            _resolve_declared_paths(
+                item,
+                root,
+                asset_root,
+                path_context=path_context,
+            )
+            for item in value
+        ]
+    return copy.deepcopy(value)
+
+
+def _media_path(value: Any) -> str:
+    if isinstance(value, str):
+        return value
+    if isinstance(value, Mapping):
+        for key in _MEDIA_PATH_KEYS:
+            candidate = value.get(key)
+            if isinstance(candidate, str) and candidate:
+                return candidate
+    return ""
+
+
+def _declared_path_values(value: Any, path_context: bool = False) -> list[str]:
+    if isinstance(value, str):
+        return [value] if path_context else []
+    if isinstance(value, Mapping):
+        paths: list[str] = []
+        for key, item in value.items():
+            paths.extend(
+                _declared_path_values(
+                    item,
+                    path_context=path_context or _is_path_key(key),
+                )
+            )
+        return paths
+    if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
+        paths = []
+        for item in value:
+            paths.extend(_declared_path_values(item, path_context=path_context))
+        return paths
+    return []
+
+
+def adapt_canonical_row(
+    record: Mapping[str, Any],
+    root: str | os.PathLike[str] | None = None,
+    asset_root: str | os.PathLike[str] | None = None,
+) -> dict[str, Any]:
+    """Resolve canonical paths and add non-destructive legacy aliases."""
+
+    if not is_canonical_row(record):
+        return copy.deepcopy(dict(record))
+    repository_root = (
+        Path(root).expanduser().resolve() if root is not None else _configured_root()
+    )
+    if repository_root is None:
+        raise CanonicalDataError(
+            "canonical rows require ORARL_EVAL_DATA_ROOT or an explicit root"
+        )
+    resolved_asset_root = (
+        Path(asset_root).expanduser().resolve()
+        if asset_root is not None
+        else _configured_asset_root()
+    )
+
+    adapted = copy.deepcopy(dict(record))
+    for field in ("images", "videos", "subtitles"):
+        values = adapted.get(field)
+        if isinstance(values, list):
+            adapted[field] = [
+                _resolve_media_entry(value, repository_root, resolved_asset_root)
+                for value in values
+            ]
+    for field in ("preprocessed", "task_payload", "metadata", "evaluation"):
+        value = adapted.get(field)
+        if isinstance(value, Mapping):
+            adapted[field] = _resolve_declared_paths(
+                value,
+                repository_root,
+                resolved_asset_root,
+            )
+
+    image_paths = [
+        path for value in adapted.get("images", []) if (path := _media_path(value))
+    ]
+    video_paths = [
+        path for value in adapted.get("videos", []) if (path := _media_path(value))
+    ]
+    subtitle_paths = [
+        path
+        for value in adapted.get("subtitles", [])
+        if (path := _media_path(value))
+    ]
+
+    adapted.setdefault("id", adapted["sample_id"])
+    adapted.setdefault("problem_id", adapted["sample_id"])
+    adapted.setdefault("question_id", adapted["sample_id"])
+    adapted.setdefault("question", adapted["problem"])
+    adapted.setdefault("prompt", adapted["problem"])
+    adapted.setdefault("ground_truth", adapted["answer"])
+    adapted.setdefault("solution", adapted["answer"])
+    adapted.setdefault("dataset", adapted["benchmark"])
+    adapted.setdefault("options", copy.deepcopy(adapted.get("choices", [])))
+    adapted.setdefault("image_list", image_paths)
+    if image_paths:
+        adapted.setdefault("image", image_paths[0])
+        adapted.setdefault("image_path", image_paths[0])
+    if video_paths:
+        adapted.setdefault("video", video_paths[0])
+        adapted.setdefault("video_path", video_paths[0])
+    if video_paths or image_paths:
+        adapted.setdefault("path", (video_paths or image_paths)[0])
+    if subtitle_paths:
+        adapted.setdefault("subtitle", subtitle_paths[0])
+        adapted.setdefault("subtitle_path", subtitle_paths[0])
+
+    metadata = adapted.get("metadata")
+    if isinstance(metadata, Mapping):
+        for key, value in metadata.items():
+            adapted.setdefault(str(key), copy.deepcopy(value))
+    payload = adapted.get("task_payload")
+    if isinstance(payload, Mapping):
+        for key, value in payload.items():
+            adapted.setdefault(str(key), copy.deepcopy(value))
+    if isinstance(adapted.get("answer"), Mapping):
+        for key in _GROUND_TRUTH_KEYS:
+            if key in adapted["answer"]:
+                adapted.setdefault(key, copy.deepcopy(adapted["answer"][key]))
+
+    preprocessed = adapted.get("preprocessed")
+    if isinstance(preprocessed, Mapping):
+        for key, value in preprocessed.items():
+            adapted.setdefault(str(key), copy.deepcopy(value))
+        paths = _declared_path_values(preprocessed)
+        if paths:
+            adapted.setdefault("preprocessed_video", paths[0])
+            adapted.setdefault("preprocessed_video_path", paths[0])
+
+    if video_paths:
+        adapted.setdefault("data_type", "video")
+    elif image_paths:
+        adapted.setdefault("data_type", "image")
+    adapted.setdefault("question_type", adapted["problem_type"])
+    adapted.setdefault("task_type", adapted["problem_type"])
+    return adapted
+
+
+def load_json_records(
+    path: str | os.PathLike[str],
+    *,
+    root: str | os.PathLike[str] | None = None,
+    asset_root: str | os.PathLike[str] | None = None,
+) -> list[dict[str, Any]]:
+    """Read JSON/JSONL and adapt only rows that use the canonical contract."""
+
+    input_path = Path(path)
+    text = input_path.read_text(encoding="utf-8")
+    if not text.strip():
+        records: list[Any] = []
+    else:
+        try:
+            payload = json.loads(text)
+        except json.JSONDecodeError:
+            records = [json.loads(line) for line in text.splitlines() if line.strip()]
+        else:
+            records = payload if isinstance(payload, list) else [payload]
+    if not all(isinstance(record, Mapping) for record in records):
+        raise CanonicalDataError(f"{input_path}: every record must be a JSON object")
+
+    repository_root = (
+        Path(root).expanduser().resolve()
+        if root is not None
+        else _infer_root(input_path)
+    )
+    return [
+        adapt_canonical_row(record, repository_root, asset_root)
+        if is_canonical_row(record)
+        else copy.deepcopy(dict(record))
+        for record in records
+    ]
+
+
+__all__ = [
+    "CanonicalDataError",
+    "adapt_canonical_row",
+    "canonical_dataset_record",
+    "is_canonical_row",
+    "load_json_records",
+    "resolve_repository_path",
+]

code/eval/task/eval.sh

@@ -0,0 +1,1018 @@
+#!/bin/bash
+# Thin shell entry for evaluation.
+#
+# Supported TASKS: videomme,videommev2,videommmu,mmvu,mvbench,videoholmes,longvideobench,lvbench,mlvu,vsi,mmsi,mindcube,revsi,spatial_grounding,tracking,stvg,temporal_grounding,segmentation,all
+# Common env overrides:
+#   MODEL_PATH=/path/to/model_or_verl_actor
+#   TASKS=videomme,videommev2,vsi
+#   GPUS=0,1,2,3,4,5,6,7
+#   TP_SIZE=1
+# Prefer explicit CLI args for new runs, which avoids stale env variables:
+#   bash eval/task/eval.sh --model /path/to/model --tasks videomme,videommev2,mvbench --gpus 0,1,2,3,4,5,6,7 --force-merge
+#
+# MODEL must be provided explicitly with --model.
+
+# # —— Video QA / MC ——
+# bash eval/task/eval.sh --model $MODEL --tasks mvbench        --gpus $GPUS --force-merge
+# bash eval/task/eval.sh --model $MODEL --tasks mmvu            --gpus $GPUS --force-merge
+# bash eval/task/eval.sh --model $MODEL --tasks videomme        --gpus $GPUS --force-merge
+# bash eval/task/eval.sh --model $MODEL --tasks videoholmes     --gpus $GPUS --force-merge
+# bash eval/task/eval.sh --model $MODEL --tasks longvideobench  --gpus $GPUS --force-merge
+# bash eval/task/eval.sh --model $MODEL --tasks mlvu            --gpus $GPUS --force-merge
+
+# # —— Spatial intelligence (MC) ——
+# bash eval/task/eval.sh --model $MODEL --tasks vsi             --gpus $GPUS --force-merge
+# bash eval/task/eval.sh --model $MODEL --tasks mmsi            --gpus $GPUS --force-merge
+# bash eval/task/eval.sh --model $MODEL --tasks mindcube        --gpus $GPUS --force-merge
+# bash eval/task/eval.sh --model $MODEL --tasks revsi           --gpus $GPUS --force-merge
+
+# # —— Grounding / tracking / stvg / temporal ——
+# bash eval/task/eval.sh --model $MODEL --tasks spatial_grounding   --gpus $GPUS --force-merge
+# bash eval/task/eval.sh --model $MODEL --tasks tracking            --gpus $GPUS --force-merge
+# bash eval/task/eval.sh --model $MODEL --tasks stvg                --gpus $GPUS --force-merge
+# charades（默认）
+# TIMELENS_DATASETS=charades-timelens TIMELENS_FPS=4 bash eval/task/eval.sh --model $MODEL --tasks temporal_grounding --gpus $GPUS --force-merge
+# TIMELENS_DATASETS=activitynet-timelens TIMELENS_FPS=2 bash eval/task/eval.sh --model $MODEL --tasks temporal_grounding --gpus $GPUS --force-merge
+# TIMELENS_DATASETS=qvhighlights-timelens TIMELENS_FPS=2 bash eval/task/eval.sh --model $MODEL --tasks temporal_grounding --gpus $GPUS --force-merge
+
+
+
+# # —— Segmentation（需要 SAM2 + OraRL 评测环境）——
+# EVAL_CONDA_ENV=orarl SEGMENTATION_RUN_SAM2=true \
+#   bash eval/task/eval.sh --model $MODEL --tasks segmentation --gpus $GPUS --force-merge
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+DEFAULT_GPUS="0,1,2,3,4,5,6,7"
+
+usage() {
+  cat <<'EOF'
+Usage:
+  bash eval/task/eval.sh [options]
+
+Options:
+  --model PATH           Model or checkpoint path.
+  --tasks LIST           Comma-separated task list, e.g. videomme,videommev2,mvbench,mlvu.
+  --gpus LIST            Comma-separated GPU ids, e.g. 0,1,2,3,4,5,6,7.
+  --tp-size N            Tensor parallel size.
+  --shard-mode MODE      Sharding mode passed to vLLM task runner.
+  --video-cache-size N   Video cache size.
+  --videomme-max-new-tokens N
+                         Max new tokens for VideoMME generation.
+  --force-merge          Force merging FSDP actor checkpoints.
+  --skip-merge           Reuse existing merged HF model for actor checkpoints.
+  --merged-model PATH    Explicit merged HF output/input path for actor checkpoints.
+  --base-model PATH      Base model path used when merging actor checkpoints.
+  --env NAME             Conda env to activate. Default: keep current env.
+  --segmentation-run-sam2
+                         Enable SAM2 post-processing for segmentation.
+  --help                 Show this help.
+
+Core launch settings intentionally ignore inherited environment variables to
+avoid stale MODEL_PATH/TASKS/GPUS/etc. Use explicit CLI options instead.
+EOF
+}
+
+# Avoid stale shell environment contaminating evaluation identity. Task-specific
+# tuning envs below are still supported, but core launch identity is CLI/defaults.
+unset MODEL_PATH TASKS GPUS TP_SIZE SHARD_MODE VIDEO_CACHE_SIZE
+unset FORCE_MERGE SKIP_MERGE MERGED_MODEL_PATH BASE_MODEL_PATH EVAL_CONDA_ENV
+
+while (($# > 0)); do
+  case "$1" in
+    --model|--model-path)
+      [[ $# -ge 2 ]] || { echo "ERROR: $1 requires a value" >&2; exit 1; }
+      MODEL_PATH="$2"
+      shift 2
+      ;;
+    --tasks)
+      [[ $# -ge 2 ]] || { echo "ERROR: $1 requires a value" >&2; exit 1; }
+      TASKS="$2"
+      shift 2
+      ;;
+    --gpus)
+      if [[ $# -ge 2 && "$2" != --* ]]; then
+        GPUS="$2"
+        shift 2
+      else
+        GPUS="$DEFAULT_GPUS"
+        shift
+      fi
+      ;;
+    --tp-size)
+      [[ $# -ge 2 ]] || { echo "ERROR: $1 requires a value" >&2; exit 1; }
+      TP_SIZE="$2"
+      shift 2
+      ;;
+    --shard-mode)
+      [[ $# -ge 2 ]] || { echo "ERROR: $1 requires a value" >&2; exit 1; }
+      SHARD_MODE="$2"
+      shift 2
+      ;;
+    --video-cache-size)
+      [[ $# -ge 2 ]] || { echo "ERROR: $1 requires a value" >&2; exit 1; }
+      VIDEO_CACHE_SIZE="$2"
+      shift 2
+      ;;
+    --videomme-max-new-tokens)
+      [[ $# -ge 2 ]] || { echo "ERROR: $1 requires a value" >&2; exit 1; }
+      VIDEOMME_MAX_NEW_TOKENS="$2"
+      shift 2
+      ;;
+    --force-merge)
+      FORCE_MERGE=1
+      shift
+      ;;
+    --skip-merge)
+      SKIP_MERGE=1
+      shift
+      ;;
+    --merged-model|--merged-model-path)
+      [[ $# -ge 2 ]] || { echo "ERROR: $1 requires a value" >&2; exit 1; }
+      MERGED_MODEL_PATH="$2"
+      shift 2
+      ;;
+    --base-model|--base-model-path)
+      [[ $# -ge 2 ]] || { echo "ERROR: $1 requires a value" >&2; exit 1; }
+      BASE_MODEL_PATH="$2"
+      shift 2
+      ;;
+    --env|--conda-env)
+      [[ $# -ge 2 ]] || { echo "ERROR: $1 requires a value" >&2; exit 1; }
+      EVAL_CONDA_ENV="$2"
+      shift 2
+      ;;
+    --segmentation-run-sam2)
+      SEGMENTATION_RUN_SAM2=true
+      shift
+      ;;
+    --help|-h)
+      usage
+      exit 0
+      ;;
+    --)
+      shift
+      break
+      ;;
+    *)
+      echo "ERROR: unknown option '$1'" >&2
+      usage >&2
+      exit 1
+      ;;
+  esac
+done
+
+# Require an explicit model so stale machine-local defaults cannot contaminate
+# benchmark results.
+if [[ -z "${MODEL_PATH:-}" ]]; then
+  echo "ERROR: --model PATH is required." >&2
+  usage >&2
+  exit 2
+fi
+
+MODEL_PATH="$(printf '%s' "$MODEL_PATH" | xargs)"
+
+# Common launch defaults.
+: "${TASKS:=all}"
+: "${GPUS:=$DEFAULT_GPUS}"
+: "${TP_SIZE:=1}"
+: "${SHARD_MODE:=contiguous}"
+: "${VIDEO_CACHE_SIZE:=2}"
+: "${PREFETCH_BATCHES:=1}"
+: "${FORCE_MERGE:=0}"
+: "${SKIP_MERGE:=0}"
+: "${MERGE_LOG_DIR:=${PROJECT_DIR}/logs/qwen3.5/eval/merge}"
+
+if [[ ! "$GPUS" =~ ^[0-9]+(,[0-9]+)*$ ]]; then
+  echo "ERROR: --gpus must be a comma-separated list of GPU ids, got '${GPUS}'" >&2
+  exit 1
+fi
+
+# VideoMME defaults.
+: "${VIDEOMME_VIDEO_BASE:=${PROJECT_DIR}/data/eval/videomme}"
+: "${VIDEOMME_VIDEO_DIR:=${VIDEOMME_VIDEO_BASE}/videos}"
+: "${VIDEOMME_VIDEO_MIN_PIXELS:=4096}"
+: "${VIDEOMME_VIDEO_MAX_PIXELS:=262144}"
+: "${VIDEOMME_VIDEO_TOTAL_PIXELS:=0}"
+: "${VIDEOMME_MAX_FRAMES:=384}"
+: "${VIDEOMME_FPS:=2}"
+: "${VIDEOMME_SETTING:=all-qwen3_vl-sub0-f${VIDEOMME_MAX_FRAMES}-fps${VIDEOMME_FPS}-min${VIDEOMME_VIDEO_MIN_PIXELS}-max${VIDEOMME_VIDEO_MAX_PIXELS}-total${VIDEOMME_VIDEO_TOTAL_PIXELS}-videomme_preprocessed_384f_262k_total0}"
+: "${VIDEOMME_DATA_FILE:=${PROJECT_DIR}/eval/data/valid_data/videomme_preprocessed_384f_262k_total0.jsonl}"
+: "${VIDEOMME_PREPROCESSED_VIDEO_DIR:=${VIDEOMME_VIDEO_BASE}/preprocessed_videos_384f_262k_total0}"
+: "${VIDEOMME_BATCH_SIZE:=1}"
+: "${VIDEOMME_MAX_MODEL_LEN:=65536}"
+: "${VIDEOMME_MAX_NEW_TOKENS:=128}"
+: "${VIDEOMME_MAX_NUM_BATCHED_TOKENS:=65536}"
+: "${VIDEOMME_GPU_MEMORY_UTILIZATION:=0.90}"
+: "${VIDEOMME_MAX_SAMPLES:=0}"
+
+# VideoMME-V2 defaults. Uses the same local schema and evaluator as VideoMME.
+: "${VIDEOMMEV2_VIDEO_BASE:=${PROJECT_DIR}/data/eval/videommev2}"
+: "${VIDEOMMEV2_VIDEO_DIR:=${VIDEOMMEV2_VIDEO_BASE}/videos}"
+: "${VIDEOMMEV2_VIDEO_MIN_PIXELS:=4096}"
+: "${VIDEOMMEV2_VIDEO_MAX_PIXELS:=262144}"
+: "${VIDEOMMEV2_VIDEO_TOTAL_PIXELS:=0}"
+: "${VIDEOMMEV2_MAX_FRAMES:=384}"
+: "${VIDEOMMEV2_FPS:=2}"
+: "${VIDEOMMEV2_SETTING:=all-qwen3_vl-sub0-f${VIDEOMMEV2_MAX_FRAMES}-fps${VIDEOMMEV2_FPS}-min${VIDEOMMEV2_VIDEO_MIN_PIXELS}-max${VIDEOMMEV2_VIDEO_MAX_PIXELS}-total${VIDEOMMEV2_VIDEO_TOTAL_PIXELS}-videommev2_preprocessed_384f_262k_total0}"
+: "${VIDEOMMEV2_DATA_FILE:=${PROJECT_DIR}/eval/data/valid_data/videommev2_preprocessed_384f_262k_total0.jsonl}"
+: "${VIDEOMMEV2_PREPROCESSED_VIDEO_DIR:=${VIDEOMMEV2_VIDEO_BASE}/preprocessed_videos_384f_262k_total0}"
+: "${VIDEOMMEV2_BATCH_SIZE:=1}"
+: "${VIDEOMMEV2_MAX_MODEL_LEN:=65536}"
+: "${VIDEOMMEV2_MAX_NEW_TOKENS:=128}"
+: "${VIDEOMMEV2_MAX_NUM_BATCHED_TOKENS:=65536}"
+: "${VIDEOMMEV2_GPU_MEMORY_UTILIZATION:=0.90}"
+: "${VIDEOMMEV2_PROMPT_MODE:=default}"
+: "${VIDEOMMEV2_ANSWER_FILTER:=}"
+: "${VIDEOMMEV2_MAX_SAMPLES:=0}"
+
+# VideoMMMU defaults.
+: "${VIDEOMMMU_SETTING:=videommmu-f128-fps2-min4096-max262144-total0-with-image}"
+: "${VIDEOMMMU_DATA_FILE:=${PROJECT_DIR}/eval/data/valid_data/videommmu_with_image.json}"
+: "${VIDEOMMMU_VIDEO_ROOT:=${PROJECT_DIR}/data/eval/videommmu}"
+: "${VIDEOMMMU_BATCH_SIZE:=4}"
+: "${VIDEOMMMU_MAX_MODEL_LEN:=65536}"
+: "${VIDEOMMMU_MAX_NEW_TOKENS:=128}"
+: "${VIDEOMMMU_MAX_NUM_BATCHED_TOKENS:=65536}"
+: "${VIDEOMMMU_GPU_MEMORY_UTILIZATION:=0.90}"
+: "${VIDEOMMMU_VIDEO_MIN_PIXELS:=4096}"
+: "${VIDEOMMMU_VIDEO_MAX_PIXELS:=262144}"
+: "${VIDEOMMMU_VIDEO_TOTAL_PIXELS:=0}"
+: "${VIDEOMMMU_MAX_FRAMES:=128}"
+: "${VIDEOMMMU_FPS:=2}"
+# OneThinker-style Adaptation uses subject videos + PNGs extracted from
+# Adaptation/test-00000-of-00001.parquet. Run once before eval:
+#   python3 eval/data/extract_videommmu_images.py
+# prompt_mode=onethink reproduces OneThinker's CoT prompt; pair it with a large
+# VIDEOMMMU_MAX_NEW_TOKENS (OneThinker uses 8192).
+: "${VIDEOMMMU_PROMPT_MODE:=default}"
+# Separate Adaptation image resolution cap (OneThinker uses 1024*32*32=1048576).
+: "${VIDEOMMMU_IMAGE_MAX_PIXELS:=1048576}"
+# Qwen3.5 native benchmarking: thinking on + recommended sampling + long output.
+# To reproduce the official VideoMMMU number, run with:
+#   VIDEOMMMU_PROMPT_MODE=qwen VIDEOMMMU_ENABLE_THINKING=true
+#   VIDEOMMMU_MAX_NEW_TOKENS=32768 VIDEOMMMU_MAX_MODEL_LEN=131072
+#   VIDEOMMMU_TEMPERATURE=1.0 VIDEOMMMU_TOP_P=0.95 VIDEOMMMU_TOP_K=20 VIDEOMMMU_PRESENCE_PENALTY=1.5
+: "${VIDEOMMMU_ENABLE_THINKING:=false}"
+: "${VIDEOMMMU_TEMPERATURE:=0.0}"
+: "${VIDEOMMMU_TOP_P:=1.0}"
+: "${VIDEOMMMU_TOP_K:=-1}"
+: "${VIDEOMMMU_PRESENCE_PENALTY:=0.0}"
+: "${VIDEOMMMU_MIN_P:=0.0}"
+
+# MMVU defaults (multiple-choice subset, raw video).
+# Build the MC JSONL once: python scripts/build_mmvu_mc.py
+: "${MMVU_DATA_FILE:=${PROJECT_DIR}/eval/data/valid_data/mmvu_mc.jsonl}"
+: "${MMVU_VIDEO_ROOT:=${PROJECT_DIR}/data/eval/mmvu}"
+: "${MMVU_BATCH_SIZE:=1}"
+: "${MMVU_MAX_MODEL_LEN:=65536}"
+: "${MMVU_MAX_NEW_TOKENS:=128}"
+: "${MMVU_MAX_NUM_BATCHED_TOKENS:=65536}"
+: "${MMVU_GPU_MEMORY_UTILIZATION:=0.90}"
+: "${MMVU_VIDEO_MIN_PIXELS:=4096}"
+: "${MMVU_VIDEO_MAX_PIXELS:=262144}"
+: "${MMVU_VIDEO_TOTAL_PIXELS:=0}"
+: "${MMVU_MAX_FRAMES:=384}"
+: "${MMVU_FPS:=2}"
+: "${MMVU_PROMPT_MODE:=default}"
+# Qwen3.5 native: MMVU_PROMPT_MODE=qwen MMVU_ENABLE_THINKING=true with large
+# MMVU_MAX_NEW_TOKENS / sampling, mirroring VideoMMMU.
+: "${MMVU_ENABLE_THINKING:=false}"
+: "${MMVU_TEMPERATURE:=0.0}"
+: "${MMVU_TOP_P:=1.0}"
+: "${MMVU_TOP_K:=-1}"
+: "${MMVU_PRESENCE_PENALTY:=0.0}"
+: "${MMVU_MIN_P:=0.0}"
+: "${MMVU_MAX_SAMPLES:=0}"
+: "${MMVU_SETTING:=mmvu-mc-f${MMVU_MAX_FRAMES}-fps${MMVU_FPS}-min${MMVU_VIDEO_MIN_PIXELS}-max${MMVU_VIDEO_MAX_PIXELS}-total${MMVU_VIDEO_TOTAL_PIXELS}-${MMVU_PROMPT_MODE}}"
+
+# MVBench defaults (multi-task short-video multiple-choice).
+: "${MVBENCH_DATA_FILE:=${PROJECT_DIR}/eval/data/valid_data/mvbench.json}"
+: "${MVBENCH_VIDEO_ROOT:=${PROJECT_DIR}/data/eval/mvbench}"
+: "${MVBENCH_BATCH_SIZE:=1}"
+: "${MVBENCH_MAX_MODEL_LEN:=65536}"
+: "${MVBENCH_MAX_NEW_TOKENS:=128}"
+: "${MVBENCH_MAX_NUM_BATCHED_TOKENS:=65536}"
+: "${MVBENCH_GPU_MEMORY_UTILIZATION:=0.90}"
+: "${MVBENCH_VIDEO_MIN_PIXELS:=4096}"
+: "${MVBENCH_VIDEO_MAX_PIXELS:=262144}"
+: "${MVBENCH_VIDEO_TOTAL_PIXELS:=0}"
+: "${MVBENCH_MAX_FRAMES:=384}"
+: "${MVBENCH_FPS:=2}"
+: "${MVBENCH_PROMPT_MODE:=default}"
+: "${MVBENCH_ENABLE_THINKING:=false}"
+: "${MVBENCH_TEMPERATURE:=0.0}"
+: "${MVBENCH_TOP_P:=1.0}"
+: "${MVBENCH_TOP_K:=-1}"
+: "${MVBENCH_PRESENCE_PENALTY:=0.0}"
+: "${MVBENCH_MIN_P:=0.0}"
+: "${MVBENCH_MAX_SAMPLES:=0}"
+: "${MVBENCH_SETTING:=mvbench-f${MVBENCH_MAX_FRAMES}-fps${MVBENCH_FPS}-min${MVBENCH_VIDEO_MIN_PIXELS}-max${MVBENCH_VIDEO_MAX_PIXELS}-total${MVBENCH_VIDEO_TOTAL_PIXELS}-${MVBENCH_PROMPT_MODE}}"
+
+# Video-Holmes defaults (all multiple-choice, raw cropped videos).
+# Download: huggingface-cli download TencentARC/Video-Holmes --repo-type dataset
+# Build: python scripts/build_videoholmes.py
+: "${VIDEOHOLMES_DATA_FILE:=${PROJECT_DIR}/eval/data/valid_data/videoholmes.jsonl}"
+: "${VIDEOHOLMES_VIDEO_ROOT:=${PROJECT_DIR}/data/eval/videoholmes}"
+: "${VIDEOHOLMES_BATCH_SIZE:=1}"
+: "${VIDEOHOLMES_MAX_MODEL_LEN:=65536}"
+: "${VIDEOHOLMES_MAX_NEW_TOKENS:=128}"
+: "${VIDEOHOLMES_MAX_NUM_BATCHED_TOKENS:=65536}"
+: "${VIDEOHOLMES_GPU_MEMORY_UTILIZATION:=0.90}"
+: "${VIDEOHOLMES_VIDEO_MIN_PIXELS:=4096}"
+: "${VIDEOHOLMES_VIDEO_MAX_PIXELS:=262144}"
+: "${VIDEOHOLMES_VIDEO_TOTAL_PIXELS:=0}"
+: "${VIDEOHOLMES_MAX_FRAMES:=384}"
+: "${VIDEOHOLMES_FPS:=2}"
+: "${VIDEOHOLMES_PROMPT_MODE:=default}"
+# Official benchmark prompt is reasoning-style:
+#   VIDEOHOLMES_PROMPT_MODE=holmes VIDEOHOLMES_ENABLE_THINKING=true VIDEOHOLMES_MAX_NEW_TOKENS=1024
+: "${VIDEOHOLMES_ENABLE_THINKING:=false}"
+: "${VIDEOHOLMES_TEMPERATURE:=0.0}"
+: "${VIDEOHOLMES_TOP_P:=1.0}"
+: "${VIDEOHOLMES_TOP_K:=-1}"
+: "${VIDEOHOLMES_PRESENCE_PENALTY:=0.0}"
+: "${VIDEOHOLMES_MIN_P:=0.0}"
+: "${VIDEOHOLMES_MAX_SAMPLES:=0}"
+: "${VIDEOHOLMES_SETTING:=videoholmes-f${VIDEOHOLMES_MAX_FRAMES}-fps${VIDEOHOLMES_FPS}-min${VIDEOHOLMES_VIDEO_MIN_PIXELS}-max${VIDEOHOLMES_VIDEO_MAX_PIXELS}-total${VIDEOHOLMES_VIDEO_TOTAL_PIXELS}-${VIDEOHOLMES_PROMPT_MODE}}"
+
+# LongVideoBench defaults. Main videos/subtitles repo is gated; after getting
+# access, download/extract to LONGVIDEOBENCH_VIDEO_ROOT with videos/ + subtitles/.
+# Build validation JSONL: python scripts/build_longvideobench.py
+: "${LONGVIDEOBENCH_DATA_FILE:=${PROJECT_DIR}/eval/data/valid_data/longvideobench_val.jsonl}"
+: "${LONGVIDEOBENCH_VIDEO_ROOT:=${PROJECT_DIR}/data/eval/longvideobench}"
+: "${LONGVIDEOBENCH_SUBTITLE_ROOT:=$LONGVIDEOBENCH_VIDEO_ROOT}"
+: "${LONGVIDEOBENCH_USE_SUBTITLES:=true}"
+: "${LONGVIDEOBENCH_BATCH_SIZE:=1}"
+: "${LONGVIDEOBENCH_MAX_MODEL_LEN:=65536}"
+: "${LONGVIDEOBENCH_MAX_NEW_TOKENS:=128}"
+: "${LONGVIDEOBENCH_MAX_NUM_BATCHED_TOKENS:=65536}"
+: "${LONGVIDEOBENCH_GPU_MEMORY_UTILIZATION:=0.90}"
+: "${LONGVIDEOBENCH_VIDEO_MIN_PIXELS:=4096}"
+: "${LONGVIDEOBENCH_VIDEO_MAX_PIXELS:=262144}"
+: "${LONGVIDEOBENCH_VIDEO_TOTAL_PIXELS:=0}"
+: "${LONGVIDEOBENCH_MAX_FRAMES:=384}"
+: "${LONGVIDEOBENCH_FPS:=2}"
+: "${LONGVIDEOBENCH_PROMPT_MODE:=default}"
+: "${LONGVIDEOBENCH_ENABLE_THINKING:=false}"
+: "${LONGVIDEOBENCH_TEMPERATURE:=0.0}"
+: "${LONGVIDEOBENCH_TOP_P:=1.0}"
+: "${LONGVIDEOBENCH_TOP_K:=-1}"
+: "${LONGVIDEOBENCH_PRESENCE_PENALTY:=0.0}"
+: "${LONGVIDEOBENCH_MIN_P:=0.0}"
+: "${LONGVIDEOBENCH_MAX_SAMPLES:=0}"
+: "${LONGVIDEOBENCH_SETTING:=longvideobench-val-f${LONGVIDEOBENCH_MAX_FRAMES}-fps${LONGVIDEOBENCH_FPS}-min${LONGVIDEOBENCH_VIDEO_MIN_PIXELS}-max${LONGVIDEOBENCH_VIDEO_MAX_PIXELS}-total${LONGVIDEOBENCH_VIDEO_TOTAL_PIXELS}-${LONGVIDEOBENCH_PROMPT_MODE}}"
+
+# LVBench defaults (long-video multiple-choice; independent from LongVideoBench).
+: "${LVBENCH_DATA_FILE:=${PROJECT_DIR}/eval/data/valid_data/lvbench.json}"
+: "${LVBENCH_VIDEO_ROOT:=${PROJECT_DIR}/data/eval/lvbench}"
+: "${LVBENCH_BATCH_SIZE:=4}"
+: "${LVBENCH_MAX_MODEL_LEN:=65536}"
+: "${LVBENCH_MAX_NEW_TOKENS:=128}"
+: "${LVBENCH_MAX_NUM_BATCHED_TOKENS:=65536}"
+: "${LVBENCH_GPU_MEMORY_UTILIZATION:=0.90}"
+: "${LVBENCH_VIDEO_MIN_PIXELS:=4096}"
+: "${LVBENCH_VIDEO_MAX_PIXELS:=262144}"
+: "${LVBENCH_VIDEO_TOTAL_PIXELS:=0}"
+: "${LVBENCH_MAX_FRAMES:=128}"
+: "${LVBENCH_FPS:=2}"
+: "${LVBENCH_PROMPT_MODE:=default}"
+: "${LVBENCH_ENABLE_THINKING:=false}"
+: "${LVBENCH_TEMPERATURE:=0.0}"
+: "${LVBENCH_TOP_P:=1.0}"
+: "${LVBENCH_TOP_K:=-1}"
+: "${LVBENCH_PRESENCE_PENALTY:=0.0}"
+: "${LVBENCH_MIN_P:=0.0}"
+: "${LVBENCH_SETTING:=lvbench-f${LVBENCH_MAX_FRAMES}-fps${LVBENCH_FPS}-min${LVBENCH_VIDEO_MIN_PIXELS}-max${LVBENCH_VIDEO_MAX_PIXELS}-total${LVBENCH_VIDEO_TOTAL_PIXELS}-${LVBENCH_PROMPT_MODE}}"
+
+# MLVU defaults (dev multiple-choice subset = 7 tasks, M-Avg).
+# Build the MC JSONL once: python scripts/build_mlvu_mc.py
+: "${MLVU_DATA_FILE:=${PROJECT_DIR}/eval/data/valid_data/mlvu_mc.jsonl}"
+: "${MLVU_VIDEO_ROOT:=${PROJECT_DIR}/data/eval/mlvu}"
+: "${MLVU_BATCH_SIZE:=1}"
+: "${MLVU_MAX_MODEL_LEN:=65536}"
+: "${MLVU_MAX_NEW_TOKENS:=128}"
+: "${MLVU_MAX_NUM_BATCHED_TOKENS:=65536}"
+: "${MLVU_GPU_MEMORY_UTILIZATION:=0.90}"
+: "${MLVU_VIDEO_MIN_PIXELS:=4096}"
+: "${MLVU_VIDEO_MAX_PIXELS:=262144}"
+: "${MLVU_VIDEO_TOTAL_PIXELS:=0}"
+: "${MLVU_MAX_FRAMES:=384}"
+: "${MLVU_FPS:=2}"
+: "${MLVU_PROMPT_MODE:=default}"
+: "${MLVU_ENABLE_THINKING:=false}"
+: "${MLVU_TEMPERATURE:=0.0}"
+: "${MLVU_TOP_P:=1.0}"
+: "${MLVU_TOP_K:=-1}"
+: "${MLVU_PRESENCE_PENALTY:=0.0}"
+: "${MLVU_MIN_P:=0.0}"
+: "${MLVU_MAX_SAMPLES:=0}"
+: "${MLVU_SETTING:=mlvu-mc-f${MLVU_MAX_FRAMES}-fps${MLVU_FPS}-min${MLVU_VIDEO_MIN_PIXELS}-max${MLVU_VIDEO_MAX_PIXELS}-total${MLVU_VIDEO_TOTAL_PIXELS}-${MLVU_PROMPT_MODE}}"
+
+# VSI-Bench defaults.
+: "${VSI_SETTING:=video128-16M-video-f128-fps2-min65536-maxnone-total16777216-vsibench_preprocessed_128f_16M}"
+: "${VSI_DATA_FILE:=${PROJECT_DIR}/eval/data/valid_data/vsibench_preprocessed_128f_16M.jsonl}"
+: "${VSI_PREPROCESSED_VIDEO_DIR:=${PROJECT_DIR}/data/eval/vsi/preprocessed_videos}"
+: "${VSI_BATCH_SIZE:=16}"
+: "${VSI_MAX_MODEL_LEN:=32768}"
+: "${VSI_MAX_NEW_TOKENS:=1024}"
+: "${VSI_GPU_MEMORY_UTILIZATION:=0.90}"
+: "${VSI_EXPECTED_SAMPLES:=5130}"
+
+# MMSI-Bench defaults. Transformers is used so every image can be retained;
+# set MMSI_BACKEND=vllm and MMSI_MAX_IMAGES=8 for the legacy path.
+: "${MMSI_BACKEND:=transformers}"
+: "${MMSI_DATA_FILE:=${PROJECT_DIR}/data/eval/mmsi/MMSI_bench.tsv}"
+: "${MMSI_IMAGE_MIN_PIXELS:=4096}"
+: "${MMSI_IMAGE_MAX_PIXELS:=262144}"
+if [[ "${MMSI_BACKEND}" == "transformers" ]]; then
+  : "${MMSI_MAX_IMAGES:=0}"
+else
+  : "${MMSI_MAX_IMAGES:=8}"
+fi
+: "${MMSI_IMAGE_COUNT_TAG:=$([[ "${MMSI_MAX_IMAGES}" -le 0 ]] && echo all || echo "${MMSI_MAX_IMAGES}")}"
+: "${MMSI_SETTING:=${MMSI_BACKEND}-img-min${MMSI_IMAGE_MIN_PIXELS}-max${MMSI_IMAGE_MAX_PIXELS}-n${MMSI_IMAGE_COUNT_TAG}-all-MMSI_bench}"
+: "${MMSI_BATCH_SIZE:=16}"
+: "${MMSI_MAX_MODEL_LEN:=32768}"
+: "${MMSI_MAX_NEW_TOKENS:=1024}"
+: "${MMSI_MAX_NUM_BATCHED_TOKENS:=32768}"
+: "${MMSI_GPU_MEMORY_UTILIZATION:=0.90}"
+: "${MMSI_MAX_SAMPLES:=0}"
+: "${MMSI_CATEGORY:=}"
+: "${MMSI_ENABLE_THINKING:=false}"
+: "${MMSI_ATTN_IMPLEMENTATION:=flash_attention_2}"
+if [[ -n "${MMSI_CATEGORY}" || "${MMSI_MAX_SAMPLES}" != "0" ]]; then
+  : "${MMSI_EXPECTED_SAMPLES:=0}"
+else
+  : "${MMSI_EXPECTED_SAMPLES:=1000}"
+fi
+
+# MindCube-Tiny defaults.
+: "${MINDCUBE_SETTING:=img-min4096-max262144-n4-official-tiny1050}"
+: "${MINDCUBE_DATA_FILE:=${PROJECT_DIR}/data/eval/mindcube_official/combined-00000-of-00001.parquet}"
+: "${MINDCUBE_MAX_IMAGES:=4}"
+: "${MINDCUBE_EXPECTED_SAMPLES:=1050}"
+: "${MINDCUBE_HF_CACHE_DIR:=}"
+: "${MINDCUBE_BATCH_SIZE:=16}"
+: "${MINDCUBE_MAX_MODEL_LEN:=32768}"
+: "${MINDCUBE_MAX_NEW_TOKENS:=1024}"
+: "${MINDCUBE_MAX_NUM_BATCHED_TOKENS:=32768}"
+: "${MINDCUBE_GPU_MEMORY_UTILIZATION:=0.90}"
+
+# ReVSI defaults. The all-frame/native-frame profile reproduces the 58.2 result
+# reported for Video-ORA-9B; ReVSI remains excluded from the three-benchmark
+# spatial-intelligence average.
+: "${REVSI_DATA_FILE:=${PROJECT_DIR}/data/eval/revsi/all_frame/test-00000-of-00001.parquet}"
+: "${REVSI_VIDEO_ROOT:=${PROJECT_DIR}/data/eval/revsi}"
+: "${REVSI_FRAME_BUDGET:=all}"
+: "${REVSI_MAX_FRAMES:=128}"
+: "${REVSI_EXACT_NFRAMES:=true}"
+: "${REVSI_FPS:=2}"
+: "${REVSI_VIDEO_MIN_PIXELS:=65536}"
+: "${REVSI_VIDEO_MAX_PIXELS:=}"
+: "${REVSI_VIDEO_TOTAL_PIXELS:=16777216}"
+: "${REVSI_BATCH_SIZE:=16}"
+: "${REVSI_MAX_MODEL_LEN:=32768}"
+: "${REVSI_MAX_NEW_TOKENS:=64}"
+: "${REVSI_GPU_MEMORY_UTILIZATION:=0.90}"
+: "${REVSI_MAX_SAMPLES:=0}"
+: "${REVSI_EXPECTED_SAMPLES:=6808}"
+: "${REVSI_TASK_FILTER:=}"
+: "${REVSI_ENABLE_THINKING:=false}"
+: "${REVSI_SETTING:=native-all-f${REVSI_MAX_FRAMES}-exact${REVSI_EXACT_NFRAMES}-fps${REVSI_FPS}-min${REVSI_VIDEO_MIN_PIXELS}-max${REVSI_VIDEO_MAX_PIXELS:-none}-total${REVSI_VIDEO_TOTAL_PIXELS}}"
+
+# Spatial grounding defaults, aligned to data/joint/sft_joint_all.jsonl.
+: "${SPATIAL_GROUNDING_DATASETS:=refcoco-val,refcoco-testA,refcoco-testB,refcoco+-val,refcoco+-testA,refcoco+-testB,refcocog-val,refcocog-test}"
+: "${SPATIAL_GROUNDING_BENCH_DIR:=${PROJECT_DIR}/eval/task/spatial_grounding}"
+: "${SPATIAL_GROUNDING_IMAGE_ROOT:=${PROJECT_DIR}/data/eval/spatial_grounding/images}"
+: "${SPATIAL_GROUNDING_PROCESSOR_PATH:=$MODEL_PATH}"
+: "${SPATIAL_GROUNDING_PROMPT_STYLE:=qwen_native}"
+: "${SPATIAL_GROUNDING_COORD_SYSTEM:=norm1000}"
+: "${SPATIAL_GROUNDING_BBOX_SELECT:=first}"
+: "${SPATIAL_GROUNDING_MIN_TOKENS:=64}"
+: "${SPATIAL_GROUNDING_TOTAL_TOKENS:=1024}"
+: "${SPATIAL_GROUNDING_BATCH_SIZE:=64}"
+: "${SPATIAL_GROUNDING_MAX_MODEL_LEN:=32768}"
+: "${SPATIAL_GROUNDING_MAX_NEW_TOKENS:=1024}"
+: "${SPATIAL_GROUNDING_MAX_NUM_BATCHED_TOKENS:=32768}"
+: "${SPATIAL_GROUNDING_GPU_MEMORY_UTILIZATION:=0.85}"
+: "${SPATIAL_GROUNDING_ENABLE_THINKING:=false}"
+: "${SPATIAL_GROUNDING_MAX_SAMPLES:=0}"
+: "${SPATIAL_GROUNDING_SETTING:=spatial-grounding-${SPATIAL_GROUNDING_DATASETS//,/_}-${SPATIAL_GROUNDING_PROMPT_STYLE}-min${SPATIAL_GROUNDING_MIN_TOKENS}-total${SPATIAL_GROUNDING_TOTAL_TOKENS}-new${SPATIAL_GROUNDING_MAX_NEW_TOKENS}}"
+
+# Tracking defaults, aligned to data/joint/sft_joint_all.jsonl tracking videos.
+: "${TRACKING_DATASETS:=eval_got10k}"
+: "${TRACKING_BENCH_DIR:=${PROJECT_DIR}/data/eval/tracking}"
+: "${TRACKING_BASE_PREFIX:=$TRACKING_BENCH_DIR}"
+: "${TRACKING_PROCESSOR_PATH:=$MODEL_PATH}"
+: "${TRACKING_VIDEO_MIN_PIXELS:=4096}"
+: "${TRACKING_VIDEO_MAX_PIXELS:=786432}"
+: "${TRACKING_VIDEO_TOTAL_PIXELS:=8388608}"
+: "${TRACKING_MAX_FRAMES:=32}"
+: "${TRACKING_FPS:=1}"
+: "${TRACKING_MAX_MODEL_LEN:=32768}"
+: "${TRACKING_MAX_NEW_TOKENS:=8192}"
+: "${TRACKING_MAX_NUM_BATCHED_TOKENS:=32768}"
+: "${TRACKING_BATCH_SIZE:=64}"
+: "${TRACKING_GPU_MEMORY_UTILIZATION:=0.95}"
+: "${TRACKING_ENABLE_THINKING:=false}"
+: "${TRACKING_PROMPT_MODE:=default}"
+: "${TRACKING_CHUNKED_REPROMPT:=0}"
+: "${TRACKING_REPROMPT_USE_GT_FIRST_BOX:=0}"
+: "${TRACKING_MAX_SAMPLES:=0}"
+: "${TRACKING_SETTING:=tracking-${TRACKING_DATASETS//,/_}-f${TRACKING_MAX_FRAMES}-fps${TRACKING_FPS}-min${TRACKING_VIDEO_MIN_PIXELS}-max${TRACKING_VIDEO_MAX_PIXELS}-total${TRACKING_VIDEO_TOTAL_PIXELS}-new${TRACKING_MAX_NEW_TOKENS}}"
+
+# STVG defaults, aligned to data/joint/sft_joint_all.jsonl stvg videos.
+: "${STVG_DATASETS:=eval_stvg}"
+: "${STVG_BENCH_DIR:=${PROJECT_DIR}/data/eval/stvg}"
+: "${STVG_BASE_PREFIX:=$STVG_BENCH_DIR}"
+: "${STVG_PROCESSOR_PATH:=$MODEL_PATH}"
+: "${STVG_VIDEO_MIN_PIXELS:=65536}"
+: "${STVG_VIDEO_MAX_PIXELS:=393216}"
+: "${STVG_VIDEO_TOTAL_PIXELS:=10485760}"
+: "${STVG_MAX_FRAMES:=128}"
+: "${STVG_FPS:=2}"
+: "${STVG_MAX_MODEL_LEN:=65536}"
+: "${STVG_MAX_NEW_TOKENS:=2048}"
+: "${STVG_MAX_NUM_BATCHED_TOKENS:=65536}"
+: "${STVG_BATCH_SIZE:=64}"
+: "${STVG_GPU_MEMORY_UTILIZATION:=0.85}"
+: "${STVG_ENABLE_THINKING:=false}"
+: "${STVG_PROMPT_MODE:=train_stvg}"
+: "${STVG_MAX_SAMPLES:=0}"
+: "${STVG_SETTING:=stvg-${STVG_DATASETS//,/_}-f${STVG_MAX_FRAMES}-fps${STVG_FPS}-min${STVG_VIDEO_MIN_PIXELS}-max${STVG_VIDEO_MAX_PIXELS}-total${STVG_VIDEO_TOTAL_PIXELS}-new${STVG_MAX_NEW_TOKENS}-${STVG_PROMPT_MODE}}"
+
+# Segmentation defaults.
+# This task is dispatched to eval/task/segmentation/run_eval_vllm.sh,
+# which performs vLLM inference plus optional SAM2 post-processing.
+: "${SEGMENTATION_DATASETS:=eval_seg_refcoco,eval_seg_refcocop,eval_seg_refcocog,eval_seg_mevis,eval_seg_reasonvos}"
+: "${SEGMENTATION_BENCH_DIR:=${PROJECT_DIR}/data/eval/segmentation}"
+: "${SEGMENTATION_DATA_ROOT:=$SEGMENTATION_BENCH_DIR}"
+: "${SEGMENTATION_PROCESSOR_PATH:=$MODEL_PATH}"
+: "${SEGMENTATION_DATA_TYPE:=all}"
+# train_seg mirrors the prompt used in joint SFT training data
+# (data/train/seg_image_nothink_evalaware.jsonl, seg_video_nothink.jsonl).
+: "${SEGMENTATION_PROMPT_MODE:=train_seg}"
+: "${SEGMENTATION_ENABLE_THINKING:=false}"
+: "${SEGMENTATION_BATCH_SIZE:=16}"
+: "${SEGMENTATION_MAX_NEW_TOKENS:=1024}"
+: "${SEGMENTATION_MAX_MODEL_LEN:=32768}"
+: "${SEGMENTATION_MAX_PIXELS_IMAGE:=1048576}"
+: "${SEGMENTATION_MIN_PIXELS_IMAGE:=4096}"
+: "${SEGMENTATION_VIDEO_MAX_PIXELS:=262144}"
+: "${SEGMENTATION_VIDEO_MIN_PIXELS:=4096}"
+: "${SEGMENTATION_VIDEO_TOTAL_PIXELS:=16777216}"
+: "${SEGMENTATION_MAX_FRAMES:=128}"
+: "${SEGMENTATION_FPS:=2}"
+: "${SEGMENTATION_VIDEO_READER:=decord}"
+: "${SEGMENTATION_GPU_MEM_UTIL:=0.85}"
+: "${SEGMENTATION_SEED:=42}"
+: "${SEGMENTATION_MAX_SAMPLES:=}"
+: "${SEGMENTATION_RUN_SAM2:=false}"
+: "${SEGMENTATION_SAM2_CKPT:=${PROJECT_DIR}/models/sam2/sam2.1_hiera_large.pt}"
+: "${SEGMENTATION_SAM2_CFG:=configs/sam2.1/sam2.1_hiera_l.yaml}"
+: "${SEGMENTATION_POSTPROCESSOR_PATH:=${PROJECT_DIR}/third_party/OneThinker/Evaluation/Eval/seg_post_sam2.py}"
+# SAM2 post-processing parallelism. world_size = NUM_GPUS * WORKERS_PER_GPU.
+# With a large SAM2_EPOCH_SIZE (single epoch) + maxtasksperchild=1, EACH worker
+# loads the SAM2 model exactly ONCE at startup and then streams its whole slice
+# (no repeated reloads). So WORKERS_PER_GPU only trades GPU utilization vs total
+# process count: too high (32 -> 256 procs) exhausts CPU/RAM/ffmpeg/handles and
+# aborts; too low (1) underutilizes the GPU. 4 per GPU is a balance (8 GPUs -> 32
+# persistent workers, 32 one-time loads); bump to 6-8 if GPU is still not full.
+# Leave NUM_GPUS empty to auto-use all visible GPUs.
+: "${SEGMENTATION_SAM2_NUM_GPUS:=}"
+: "${SEGMENTATION_SAM2_WORKERS_PER_GPU:=8}"
+: "${SEGMENTATION_SETTING:=segmentation-${SEGMENTATION_DATASETS//,/_}-${SEGMENTATION_PROMPT_MODE}-f${SEGMENTATION_MAX_FRAMES}-fps${SEGMENTATION_FPS}-min${SEGMENTATION_VIDEO_MIN_PIXELS}-max${SEGMENTATION_VIDEO_MAX_PIXELS}-total${SEGMENTATION_VIDEO_TOTAL_PIXELS}-new${SEGMENTATION_MAX_NEW_TOKENS}}"
+
+# TimeLens-Bench temporal grounding defaults.
+# This task uses HuggingFace transformers, not vLLM.
+: "${TIMELENS_DATASETS:=charades-timelens}"
+: "${TIMELENS_BENCH_DIR:=${PROJECT_DIR}/data/eval/temporal_grounding}"
+: "${TIMELENS_ENABLE_THINKING:=false}"
+: "${TIMELENS_MIN_TOKENS:=1}"
+: "${TIMELENS_TOTAL_TOKENS:=128000}"
+: "${TIMELENS_MAX_PIXELS:=409600}"
+: "${TIMELENS_MAX_FRAMES:=2048}"
+: "${TIMELENS_FPS:=4}"
+: "${TIMELENS_MAX_NEW_TOKENS:=128}"
+: "${TIMELENS_REPETITION_PENALTY:=1.0}"
+: "${TIMELENS_STOP_AFTER_ANSWER:=true}"
+: "${TIMELENS_PROMPT_MODE:=same}"
+: "${TIMELENS_NUM_WORKERS:=2}"
+: "${TIMELENS_MAX_SAMPLES:=0}"
+
+export MODEL_PATH TASKS GPUS TP_SIZE SHARD_MODE VIDEO_CACHE_SIZE PREFETCH_BATCHES
+export FORCE_MERGE SKIP_MERGE MERGE_LOG_DIR
+export VIDEOMME_SETTING VIDEOMME_DATA_FILE VIDEOMME_VIDEO_BASE VIDEOMME_VIDEO_DIR
+export VIDEOMME_PREPROCESSED_VIDEO_DIR VIDEOMME_VIDEO_MIN_PIXELS VIDEOMME_VIDEO_MAX_PIXELS
+export VIDEOMME_VIDEO_TOTAL_PIXELS VIDEOMME_MAX_FRAMES VIDEOMME_FPS
+export VIDEOMME_BATCH_SIZE VIDEOMME_MAX_MODEL_LEN VIDEOMME_MAX_NEW_TOKENS
+export VIDEOMME_MAX_NUM_BATCHED_TOKENS VIDEOMME_GPU_MEMORY_UTILIZATION
+export VIDEOMME_MAX_SAMPLES
+export VIDEOMMEV2_SETTING VIDEOMMEV2_DATA_FILE VIDEOMMEV2_VIDEO_BASE VIDEOMMEV2_VIDEO_DIR
+export VIDEOMMEV2_PREPROCESSED_VIDEO_DIR VIDEOMMEV2_VIDEO_MIN_PIXELS VIDEOMMEV2_VIDEO_MAX_PIXELS
+export VIDEOMMEV2_VIDEO_TOTAL_PIXELS VIDEOMMEV2_MAX_FRAMES VIDEOMMEV2_FPS
+export VIDEOMMEV2_BATCH_SIZE VIDEOMMEV2_MAX_MODEL_LEN VIDEOMMEV2_MAX_NEW_TOKENS
+export VIDEOMMEV2_MAX_NUM_BATCHED_TOKENS VIDEOMMEV2_GPU_MEMORY_UTILIZATION
+export VIDEOMMEV2_PROMPT_MODE VIDEOMMEV2_ANSWER_FILTER VIDEOMMEV2_MAX_SAMPLES
+export VIDEOMMMU_SETTING VIDEOMMMU_DATA_FILE VIDEOMMMU_VIDEO_ROOT
+export VIDEOMMMU_BATCH_SIZE VIDEOMMMU_MAX_MODEL_LEN VIDEOMMMU_MAX_NEW_TOKENS
+export VIDEOMMMU_MAX_NUM_BATCHED_TOKENS VIDEOMMMU_GPU_MEMORY_UTILIZATION
+export VIDEOMMMU_VIDEO_MIN_PIXELS VIDEOMMMU_VIDEO_MAX_PIXELS VIDEOMMMU_VIDEO_TOTAL_PIXELS
+export VIDEOMMMU_MAX_FRAMES VIDEOMMMU_FPS VIDEOMMMU_PROMPT_MODE VIDEOMMMU_IMAGE_MAX_PIXELS
+export VIDEOMMMU_ENABLE_THINKING VIDEOMMMU_TEMPERATURE VIDEOMMMU_TOP_P VIDEOMMMU_TOP_K
+export VIDEOMMMU_PRESENCE_PENALTY VIDEOMMMU_MIN_P
+export MMVU_SETTING MMVU_DATA_FILE MMVU_VIDEO_ROOT MMVU_BATCH_SIZE
+export MMVU_MAX_MODEL_LEN MMVU_MAX_NEW_TOKENS MMVU_MAX_NUM_BATCHED_TOKENS
+export MMVU_GPU_MEMORY_UTILIZATION MMVU_VIDEO_MIN_PIXELS MMVU_VIDEO_MAX_PIXELS
+export MMVU_VIDEO_TOTAL_PIXELS MMVU_MAX_FRAMES MMVU_FPS MMVU_PROMPT_MODE
+export MMVU_ENABLE_THINKING MMVU_TEMPERATURE MMVU_TOP_P MMVU_TOP_K
+export MMVU_PRESENCE_PENALTY MMVU_MIN_P MMVU_MAX_SAMPLES
+export MVBENCH_SETTING MVBENCH_DATA_FILE MVBENCH_VIDEO_ROOT MVBENCH_BATCH_SIZE
+export MVBENCH_MAX_MODEL_LEN MVBENCH_MAX_NEW_TOKENS MVBENCH_MAX_NUM_BATCHED_TOKENS
+export MVBENCH_GPU_MEMORY_UTILIZATION MVBENCH_VIDEO_MIN_PIXELS MVBENCH_VIDEO_MAX_PIXELS
+export MVBENCH_VIDEO_TOTAL_PIXELS MVBENCH_MAX_FRAMES MVBENCH_FPS MVBENCH_PROMPT_MODE
+export MVBENCH_ENABLE_THINKING MVBENCH_TEMPERATURE MVBENCH_TOP_P MVBENCH_TOP_K
+export MVBENCH_PRESENCE_PENALTY MVBENCH_MIN_P MVBENCH_MAX_SAMPLES
+export VIDEOHOLMES_SETTING VIDEOHOLMES_DATA_FILE VIDEOHOLMES_VIDEO_ROOT VIDEOHOLMES_BATCH_SIZE
+export VIDEOHOLMES_MAX_MODEL_LEN VIDEOHOLMES_MAX_NEW_TOKENS VIDEOHOLMES_MAX_NUM_BATCHED_TOKENS
+export VIDEOHOLMES_GPU_MEMORY_UTILIZATION VIDEOHOLMES_VIDEO_MIN_PIXELS VIDEOHOLMES_VIDEO_MAX_PIXELS
+export VIDEOHOLMES_VIDEO_TOTAL_PIXELS VIDEOHOLMES_MAX_FRAMES VIDEOHOLMES_FPS VIDEOHOLMES_PROMPT_MODE
+export VIDEOHOLMES_ENABLE_THINKING VIDEOHOLMES_TEMPERATURE VIDEOHOLMES_TOP_P VIDEOHOLMES_TOP_K
+export VIDEOHOLMES_PRESENCE_PENALTY VIDEOHOLMES_MIN_P VIDEOHOLMES_MAX_SAMPLES
+export LONGVIDEOBENCH_SETTING LONGVIDEOBENCH_DATA_FILE LONGVIDEOBENCH_VIDEO_ROOT
+export LONGVIDEOBENCH_SUBTITLE_ROOT LONGVIDEOBENCH_USE_SUBTITLES LONGVIDEOBENCH_BATCH_SIZE
+export LONGVIDEOBENCH_MAX_MODEL_LEN LONGVIDEOBENCH_MAX_NEW_TOKENS LONGVIDEOBENCH_MAX_NUM_BATCHED_TOKENS
+export LONGVIDEOBENCH_GPU_MEMORY_UTILIZATION LONGVIDEOBENCH_VIDEO_MIN_PIXELS LONGVIDEOBENCH_VIDEO_MAX_PIXELS
+export LONGVIDEOBENCH_VIDEO_TOTAL_PIXELS LONGVIDEOBENCH_MAX_FRAMES LONGVIDEOBENCH_FPS LONGVIDEOBENCH_PROMPT_MODE
+export LONGVIDEOBENCH_ENABLE_THINKING LONGVIDEOBENCH_TEMPERATURE LONGVIDEOBENCH_TOP_P LONGVIDEOBENCH_TOP_K
+export LONGVIDEOBENCH_PRESENCE_PENALTY LONGVIDEOBENCH_MIN_P LONGVIDEOBENCH_MAX_SAMPLES
+export LVBENCH_SETTING LVBENCH_DATA_FILE LVBENCH_VIDEO_ROOT LVBENCH_BATCH_SIZE
+export LVBENCH_MAX_MODEL_LEN LVBENCH_MAX_NEW_TOKENS LVBENCH_MAX_NUM_BATCHED_TOKENS
+export LVBENCH_GPU_MEMORY_UTILIZATION LVBENCH_VIDEO_MIN_PIXELS LVBENCH_VIDEO_MAX_PIXELS
+export LVBENCH_VIDEO_TOTAL_PIXELS LVBENCH_MAX_FRAMES LVBENCH_FPS LVBENCH_PROMPT_MODE
+export LVBENCH_ENABLE_THINKING LVBENCH_TEMPERATURE LVBENCH_TOP_P LVBENCH_TOP_K
+export LVBENCH_PRESENCE_PENALTY LVBENCH_MIN_P
+export MLVU_SETTING MLVU_DATA_FILE MLVU_VIDEO_ROOT MLVU_BATCH_SIZE
+export MLVU_MAX_MODEL_LEN MLVU_MAX_NEW_TOKENS MLVU_MAX_NUM_BATCHED_TOKENS
+export MLVU_GPU_MEMORY_UTILIZATION MLVU_VIDEO_MIN_PIXELS MLVU_VIDEO_MAX_PIXELS
+export MLVU_VIDEO_TOTAL_PIXELS MLVU_MAX_FRAMES MLVU_FPS MLVU_PROMPT_MODE
+export MLVU_ENABLE_THINKING MLVU_TEMPERATURE MLVU_TOP_P MLVU_TOP_K
+export MLVU_PRESENCE_PENALTY MLVU_MIN_P MLVU_MAX_SAMPLES
+export VSI_SETTING VSI_DATA_FILE VSI_PREPROCESSED_VIDEO_DIR
+export VSI_BATCH_SIZE VSI_MAX_MODEL_LEN VSI_MAX_NEW_TOKENS VSI_GPU_MEMORY_UTILIZATION
+export VSI_EXPECTED_SAMPLES
+export MMSI_BACKEND MMSI_SETTING MMSI_DATA_FILE MMSI_MAX_IMAGES MMSI_BATCH_SIZE
+export MMSI_IMAGE_MIN_PIXELS MMSI_IMAGE_MAX_PIXELS MMSI_IMAGE_COUNT_TAG
+export MMSI_MAX_MODEL_LEN MMSI_MAX_NEW_TOKENS MMSI_MAX_NUM_BATCHED_TOKENS
+export MMSI_GPU_MEMORY_UTILIZATION MMSI_MAX_SAMPLES MMSI_CATEGORY
+export MMSI_ENABLE_THINKING MMSI_ATTN_IMPLEMENTATION MMSI_EXPECTED_SAMPLES
+export MINDCUBE_SETTING MINDCUBE_DATA_FILE MINDCUBE_MAX_IMAGES MINDCUBE_BATCH_SIZE
+export MINDCUBE_MAX_MODEL_LEN MINDCUBE_MAX_NEW_TOKENS MINDCUBE_MAX_NUM_BATCHED_TOKENS
+export MINDCUBE_GPU_MEMORY_UTILIZATION MINDCUBE_EXPECTED_SAMPLES MINDCUBE_HF_CACHE_DIR
+export REVSI_SETTING REVSI_DATA_FILE REVSI_VIDEO_ROOT REVSI_FRAME_BUDGET
+export REVSI_MAX_FRAMES REVSI_EXACT_NFRAMES REVSI_FPS
+export REVSI_VIDEO_MIN_PIXELS REVSI_VIDEO_MAX_PIXELS REVSI_VIDEO_TOTAL_PIXELS
+export REVSI_BATCH_SIZE REVSI_MAX_MODEL_LEN REVSI_MAX_NEW_TOKENS
+export REVSI_GPU_MEMORY_UTILIZATION REVSI_MAX_SAMPLES REVSI_EXPECTED_SAMPLES
+export REVSI_TASK_FILTER REVSI_ENABLE_THINKING
+export SPATIAL_GROUNDING_SETTING SPATIAL_GROUNDING_DATASETS SPATIAL_GROUNDING_BENCH_DIR
+export SPATIAL_GROUNDING_IMAGE_ROOT SPATIAL_GROUNDING_PROCESSOR_PATH SPATIAL_GROUNDING_PROMPT_STYLE
+export SPATIAL_GROUNDING_COORD_SYSTEM SPATIAL_GROUNDING_BBOX_SELECT
+export SPATIAL_GROUNDING_MIN_TOKENS SPATIAL_GROUNDING_TOTAL_TOKENS
+export SPATIAL_GROUNDING_BATCH_SIZE SPATIAL_GROUNDING_MAX_MODEL_LEN
+export SPATIAL_GROUNDING_MAX_NEW_TOKENS SPATIAL_GROUNDING_MAX_NUM_BATCHED_TOKENS
+export SPATIAL_GROUNDING_GPU_MEMORY_UTILIZATION SPATIAL_GROUNDING_ENABLE_THINKING
+export SPATIAL_GROUNDING_MAX_SAMPLES
+export TRACKING_SETTING TRACKING_DATASETS TRACKING_BENCH_DIR TRACKING_BASE_PREFIX
+export TRACKING_PROCESSOR_PATH TRACKING_VIDEO_MIN_PIXELS TRACKING_VIDEO_MAX_PIXELS
+export TRACKING_VIDEO_TOTAL_PIXELS TRACKING_MAX_FRAMES TRACKING_FPS
+export TRACKING_MAX_MODEL_LEN TRACKING_MAX_NEW_TOKENS TRACKING_MAX_NUM_BATCHED_TOKENS
+export TRACKING_BATCH_SIZE TRACKING_GPU_MEMORY_UTILIZATION TRACKING_ENABLE_THINKING
+export TRACKING_PROMPT_MODE TRACKING_CHUNKED_REPROMPT TRACKING_REPROMPT_USE_GT_FIRST_BOX
+export TRACKING_MAX_SAMPLES
+export STVG_SETTING STVG_DATASETS STVG_BENCH_DIR STVG_BASE_PREFIX STVG_PROCESSOR_PATH
+export STVG_VIDEO_MIN_PIXELS STVG_VIDEO_MAX_PIXELS STVG_VIDEO_TOTAL_PIXELS
+export STVG_MAX_FRAMES STVG_FPS STVG_MAX_MODEL_LEN STVG_MAX_NEW_TOKENS
+export STVG_MAX_NUM_BATCHED_TOKENS STVG_BATCH_SIZE STVG_GPU_MEMORY_UTILIZATION
+export STVG_ENABLE_THINKING STVG_PROMPT_MODE STVG_MAX_SAMPLES
+export SEGMENTATION_SETTING SEGMENTATION_DATASETS SEGMENTATION_BENCH_DIR SEGMENTATION_DATA_ROOT
+export SEGMENTATION_PROCESSOR_PATH SEGMENTATION_DATA_TYPE SEGMENTATION_PROMPT_MODE
+export SEGMENTATION_ENABLE_THINKING SEGMENTATION_BATCH_SIZE SEGMENTATION_MAX_NEW_TOKENS
+export SEGMENTATION_MAX_MODEL_LEN SEGMENTATION_MAX_PIXELS_IMAGE SEGMENTATION_MIN_PIXELS_IMAGE
+export SEGMENTATION_VIDEO_MAX_PIXELS SEGMENTATION_VIDEO_MIN_PIXELS SEGMENTATION_VIDEO_TOTAL_PIXELS
+export SEGMENTATION_MAX_FRAMES SEGMENTATION_FPS SEGMENTATION_VIDEO_READER
+export SEGMENTATION_GPU_MEM_UTIL SEGMENTATION_SEED
+export SEGMENTATION_MAX_SAMPLES
+export SEGMENTATION_RUN_SAM2 SEGMENTATION_SAM2_CKPT SEGMENTATION_SAM2_CFG
+export SEGMENTATION_SAM2_NUM_GPUS SEGMENTATION_SAM2_WORKERS_PER_GPU
+export TIMELENS_BENCH_DIR TIMELENS_MAX_SAMPLES
+
+# Conda env used to run eval. By default, keep the currently active environment.
+# Override with EVAL_CONDA_ENV=xxx only when an explicit env switch is desired.
+: "${EVAL_CONDA_ENV:=keep}"
+if [[ "${EVAL_CONDA_ENV}" != "keep" && "${CONDA_DEFAULT_ENV:-}" != "${EVAL_CONDA_ENV}" ]]; then
+  if command -v conda >/dev/null 2>&1; then
+    eval "$(conda shell.bash hook)"
+    conda activate "${EVAL_CONDA_ENV}"
+  else
+    echo "ERROR: conda not found and CONDA_DEFAULT_ENV=${CONDA_DEFAULT_ENV:-<unset>}" >&2
+    exit 1
+  fi
+fi
+
+cd "${PROJECT_DIR}"
+
+resolve_model_path() {
+  local input_path="${MODEL_PATH%/}"
+
+  if ls "${input_path}"/model_world_size_*_rank_0.pt >/dev/null 2>&1; then
+    local actor_dir="$input_path"
+    local hf_dir="${MERGED_MODEL_PATH:-${actor_dir}/huggingface}"
+    local need_merge=0
+
+    if [[ "${SKIP_MERGE}" = "1" ]]; then
+      need_merge=0
+    elif [[ "${FORCE_MERGE}" = "1" ]]; then
+      need_merge=1
+    elif [[ ! -f "${hf_dir}/model.safetensors" ]]; then
+      need_merge=1
+    fi
+
+    if [[ "${need_merge}" = "1" ]]; then
+      mkdir -p "${MERGE_LOG_DIR}"
+      local merge_log="${MERGE_LOG_DIR}/merge-$(basename "$(dirname "${actor_dir}")")-$(date +%Y%m%d_%H%M%S).log"
+      echo "[merge] FSDP actor checkpoint detected: ${actor_dir}"
+      echo "[merge] Merging shards -> ${hf_dir}"
+      if [[ -n "${BASE_MODEL_PATH:-}" ]]; then
+        python -u scripts/model_merger.py \
+          --local_dir "${actor_dir}" \
+          --base_model_path "${BASE_MODEL_PATH}" \
+          2>&1 | tee "${merge_log}"
+      else
+        python -u scripts/model_merger.py \
+          --local_dir "${actor_dir}" \
+          2>&1 | tee "${merge_log}"
+      fi
+      if [[ ! -f "${hf_dir}/model.safetensors" ]]; then
+        echo "ERROR: merge finished but ${hf_dir}/model.safetensors was not created. Log: ${merge_log}" >&2
+        exit 1
+      fi
+    else
+      echo "[merge] Using existing merged HF model: ${hf_dir}"
+    fi
+
+    MODEL_PATH="${hf_dir}"
+    export MODEL_PATH
+
+    if [[ "${TRACKING_PROCESSOR_PATH%/}" = "${input_path}" ]]; then
+      TRACKING_PROCESSOR_PATH="${MODEL_PATH}"
+      export TRACKING_PROCESSOR_PATH
+    fi
+    if [[ "${STVG_PROCESSOR_PATH%/}" = "${input_path}" ]]; then
+      STVG_PROCESSOR_PATH="${MODEL_PATH}"
+      export STVG_PROCESSOR_PATH
+    fi
+    if [[ "${SPATIAL_GROUNDING_PROCESSOR_PATH%/}" = "${input_path}" ]]; then
+      SPATIAL_GROUNDING_PROCESSOR_PATH="${MODEL_PATH}"
+      export SPATIAL_GROUNDING_PROCESSOR_PATH
+    fi
+    if [[ "${SEGMENTATION_PROCESSOR_PATH%/}" = "${input_path}" ]]; then
+      SEGMENTATION_PROCESSOR_PATH="${MODEL_PATH}"
+      export SEGMENTATION_PROCESSOR_PATH
+    fi
+  elif [[ -d "${input_path}/huggingface" && -f "${input_path}/huggingface/model.safetensors" && ! -f "${input_path}/config.json" ]]; then
+    MODEL_PATH="${input_path}/huggingface"
+    export MODEL_PATH
+    echo "[merge] MODEL_PATH points to a checkpoint wrapper; using ${MODEL_PATH}"
+  fi
+}
+
+resolve_model_path
+
+VLLM_TASKS=""
+RUN_TIMELENS=0
+RUN_SEGMENTATION=0
+RUN_MMSI_TRANSFORMERS=0
+RUN_REVSI=0
+
+if [[ "${MMSI_BACKEND}" != "transformers" && "${MMSI_BACKEND}" != "vllm" ]]; then
+  echo "ERROR: MMSI_BACKEND must be 'transformers' or 'vllm', got '${MMSI_BACKEND}'" >&2
+  exit 1
+fi
+
+append_vllm_task() {
+  local task="$1"
+  if [[ -z "$VLLM_TASKS" ]]; then
+    VLLM_TASKS="$task"
+  else
+    VLLM_TASKS="${VLLM_TASKS},${task}"
+  fi
+}
+
+model_family_tag() {
+  local model_path="${1%/}"
+  local name
+  name="$(basename "$model_path")"
+  local parent
+  local grandparent
+  parent="$(basename "$(dirname "$model_path")")"
+  grandparent="$(basename "$(dirname "$(dirname "$model_path")")")"
+  if [[ "$name" == "huggingface" && "$parent" == "actor" && "$grandparent" == global_step_* ]]; then
+    basename "$(dirname "$(dirname "$(dirname "$model_path")")")"
+    return
+  fi
+  if [[ "$name" == checkpoint-* ]]; then
+    basename "$(dirname "$(dirname "$model_path")")"
+  else
+    echo "$name"
+  fi
+}
+
+checkpoint_tag() {
+  local model_path="${1%/}"
+  local name
+  name="$(basename "$model_path")"
+  local parent
+  local grandparent
+  parent="$(basename "$(dirname "$model_path")")"
+  grandparent="$(basename "$(dirname "$(dirname "$model_path")")")"
+  if [[ "$name" == "huggingface" && "$parent" == "actor" && "$grandparent" == global_step_* ]]; then
+    basename "$(dirname "$(dirname "$model_path")")"
+    return
+  fi
+  if [[ "$name" == checkpoint-* ]]; then
+    echo "$name"
+  else
+    echo "base"
+  fi
+}
+
+IFS=',' read -ra TASK_LIST <<< "$TASKS"
+for RAW_TASK in "${TASK_LIST[@]}"; do
+  TASK="$(echo "$RAW_TASK" | xargs)"
+  case "$TASK" in
+    all)
+      VLLM_TASKS="videomme,videommev2,videommmu,mmvu,mvbench,videoholmes,longvideobench,lvbench,mlvu,vsi,mindcube,spatial_grounding,tracking,stvg"
+      if [[ "${MMSI_BACKEND}" == "transformers" ]]; then
+        RUN_MMSI_TRANSFORMERS=1
+      else
+        append_vllm_task "mmsi"
+      fi
+      RUN_REVSI=1
+      ;;
+    videomme|videommev2|videommmu|mmvu|mvbench|videoholmes|longvideobench|lvbench|mlvu|vsi|mindcube|spatial_grounding|tracking|stvg)
+      append_vllm_task "$TASK"
+      ;;
+    mmsi)
+      if [[ "${MMSI_BACKEND}" == "transformers" ]]; then
+        RUN_MMSI_TRANSFORMERS=1
+      else
+        append_vllm_task "mmsi"
+      fi
+      ;;
+    revsi)
+      RUN_REVSI=1
+      ;;
+    spatial|grounding|refcoco|spatial_grounding_eval)
+      append_vllm_task "spatial_grounding"
+      ;;
+    spatial_temporal_grounding)
+      append_vllm_task "stvg"
+      ;;
+    temporal_grounding|timelens|timelens_bench)
+      RUN_TIMELENS=1
+      ;;
+    segmentation|seg|reasonseg|refseg)
+      RUN_SEGMENTATION=1
+      ;;
+    "")
+      ;;
+    *)
+      echo "ERROR: unknown TASK '${TASK}'. Supported: videomme,videommev2,videommmu,mmvu,mvbench,videoholmes,longvideobench,lvbench,mlvu,vsi,mmsi,mindcube,revsi,spatial_grounding,tracking,stvg,temporal_grounding,segmentation,all" >&2
+      exit 1
+      ;;
+  esac
+done
+
+if [[ -n "$VLLM_TASKS" ]]; then
+  TASKS="$VLLM_TASKS" python eval/task/eval_vllm.py "$@"
+fi
+
+if [[ "$RUN_MMSI_TRANSFORMERS" = "1" ]]; then
+  MMSI_FAMILY="$(model_family_tag "$MODEL_PATH")"
+  MMSI_CKPT="$(checkpoint_tag "$MODEL_PATH")"
+  MMSI_RUN_ROOT="${PROJECT_DIR}/outputs/${MMSI_FAMILY}/spatial_intelligence/mmsi/${MMSI_CKPT}/${MMSI_SETTING}/$(date +%Y%m%d_%H%M%S)"
+  CUDA_VISIBLE_DEVICES="$GPUS" \
+  PROCESSOR_PATH="$MODEL_PATH" \
+  DATA_FILE="$MMSI_DATA_FILE" \
+  OUTPUT_DIR="$MMSI_RUN_ROOT" \
+  IMAGE_MIN_PIXELS="$MMSI_IMAGE_MIN_PIXELS" \
+  IMAGE_MAX_PIXELS="$MMSI_IMAGE_MAX_PIXELS" \
+  MAX_IMAGES="$MMSI_MAX_IMAGES" \
+  MAX_NEW_TOKENS="$MMSI_MAX_NEW_TOKENS" \
+  MAX_SAMPLES="$MMSI_MAX_SAMPLES" \
+  CATEGORY="$MMSI_CATEGORY" \
+  ENABLE_THINKING="$MMSI_ENABLE_THINKING" \
+  ATTN_IMPLEMENTATION="$MMSI_ATTN_IMPLEMENTATION" \
+  EXPECTED_SAMPLES="$MMSI_EXPECTED_SAMPLES" \
+  bash eval/task/mmsi/run_eval_transformers.sh "$MODEL_PATH"
+fi
+
+if [[ "$RUN_REVSI" = "1" ]]; then
+  REVSI_FAMILY="$(model_family_tag "$MODEL_PATH")"
+  REVSI_CKPT="$(checkpoint_tag "$MODEL_PATH")"
+  REVSI_RUN_EXPECTED_SAMPLES="$REVSI_EXPECTED_SAMPLES"
+  if [[ "$REVSI_MAX_SAMPLES" != "0" || -n "$REVSI_TASK_FILTER" ]]; then
+    REVSI_RUN_EXPECTED_SAMPLES=0
+  fi
+  REVSI_RUN_ROOT="${PROJECT_DIR}/outputs/${REVSI_FAMILY}/spatial_intelligence/revsi/${REVSI_CKPT}/${REVSI_SETTING}/$(date +%Y%m%d_%H%M%S)"
+  CUDA_VISIBLE_DEVICES="$GPUS" \
+  REVSI_ROOT="$REVSI_VIDEO_ROOT" \
+  QA_FILE="$REVSI_DATA_FILE" \
+  VIDEO_ROOT="$REVSI_VIDEO_ROOT" \
+  FRAME_BUDGET="$REVSI_FRAME_BUDGET" \
+  OUTPUT_DIR="$REVSI_RUN_ROOT" \
+  MAX_FRAMES="$REVSI_MAX_FRAMES" \
+  EXACT_NFRAMES="$REVSI_EXACT_NFRAMES" \
+  FPS="$REVSI_FPS" \
+  VIDEO_MIN_PIXELS="$REVSI_VIDEO_MIN_PIXELS" \
+  VIDEO_MAX_PIXELS="$REVSI_VIDEO_MAX_PIXELS" \
+  VIDEO_TOTAL_PIXELS="$REVSI_VIDEO_TOTAL_PIXELS" \
+  BATCH_SIZE="$REVSI_BATCH_SIZE" \
+  MAX_MODEL_LEN="$REVSI_MAX_MODEL_LEN" \
+  MAX_NEW_TOKENS="$REVSI_MAX_NEW_TOKENS" \
+  GPU_MEM_UTIL="$REVSI_GPU_MEMORY_UTILIZATION" \
+  MAX_SAMPLES="$REVSI_MAX_SAMPLES" \
+  EXPECTED_SAMPLES="$REVSI_RUN_EXPECTED_SAMPLES" \
+  TASK_FILTER="$REVSI_TASK_FILTER" \
+  ENABLE_THINKING="$REVSI_ENABLE_THINKING" \
+  TP_SIZE="$TP_SIZE" \
+  bash eval/task/revsi/run_eval_vllm.sh "$MODEL_PATH"
+fi
+
+if [[ "$RUN_TIMELENS" = "1" ]]; then
+  TIMELENS_FAMILY="$(model_family_tag "$MODEL_PATH")"
+  TIMELENS_CKPT="$(checkpoint_tag "$MODEL_PATH")"
+  TIMELENS_SETTING="timelens-fps${TIMELENS_FPS}-min${TIMELENS_MIN_TOKENS}-total${TIMELENS_TOTAL_TOKENS}-new${TIMELENS_MAX_NEW_TOKENS}-${TIMELENS_DATASETS//,/_}"
+  TIMELENS_RUN_ROOT="${PROJECT_DIR}/outputs/${TIMELENS_FAMILY}/temporal_grounding/${TIMELENS_CKPT}/${TIMELENS_SETTING}/$(date +%Y%m%d_%H%M%S)"
+  CUDA_VISIBLE_DEVICES="$GPUS" \
+  DATASETS="$TIMELENS_DATASETS" \
+  SETTING="$TIMELENS_SETTING" \
+  OUTPUT_ROOT="$TIMELENS_RUN_ROOT" \
+  TIMELENS_USE_OUTPUT_ROOT_DIRECT=1 \
+  MIN_TOKENS="$TIMELENS_MIN_TOKENS" \
+  TOTAL_TOKENS="$TIMELENS_TOTAL_TOKENS" \
+  MAX_PIXELS="$TIMELENS_MAX_PIXELS" \
+  MAX_FRAMES="$TIMELENS_MAX_FRAMES" \
+  FPS="$TIMELENS_FPS" \
+  MAX_NEW_TOKENS="$TIMELENS_MAX_NEW_TOKENS" \
+  REPETITION_PENALTY="$TIMELENS_REPETITION_PENALTY" \
+  STOP_AFTER_ANSWER="$TIMELENS_STOP_AFTER_ANSWER" \
+  PROMPT_MODE="$TIMELENS_PROMPT_MODE" \
+  NUM_WORKERS="$TIMELENS_NUM_WORKERS" \
+  bash eval/task/temporal_grounding/run_eval.sh \
+    "$MODEL_PATH" \
+    "$TIMELENS_ENABLE_THINKING" \
+    "$TIMELENS_RUN_ROOT"
+fi
+
+if [[ "$RUN_SEGMENTATION" = "1" ]]; then
+  SEG_FAMILY="$(model_family_tag "$MODEL_PATH")"
+  SEG_CKPT="$(checkpoint_tag "$MODEL_PATH")"
+  SEG_RUN_ROOT="${PROJECT_DIR}/outputs/${SEG_FAMILY}/segmentation/${SEG_CKPT}/${SEGMENTATION_SETTING}/$(date +%Y%m%d_%H%M%S)"
+  CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-$GPUS}" \
+  PROCESSOR_PATH="$SEGMENTATION_PROCESSOR_PATH" \
+  BENCH_DIR="$SEGMENTATION_BENCH_DIR" \
+  DATA_ROOT="$SEGMENTATION_DATA_ROOT" \
+  DATASETS="$SEGMENTATION_DATASETS" \
+  DATA_TYPE="$SEGMENTATION_DATA_TYPE" \
+  PROMPT_MODE="$SEGMENTATION_PROMPT_MODE" \
+  ENABLE_THINKING="$SEGMENTATION_ENABLE_THINKING" \
+  BATCH_SIZE="$SEGMENTATION_BATCH_SIZE" \
+  MAX_NEW_TOKENS="$SEGMENTATION_MAX_NEW_TOKENS" \
+  MAX_MODEL_LEN="$SEGMENTATION_MAX_MODEL_LEN" \
+  MAX_PIXELS_IMAGE="$SEGMENTATION_MAX_PIXELS_IMAGE" \
+  MIN_PIXELS_IMAGE="$SEGMENTATION_MIN_PIXELS_IMAGE" \
+  VIDEO_MAX_PIXELS="$SEGMENTATION_VIDEO_MAX_PIXELS" \
+  VIDEO_MIN_PIXELS="$SEGMENTATION_VIDEO_MIN_PIXELS" \
+  VIDEO_TOTAL_PIXELS="$SEGMENTATION_VIDEO_TOTAL_PIXELS" \
+  MAX_FRAMES="$SEGMENTATION_MAX_FRAMES" \
+  FPS="$SEGMENTATION_FPS" \
+  VIDEO_READER="$SEGMENTATION_VIDEO_READER" \
+  TP_SIZE="$TP_SIZE" \
+  GPU_MEM_UTIL="$SEGMENTATION_GPU_MEM_UTIL" \
+  SEED="$SEGMENTATION_SEED" \
+  MAX_SAMPLES="$SEGMENTATION_MAX_SAMPLES" \
+  RUN_SAM2="$SEGMENTATION_RUN_SAM2" \
+  SAM2_CKPT="$SEGMENTATION_SAM2_CKPT" \
+  SAM2_CFG="$SEGMENTATION_SAM2_CFG" \
+  ONETHINKER_SEG_POST="$SEGMENTATION_POSTPROCESSOR_PATH" \
+  SAM2_NUM_GPUS="$SEGMENTATION_SAM2_NUM_GPUS" \
+  SAM2_WORKERS_PER_GPU="$SEGMENTATION_SAM2_WORKERS_PER_GPU" \
+  bash eval/task/segmentation/run_eval_vllm.sh \
+    "$MODEL_PATH" \
+    "$SEG_RUN_ROOT"
+fi

code/eval/task/eval_image_mc_vllm.py

@@ -0,0 +1,511 @@
+#!/usr/bin/env python3
+"""Unified vLLM evaluator for image-sequence multiple-choice benchmarks.
+
+Supported benchmarks:
+  - MMSI-Bench: TSV/parquet with base64 image list in `image`.
+  - MindCube-Tiny: official 1,050-sample Hugging Face split.
+"""
+
+from __future__ import annotations
+
+import argparse
+import ast
+import base64
+import csv
+import io
+import json
+import os
+import re
+import sys
+import time
+from collections import defaultdict
+from typing import Any, Dict, List
+
+from canonical_data import load_json_records
+from mindcube.data_utils import (
+    OFFICIAL_DATA_SOURCE,
+    decode_mindcube_images,
+    load_mindcube_records,
+    mindcube_answer_value,
+    mindcube_group,
+    mindcube_prompt,
+)
+from PIL import Image
+
+CHOICES = list("ABCDEFGH")
+ANSWER_RE = re.compile(r"<answer>\s*(.*?)\s*</answer>", re.DOTALL | re.IGNORECASE)
+TAG_RE = re.compile(r"</?think>", re.IGNORECASE)
+FORMAT_PRIORITY = {
+    "start": 10,
+    "end": 9,
+    "phrase": 7,
+    "parentheses": 6,
+    "period": 5,
+    "colon": 4,
+    "right_paren": 3,
+    "space": 2,
+    "fallback": 0,
+}
+ANSWER_PHRASES = [
+    "the answer is",
+    "answer is",
+    "the correct answer is",
+    "correct answer is",
+    "the best answer is",
+    "best answer is",
+    "the correct option is",
+    "correct option is",
+    "i choose",
+    "i select",
+    "my answer is",
+    "答案是",
+    "答案为",
+]
+PROMPT_TAIL = (
+    "Choose the best answer from the options. "
+    "Put exactly one uppercase option letter inside <answer>...</answer> "
+    "Do not explain. Example: <answer>A</answer>"
+)
+
+
+def strip_think_block(text: str) -> str:
+    text = (text or "").strip()
+    if not text:
+        return ""
+    parts = re.split(r"</think>", text, flags=re.IGNORECASE)
+    if len(parts) > 1:
+        text = parts[-1]
+    else:
+        text = re.sub(r"<think>.*?</think>", "", text, flags=re.DOTALL | re.IGNORECASE)
+    return TAG_RE.sub("", text).strip()
+
+
+def strip_answer_tags(text: str) -> str:
+    matches = ANSWER_RE.findall(text or "")
+    if matches:
+        return matches[-1].strip()
+    return (text or "").strip()
+
+
+def extract_mcq_answer(response: str, choices: List[str] | None = None) -> str:
+    if not response or not response.strip():
+        return ""
+    all_choices = choices or CHOICES
+    text = strip_answer_tags(strip_think_block(response)).strip()
+    if not text:
+        return ""
+    for char in [",", ".", "!", "?", ";", ":", "'", '"', "。", "："]:
+        text = text.strip(char)
+    padded = " " + text + " "
+    candidates = []
+
+    for ch in all_choices:
+        for token, fmt in (
+            (f"({ch})", "parentheses"),
+            (f"{ch}.", "period"),
+            (f"{ch}:", "colon"),
+            (f"{ch})", "right_paren"),
+            (f"{ch} ", "space"),
+        ):
+            pos = padded.rfind(token)
+            if pos != -1:
+                candidates.append((ch, pos, fmt))
+
+    lower = padded.lower()
+    for phrase in ANSWER_PHRASES:
+        idx = lower.rfind(phrase.lower())
+        if idx != -1:
+            after = idx + len(phrase)
+            for ch in all_choices:
+                m = re.search(rf"\b{re.escape(ch)}\b", padded[after:], flags=re.IGNORECASE)
+                if m:
+                    candidates.append((ch, after + m.start(), "phrase"))
+
+    stripped = padded.strip()
+    for ch in all_choices:
+        if stripped.upper() == ch:
+            candidates.append((ch, 0, "start"))
+        elif stripped.startswith(ch) and (len(stripped) == 1 or not stripped[1].isalpha()):
+            candidates.append((ch, 0, "start"))
+        elif stripped.endswith(ch) and (len(stripped) == 1 or not stripped[-2].isalpha()):
+            candidates.append((ch, len(padded) - 1, "end"))
+
+    if not candidates:
+        for ch in all_choices:
+            m = re.search(rf"\b{re.escape(ch)}\b", padded)
+            if m:
+                candidates.append((ch, m.start(), "fallback"))
+    if not candidates:
+        return ""
+    candidates.sort(key=lambda x: (FORMAT_PRIORITY.get(x[2], 0), x[1]), reverse=True)
+    return candidates[0][0]
+
+
+def load_records(args) -> List[Dict[str, Any]]:
+    if args.bench == "mindcube":
+        return load_mindcube_records(
+            args.data_file,
+            chunk=args.chunk,
+            index=args.index,
+            expected_samples=args.expected_samples,
+            cache_dir=args.hf_cache_dir or None,
+        )
+    if args.data_file.endswith((".jsonl", ".json")):
+        return load_json_records(args.data_file)
+    if args.data_file.endswith(".parquet"):
+        import pandas as pd
+
+        return pd.read_parquet(args.data_file).to_dict("records")
+
+    csv.field_size_limit(sys.maxsize)
+    with open(args.data_file, encoding="utf-8", errors="replace", newline="") as f:
+        return list(csv.DictReader(f, delimiter="\t"))
+
+
+def shard(records: List[Dict[str, Any]], chunk: int, index: int) -> List[Dict[str, Any]]:
+    if chunk <= 1:
+        return records
+    return records[index::chunk]
+
+
+def decode_mmsi_images(value: Any, max_images: int) -> List[Image.Image]:
+    images_b64 = ast.literal_eval(value) if isinstance(value, str) else value
+    if not isinstance(images_b64, (list, tuple)):
+        images_b64 = [images_b64]
+    images = []
+    for item in images_b64[:max_images]:
+        if not item:
+            continue
+        path = ""
+        if isinstance(item, dict):
+            path = str(
+                item.get("path")
+                or item.get("image")
+                or item.get("image_path")
+                or ""
+            )
+        elif isinstance(item, str) and os.path.isfile(item):
+            path = item
+        if path:
+            images.append(Image.open(path).convert("RGB"))
+            continue
+        raw = item if isinstance(item, bytes) else base64.b64decode(str(item))
+        images.append(Image.open(io.BytesIO(raw)).convert("RGB"))
+    return images
+
+
+def normalize_choices(value: Any) -> List[str]:
+    if value is None:
+        return []
+    if hasattr(value, "tolist"):
+        value = value.tolist()
+    if isinstance(value, (list, tuple)):
+        return [str(v) for v in value]
+    return [str(value)]
+
+
+def answer_choices(choices: List[Any] | None) -> List[str]:
+    n = len(choices) if choices is not None else 0
+    return CHOICES[:n] if n > 0 else list("ABCD")
+
+
+def choices_from_question(question: str) -> List[str]:
+    found = re.findall(r"(?:^|[\s,])([A-H])\s*[:\).]", question or "")
+    out = []
+    for ch in found:
+        if ch not in out:
+            out.append(ch)
+    return out or list("ABCD")
+
+
+def build_prompt(args, rec: Dict[str, Any]) -> tuple[str, List[str]]:
+    if args.bench == "mmsi":
+        question = str(rec.get("question") or "").strip()
+        labels = choices_from_question(question)
+        return f"{question}\n{PROMPT_TAIL}", labels
+
+    return mindcube_prompt(rec, PROMPT_TAIL)
+
+
+def get_answer(args, rec: Dict[str, Any]) -> str:
+    answer = (
+        mindcube_answer_value(rec)
+        if args.bench == "mindcube"
+        else rec.get("answer")
+    )
+    if answer is None:
+        return ""
+    if args.bench == "mindcube" and isinstance(answer, (int, float)) and not isinstance(answer, bool):
+        idx = int(answer)
+        labels = answer_choices(normalize_choices(rec.get("choices")))
+        if 0 <= idx < len(labels):
+            return labels[idx]
+    return extract_mcq_answer(str(answer), CHOICES) or str(answer).strip().upper()[:1]
+
+
+def get_group_key(args, rec: Dict[str, Any]) -> str:
+    if args.bench == "mmsi":
+        return str(rec.get("category") or "unknown")
+    return mindcube_group(rec)
+
+
+def get_filter_key(args) -> str:
+    return "category" if args.bench == "mmsi" else "task"
+
+
+def render_chat_prompt(messages, processor, enable_thinking: bool = False) -> str:
+    return processor.apply_chat_template(
+        messages,
+        tokenize=False,
+        add_generation_prompt=True,
+        enable_thinking=enable_thinking,
+    )
+
+
+def prepare_for_vllm(images: List[Image.Image], prompt: str, processor, args):
+    from qwen_vl_utils import process_vision_info
+
+    content: List[Dict[str, Any]] = []
+    for image in images:
+        item: Dict[str, Any] = {
+            "type": "image",
+            "image": image,
+            "max_pixels": args.image_max_pixels,
+        }
+        if args.image_min_pixels > 0:
+            item["min_pixels"] = args.image_min_pixels
+        content.append(item)
+    content.append({"type": "text", "text": prompt})
+    messages = [{"role": "user", "content": content}]
+    text = render_chat_prompt(messages, processor, args.enable_thinking)
+    image_inputs, _video_inputs = process_vision_info(messages, image_patch_size=args.patch_size)
+    return {
+        "prompt": text,
+        "multi_modal_data": {"image": image_inputs},
+        "mm_processor_kwargs": {"do_resize": False},
+    }
+
+
+def pct(correct: int, total: int) -> float:
+    return round(100.0 * correct / total, 2) if total else 0.0
+
+
+def aggregate(args, results: List[Dict[str, Any]]) -> Dict[str, Any]:
+    correct = sum(int(r.get("score", 0)) for r in results)
+    parsed = sum(1 for r in results if r.get("pred_answer"))
+    groups = defaultdict(lambda: {"correct": 0, "total": 0})
+    for r in results:
+        groups[str(r.get("group") or "unknown")]["total"] += 1
+        groups[str(r.get("group") or "unknown")]["correct"] += int(r.get("score", 0))
+    group_name = "by_category" if args.bench == "mmsi" else "by_task"
+    return {
+        "num_samples": len(results),
+        "correct": correct,
+        "accuracy": pct(correct, len(results)),
+        "parse_rate": pct(parsed, len(results)),
+        group_name: {
+            k: {"accuracy": pct(v["correct"], v["total"]), "correct": v["correct"], "total": v["total"]}
+            for k, v in sorted(groups.items())
+        },
+    }
+
+
+def decode_images_for_record(args, rec: Dict[str, Any]) -> List[Image.Image]:
+    if args.bench == "mmsi":
+        return decode_mmsi_images(
+            rec.get("images") or rec.get("image"),
+            args.max_images,
+        )
+    return decode_mindcube_images(rec, args.max_images, args.min_image_bytes)
+
+
+def evaluate(llm, sampling_params, processor, args) -> Dict[str, Any]:
+    records = load_records(args)
+    filter_value = args.category if args.bench == "mmsi" else args.task
+    if filter_value:
+        if args.bench == "mindcube":
+            records = [
+                record
+                for record in records
+                if mindcube_group(record) == filter_value
+            ]
+        else:
+            key = get_filter_key(args)
+            records = [
+                record
+                for record in records
+                if str(record.get(key) or "") == filter_value
+            ]
+    if args.max_samples and args.max_samples > 0:
+        records = records[: args.max_samples]
+    if args.bench == "mmsi":
+        records = shard(records, args.chunk, args.index)
+    print(f"Loaded {len(records)} {args.bench} samples for shard {args.index}/{args.chunk}", flush=True)
+
+    results: List[Dict[str, Any]] = []
+    t0 = time.time()
+    for start in range(0, len(records), args.batch_size):
+        batch = records[start:start + args.batch_size]
+        inputs = []
+        keep = []
+        prompts: Dict[int, str] = {}
+        labels_by_j: Dict[int, List[str]] = {}
+        for j, rec in enumerate(batch):
+            try:
+                images = decode_images_for_record(args, rec)
+                if not images:
+                    raise ValueError("no decoded images")
+                prompt, labels = build_prompt(args, rec)
+                inputs.append(prepare_for_vllm(images, prompt, processor, args))
+                keep.append(j)
+                prompts[j] = prompt
+                labels_by_j[j] = labels
+            except Exception as e:
+                print(f"[warn] skip sample {start + j}: {type(e).__name__}: {e}", flush=True)
+
+        outputs = llm.generate(inputs, sampling_params=sampling_params) if inputs else []
+        out_by_j = {j: out for j, out in zip(keep, outputs)}
+
+        for j, rec in enumerate(batch):
+            if j not in out_by_j:
+                continue
+            raw = out_by_j[j].outputs[0].text
+            labels = labels_by_j[j]
+            pred = extract_mcq_answer(raw, labels)
+            gt = get_answer(args, rec)
+            score = 1.0 if pred and gt and pred.upper() == gt.upper() else 0.0
+            results.append({
+                "id": str(rec.get("index") or rec.get("id") or rec.get("sample_id") or start + j),
+                "bench": args.bench,
+                "group": get_group_key(args, rec),
+                "question": rec.get("question"),
+                "prompt": prompts.get(j, ""),
+                "answer": gt,
+                "pred_answer": pred,
+                "raw_prediction": raw,
+                "score": score,
+            })
+
+        done = min(start + args.batch_size, len(records))
+        if done % 200 == 0 or done == len(records):
+            elapsed = max(time.time() - t0, 1e-6)
+            summary = aggregate(args, results)
+            print(
+                f"[{done}/{len(records)}] {elapsed:.1f}s "
+                f"acc={summary['accuracy']:.2f}% parse={summary['parse_rate']:.2f}%",
+                flush=True,
+            )
+
+    full_mindcube_eval = (
+        args.bench == "mindcube"
+        and args.chunk == 1
+        and not args.task
+        and not args.max_samples
+    )
+    if (
+        full_mindcube_eval
+        and args.expected_samples > 0
+        and len(results) != args.expected_samples
+    ):
+        raise RuntimeError(
+            f"Expected {args.expected_samples} official MindCube-Tiny "
+            f"results, got {len(results)}."
+        )
+
+    out_name = f"results_{args.bench}"
+    if args.chunk > 1:
+        out_name += f"_shard{args.index}"
+    out_path = os.path.join(args.output_dir, out_name + ".json")
+    with open(out_path, "w", encoding="utf-8") as f:
+        json.dump(results, f, ensure_ascii=False, indent=2)
+
+    summary = aggregate(args, results)
+    if args.bench == "mindcube":
+        summary["data_source"] = args.data_file
+    if full_mindcube_eval and args.expected_samples > 0:
+        summary["expected_samples"] = args.expected_samples
+        summary["coverage"] = round(
+            100.0 * len(results) / args.expected_samples, 2
+        )
+    summary_name = f"summary_shard{args.index}.json" if args.chunk > 1 else "summary.json"
+    with open(os.path.join(args.output_dir, summary_name), "w", encoding="utf-8") as f:
+        json.dump(summary, f, ensure_ascii=False, indent=2)
+    print(json.dumps(summary, ensure_ascii=False, indent=2), flush=True)
+    return summary
+
+
+def parse_args():
+    p = argparse.ArgumentParser(description="Unified image-sequence MC evaluation via vLLM.")
+    p.add_argument("--bench", required=True, choices=["mmsi", "mindcube"])
+    p.add_argument("--model_path", required=True)
+    p.add_argument("--processor_path", default="")
+    p.add_argument("--data_file", default="")
+    p.add_argument("--output_dir", required=True)
+    p.add_argument("--image_min_pixels", type=int, default=4096)
+    p.add_argument("--image_max_pixels", type=int, default=262144)
+    p.add_argument("--max_images", type=int, default=8)
+    p.add_argument("--min_image_bytes", type=int, default=1024)
+    p.add_argument("--patch_size", type=int, default=16)
+    p.add_argument("--tensor_parallel_size", type=int, default=1)
+    p.add_argument("--gpu_memory_utilization", type=float, default=0.95)
+    p.add_argument("--max_model_len", type=int, default=65536)
+    p.add_argument("--max_new_tokens", type=int, default=64)
+    p.add_argument("--max_num_batched_tokens", type=int, default=65536)
+    p.add_argument("--batch_size", type=int, default=16)
+    p.add_argument("--max_samples", type=int, default=0)
+    p.add_argument("--expected_samples", type=int, default=1050)
+    p.add_argument("--hf_cache_dir", default="")
+    p.add_argument("--category", default="")
+    p.add_argument("--task", default="")
+    p.add_argument("--temperature", type=float, default=0.0)
+    p.add_argument("--top_p", type=float, default=1.0)
+    p.add_argument("--top_k", type=int, default=-1)
+    p.add_argument("--chunk", type=int, default=1)
+    p.add_argument("--index", type=int, default=0)
+    p.add_argument("--enable_thinking", action="store_true")
+    return p.parse_args()
+
+
+def main():
+    args = parse_args()
+    if args.bench == "mindcube" and not args.data_file:
+        args.data_file = OFFICIAL_DATA_SOURCE
+    if not args.data_file:
+        raise ValueError("--data_file is required for MMSI-Bench")
+    args.processor_path = args.processor_path or args.model_path
+    os.makedirs(args.output_dir, exist_ok=True)
+
+    from transformers import AutoProcessor, AutoTokenizer
+    from vllm import LLM, SamplingParams
+
+    processor = AutoProcessor.from_pretrained(
+        args.processor_path,
+        padding_side="left",
+        trust_remote_code=True,
+        min_pixels=args.image_min_pixels,
+        max_pixels=args.image_max_pixels,
+    )
+    tokenizer = AutoTokenizer.from_pretrained(args.processor_path, trust_remote_code=True)
+    tokenizer.padding_side = "left"
+    processor.tokenizer = tokenizer
+
+    llm = LLM(
+        model=args.model_path,
+        tensor_parallel_size=args.tensor_parallel_size,
+        gpu_memory_utilization=args.gpu_memory_utilization,
+        max_model_len=args.max_model_len,
+        max_num_batched_tokens=args.max_num_batched_tokens,
+        trust_remote_code=True,
+        limit_mm_per_prompt={"image": args.max_images},
+    )
+    sampling = SamplingParams(
+        max_tokens=args.max_new_tokens,
+        temperature=args.temperature,
+        top_p=args.top_p,
+        top_k=args.top_k,
+    )
+    evaluate(llm, sampling, processor, args)
+
+
+if __name__ == "__main__":
+    main()

code/eval/task/eval_prompt.py

@@ -0,0 +1,247 @@
+#!/usr/bin/env python3
+"""Single source of truth for evaluation prompts.
+
+Exactly ONE prompt per task — the one used by ``eval/task/eval.sh`` — kept
+byte-for-byte aligned with the SFT training data in
+``data/joint/sft_joint_all.jsonl``.
+
+Task -> prompt mapping (verbatim from sft_joint_all.jsonl):
+
+  video_qa_mc (videomme, videommmu, mmvu, mvbench, videoholmes,
+               longvideobench, lvbench, mlvu)
+      "{question}\\nOptions:\\n{opts}\\n" + VIDEO_QA_MC_TAIL
+
+  spatial_intelligence (vsi)
+      MC      -> "{q}\\nOptions:\\n{opts}\\n" + VSI_MC_TAIL
+      integer -> "{q} " + VSI_INTEGER_TAIL
+      meters  -> "{q} " + VSI_METERS_TAIL
+      cm      -> "{q} " + VSI_CENTIMETERS_TAIL
+      room m2 -> "{q} " + VSI_SQUARE_METERS_TAIL
+      (each prefixed with "These are frames of a video.\\n")
+
+  sensenova_si / image_sequence_mc (mmsi, mindcube)
+      "{q}\\n" + SENSENOVA_MC_TAIL   (alias PROMPT_TAIL)
+
+  spatial grounding (spatial_grounding)
+      build_spatial_grounding_prompt(expr)  == QWEN_NATIVE_PROMPT_SG
+
+  tracking
+      GROUNDING_QUESTION_TEMPLATE_NO_THINK.format(Question=q) + TRACKING_TAIL
+
+  stvg
+      GROUNDING_QUESTION_TEMPLATE_NO_THINK.format(
+          Question=TRAIN_STVG_QUESTION_PREFIX.format(query=q)) + STVG_TAIL
+
+  segmentation
+      "{question}\\n" + TRAIN_SEG_IMAGE_TAIL / TRAIN_SEG_VIDEO_TAIL
+
+  temporal grounding (temporal_grounding)
+      TEMPORAL_GROUNDING_PROMPT.format(event)
+
+Keep this file dependency-free (stdlib only) so it can be imported from every
+worker regardless of the runtime environment.
+"""
+
+# ===========================================================================
+# Multiple-choice / numeric answer tails
+# ===========================================================================
+
+# video_qa_mc — videomme, videommmu, mmvu, mvbench, videoholmes,
+# longvideobench, lvbench, mlvu.
+VIDEO_QA_MC_TAIL = (
+    "Answer with the option letter only within <answer>...</answer> tags. "
+    "Example: <answer>A</answer>"
+)
+
+# spatial_intelligence multiple choice — vsi (object_rel_distance,
+# object_rel_direction, obj_appearance_order, route_plan, ...).
+VSI_MC_TAIL = (
+    "Answer with the option letter within <answer>...</answer> tags. "
+    "Example: <answer>A</answer>"
+)
+
+# spatial_intelligence numeric — vsi.
+VSI_INTEGER_TAIL = (
+    "Answer with an integer within <answer>...</answer> tags. Example: <answer>3</answer>"
+)
+VSI_METERS_TAIL = (
+    "Answer with a number in meters within <answer>...</answer> tags. Example: <answer>2.3</answer>"
+)
+VSI_CENTIMETERS_TAIL = (
+    "Answer with a number in centimeters within <answer>...</answer> tags. Example: <answer>120.5</answer>"
+)
+VSI_SQUARE_METERS_TAIL = (
+    "Answer with a number in square meters within <answer>...</answer> tags. Example: <answer>25.5</answer>"
+)
+
+# sensenova_si / image_sequence_mc_answer_only — mmsi, mindcube.
+SENSENOVA_MC_TAIL = (
+    "Choose the best answer from the options. "
+    "Put exactly one uppercase option letter inside <answer>...</answer> "
+    "Do not explain. Example: <answer>A</answer>"
+)
+# Backwards-compatible alias used throughout eval_vllm.py.
+PROMPT_TAIL = SENSENOVA_MC_TAIL
+
+
+# ===========================================================================
+# Tracking
+# ===========================================================================
+
+# Question scaffold shared by tracking + stvg (the tail carries the full
+# answer-format spec).
+GROUNDING_QUESTION_TEMPLATE_NO_THINK = (
+    "{Question}\n"
+    "Please answer this question based on the visual content. "
+)
+
+TRACKING_TAIL = (
+    "Please track the target object throughout the video and provide one bounding box per second, "
+    "ONLY up to 32 seconds, within the <answer>...</answer> tags.\n"
+    "Example:\n"
+    "<answer>{\"boxes\": {\"1\": [405, 230, 654, 463], \"2\": [435, 223, 678, 446], "
+    "\"32\": [415, 203, 691, 487]}}</answer>\n"
+    "Note: Each key in 'boxes' must correspond to a second (1, 2, 3, ..., 32) "
+    "and contain a 4-number bounding box [x1, y1, x2, y2]."
+)
+
+
+# ===========================================================================
+# Spatial-temporal grounding (STVG)
+# ===========================================================================
+
+STVG_TAIL = (
+    "Please provide only the time span in seconds and bounding boxes as JSON "
+    "within the <answer>...</answer> tags.\n"
+    "You MUST output one bounding box for every integer second within the "
+    "given time span (inclusive).\n"
+    "Example:\n"
+    "<answer>{\"time\": [8.1, 13.5], \"boxes\": {\"9\": [317, 422, 582, 997], "
+    "\"10\": [332, 175, 442, 369], \"11\": [340, 180, 450, 370]}}</answer>\n"
+    "Note: Each key in 'boxes' must be an integer second within the span, "
+    "and its value must be a 4-number bounding box [x1, y1, x2, y2]."
+)
+
+# Rewrites an eval question into the exact wording used during STVG training.
+TRAIN_STVG_QUESTION_PREFIX = (
+    'Given the query "{query}", when and where does the described content '
+    'occur in the video? please firstly give the start and end time, spatial '
+    'bounding box corresponding to each integer second.'
+)
+
+
+# ===========================================================================
+# Segmentation
+# ===========================================================================
+
+# Matches sft_joint_all.jsonl seg_image / seg_video samples.
+TRAIN_SEG_IMAGE_TAIL = (
+    "Please answer this question based on the visual content. "
+    "This task prepares inputs for image object segmentation with a specialized model (e.g., SAM2).\n"
+    "Please provide ONE bounding box, 3 positive points (clearly INSIDE the object), "
+    "and 3 negative points (clearly OUTSIDE the object) within the <answer>...</answer> tags.\n"
+    "Choose informative points that help distinguish object vs. background. Prefer negatives on clear non-object "
+    "pixels INSIDE the box when safe; otherwise place them just outside on obvious background. "
+    "Negatives must NEVER be on the object or on its boundary.\n"
+    "Example: <answer>{\"boxes\": [x1, y1, x2, y2], \"positive_points\": [[x,y],[x,y],[x,y]], "
+    "\"negative_points\": [[x,y],[x,y],[x,y]]}</answer>"
+)
+
+TRAIN_SEG_VIDEO_TAIL = (
+    "Please answer this question based on the visual content. "
+    "This task prepares inputs for video object segmentation with a specialized model (e.g., SAM2).\n"
+    "Please select ONE representative time (in seconds), and provide ONE bounding box, "
+    "3 positive points (clearly INSIDE the object), and 3 negative points (clearly OUTSIDE the object) "
+    "within the <answer>...</answer> tags.\n"
+    "Choose informative points that help distinguish object vs. background. Prefer negatives on clear non-object "
+    "pixels INSIDE the box when safe; otherwise place them just outside on obvious background. "
+    "Negatives must NEVER be on the object or on its boundary.\n"
+    "Example: <answer>{\"time\": <time_in_seconds>, \"boxes\": [x1, y1, x2, y2], "
+    "\"positive_points\": [[x,y],[x,y],[x,y]], \"negative_points\": [[x,y],[x,y],[x,y]]}</answer>"
+)
+
+
+# ===========================================================================
+# Spatial grounding (RefCOCO / spatial_grounding)
+# ===========================================================================
+
+# Matches sft_joint_all.jsonl spatial grounding samples after the <image> token.
+QWEN_NATIVE_PROMPT_SG = (
+    'Locate "{}" in the image. Output its bounding box in JSON format '
+    'within <answer>...</answer> tags. '
+    'Example: <answer>[{{"bbox_2d": [123, 30, 404, 846]}}]</answer>'
+)
+
+
+# ===========================================================================
+# Temporal grounding (TimeLens / temporal_grounding)
+# ===========================================================================
+
+# Matches sft_joint_all.jsonl temporal grounding samples (after <video>).
+TEMPORAL_GROUNDING_PROMPT = (
+    'To accurately pinpoint the event "{}" in the video, '
+    "determine the precise time period of the event. "
+    "Provide the start and end times (in seconds) "
+    'in the format "start time to end time" within <answer> </answer> tags. '
+    "Example: <answer> 12 to 18 </answer>"
+)
+# Exact prompt used by the official TimeLens evaluation.
+TIMELENS_OFFICIAL_PROMPT = (
+    "Please find the visual event described by the sentence '{}', "
+    "determining its starting and ending times. "
+    "The format should be: 'The event happens in <start time> - <end time> seconds'."
+)
+# Alias used by eval_timelens_hf.py.
+PROMPT_WO_THINK = TEMPORAL_GROUNDING_PROMPT
+TEMPORAL_GROUNDING_PROMPT_MODES = {
+    "same": TEMPORAL_GROUNDING_PROMPT,
+    "timelens_official": TIMELENS_OFFICIAL_PROMPT,
+}
+
+
+# ===========================================================================
+# Builder helpers
+# ===========================================================================
+
+def build_video_qa_mc_prompt(question, options_block):
+    """`{question}\\nOptions:\\n{options_block}\\n{VIDEO_QA_MC_TAIL}`."""
+    return f"{question}\nOptions:\n{options_block}\n{VIDEO_QA_MC_TAIL}"
+
+
+def build_sensenova_mc_prompt(question):
+    """`{question}\\n{SENSENOVA_MC_TAIL}`."""
+    return f"{question}\n{SENSENOVA_MC_TAIL}"
+
+
+def build_spatial_grounding_prompt(expression):
+    """Training-aligned spatial grounding prompt."""
+    expression = (expression or "").strip()
+    if expression and expression[-1] not in ".?!":
+        expression += "."
+    return QWEN_NATIVE_PROMPT_SG.format(expression)
+
+
+def build_tracking_prompt(question):
+    return GROUNDING_QUESTION_TEMPLATE_NO_THINK.format(Question=question) + TRACKING_TAIL
+
+
+def build_stvg_prompt(raw_query):
+    q = TRAIN_STVG_QUESTION_PREFIX.format(query=raw_query)
+    return GROUNDING_QUESTION_TEMPLATE_NO_THINK.format(Question=q) + STVG_TAIL
+
+
+def build_seg_prompt(question, data_type):
+    tail = TRAIN_SEG_VIDEO_TAIL if str(data_type).strip().lower() == "video" else TRAIN_SEG_IMAGE_TAIL
+    return f"{question}\n{tail}"
+
+
+def build_temporal_grounding_prompt(event, prompt_mode="same"):
+    try:
+        template = TEMPORAL_GROUNDING_PROMPT_MODES[prompt_mode]
+    except KeyError as exc:
+        choices = ", ".join(sorted(TEMPORAL_GROUNDING_PROMPT_MODES))
+        raise ValueError(
+            f"Unknown temporal grounding prompt mode {prompt_mode!r}; "
+            f"choose one of: {choices}"
+        ) from exc
+    return template.format(event)

code/eval/task/eval_revsi_multinode.sh

@@ -0,0 +1,229 @@
+#!/usr/bin/env bash
+# ReVSI evaluation using the same vLLM multi-node sharding path as VSI-Bench.
+set -euo pipefail
+
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+PROJECT_DIR=$(cd -- "${SCRIPT_DIR}/../.." && pwd)
+RUNNER="${SCRIPT_DIR}/revsi/run_eval_vllm.sh"
+MERGER="${SCRIPT_DIR}/revsi/merge_multinode_shards.py"
+PYTHON_BIN=${PYTHON_BIN:-$(command -v python)}
+HOSTFILE=${HOSTFILE:-}
+NUM_NODES=${NUM_NODES:-4}
+GPUS_PER_NODE=${GPUS_PER_NODE:-8}
+
+MODEL=${MODEL:-}
+MODEL=${MODEL%/}
+REVSI_ROOT=${REVSI_ROOT:-}
+FRAME_BUDGET=${REVSI_FRAME_BUDGET:-all}
+FRAME_DIR="${REVSI_ROOT:+${REVSI_ROOT}/${FRAME_BUDGET}_frame}"
+QA_FILE=${REVSI_QA_FILE:-"${FRAME_DIR:+${FRAME_DIR}/test-00000-of-00001.parquet}"}
+VIDEO_ROOT=${REVSI_VIDEO_ROOT:-${REVSI_ROOT}}
+TASK_FILTER=${REVSI_TASK_FILTER:-}
+MAX_SAMPLES=${REVSI_MAX_SAMPLES:-0}
+REMOTE_SETUP=${REMOTE_SETUP:-}
+SSH_STRICT_HOST_KEY_CHECKING=${SSH_STRICT_HOST_KEY_CHECKING:-yes}
+
+case "${FRAME_BUDGET}" in
+  16|32|64|all) ;;
+  *)
+    echo "[FATAL] REVSI_FRAME_BUDGET must be 16, 32, 64, or all." >&2
+    exit 2
+    ;;
+esac
+[[ -n "${MODEL}" ]] || {
+  echo "[FATAL] Set MODEL to the model directory." >&2
+  exit 2
+}
+[[ -n "${QA_FILE}" ]] || {
+  echo "[FATAL] Set REVSI_QA_FILE or REVSI_ROOT." >&2
+  exit 2
+}
+[[ -n "${HOSTFILE}" ]] || {
+  echo "[FATAL] Set HOSTFILE to one host per line." >&2
+  exit 2
+}
+for path in \
+  "${RUNNER}" "${MERGER}" "${HOSTFILE}" "${MODEL}/config.json" "${QA_FILE}"; do
+  [[ -f "${path}" ]] || {
+    echo "[FATAL] Required input is unavailable: ${path}" >&2
+    exit 1
+  }
+done
+if [[ "${QA_FILE}" == *.parquet ]]; then
+  [[ -n "${FRAME_DIR}" ]] || {
+    echo "[FATAL] Parquet input requires REVSI_ROOT." >&2
+    exit 2
+  }
+  compgen -G "${FRAME_DIR}/*.mp4" >/dev/null || {
+    echo "[FATAL] No ReVSI videos found under ${FRAME_DIR}." >&2
+    exit 1
+  }
+fi
+
+if [[ -n "${REVSI_EXPECTED_SAMPLES:-}" ]]; then
+  EXPECTED_SAMPLES=${REVSI_EXPECTED_SAMPLES}
+elif [[ -z "${TASK_FILTER}" && "${MAX_SAMPLES}" == "0" ]]; then
+  EXPECTED_SAMPLES=6808
+else
+  EXPECTED_SAMPLES=0
+fi
+
+mapfile -t AVAILABLE_HOSTS < <(awk 'NF && !seen[$1]++ {print $1}' "${HOSTFILE}")
+[[ "${NUM_NODES}" =~ ^[1-9][0-9]*$ ]] || {
+  echo "[FATAL] NUM_NODES must be a positive integer." >&2
+  exit 2
+}
+((NUM_NODES <= ${#AVAILABLE_HOSTS[@]})) || {
+  echo "[FATAL] Requested ${NUM_NODES} nodes, only ${#AVAILABLE_HOSTS[@]} available." >&2
+  exit 2
+}
+HOSTS=("${AVAILABLE_HOSTS[@]:0:NUM_NODES}")
+GLOBAL_SHARDS=$((NUM_NODES * GPUS_PER_NODE))
+
+checkpoint_dir=$(dirname "$(dirname "${MODEL%/}")")
+checkpoint_tag=$(basename "${checkpoint_dir}")
+experiment_tag=$(basename "$(dirname "${checkpoint_dir}")")
+timestamp=$(date +%Y%m%d_%H%M%S)
+EVAL_MAX_FRAMES=${REVSI_MAX_FRAMES:-${FRAME_BUDGET}}
+EVAL_EXACT_NFRAMES=${REVSI_EXACT_NFRAMES:-1}
+if [[ "${FRAME_BUDGET}" == "all" ]]; then
+  EVAL_MAX_FRAMES=${REVSI_MAX_FRAMES:-128}
+fi
+setting_tag="revsi-${FRAME_BUDGET}frame-multinode${NUM_NODES}x${GPUS_PER_NODE}-f${EVAL_MAX_FRAMES}-exact${EVAL_EXACT_NFRAMES}-fps${REVSI_FPS:-2}-total${REVSI_VIDEO_TOTAL_PIXELS:-16777216}"
+OUTPUT_DIR=${OUTPUT_DIR:-"${PROJECT_DIR}/outputs/${experiment_tag}/spatial_intelligence/revsi/${checkpoint_tag}/${setting_tag}/${timestamp}"}
+LOG_DIR=${LOG_DIR:-"${PROJECT_DIR}/logs/revsi_multinode/${timestamp}"}
+mkdir -p "${OUTPUT_DIR}" "${LOG_DIR}"
+
+SSH_OPTS=(-o "StrictHostKeyChecking=${SSH_STRICT_HOST_KEY_CHECKING}" -o BatchMode=yes -o ServerAliveInterval=60)
+WORKER_PATTERN='eval/task/revsi/eval_revsi_vllm.py'
+
+run_on_host() {
+  local node=$1
+  shift
+  if ((node == 0)); then
+    bash -s <<<"$*"
+  else
+    ssh "${SSH_OPTS[@]}" "${HOSTS[node]}" bash -s <<<"$*"
+  fi
+}
+
+kill_stale_workers() {
+  local node reap_pids=() command
+  command="pkill -f $(printf %q "${WORKER_PATTERN}") >/dev/null 2>&1 || true; sleep 5; pkill -9 -f $(printf %q "${WORKER_PATTERN}") >/dev/null 2>&1 || true; sleep 3"
+  for ((node = 0; node < NUM_NODES; node++)); do
+    run_on_host "${node}" "${command}" >/dev/null 2>&1 &
+    reap_pids+=("$!")
+  done
+  wait "${reap_pids[@]}" 2>/dev/null || true
+}
+
+require_free_gpus() {
+  local node free bad=0
+  local need_gib=${REVSI_MIN_FREE_GIB:-75}
+  for ((node = 0; node < NUM_NODES; node++)); do
+    free=$(run_on_host "${node}" \
+      "nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits | sort -n | awk 'NR==1 {print}'" 2>/dev/null |
+      tr -d '\r' | awk '/^[0-9]+$/ {value=$0} END {print value}')
+    if [[ -z "${free}" ]]; then
+      echo "[FATAL] node=${node} host=${HOSTS[node]}: cannot query GPU memory." >&2
+      bad=1
+    elif ((free / 1024 < need_gib)); then
+      echo "[FATAL] node=${node} host=${HOSTS[node]}: only $((free / 1024)) GiB free (need ${need_gib} GiB)." >&2
+      bad=1
+    fi
+  done
+  ((bad == 0)) || exit 1
+}
+
+pids=()
+cleanup() {
+  local status=$?
+  trap - EXIT INT TERM
+  for pid in "${pids[@]:-}"; do
+    [[ -n "${pid}" ]] && kill "${pid}" >/dev/null 2>&1 || true
+  done
+  ((status == 0)) || kill_stale_workers
+  exit "${status}"
+}
+trap cleanup EXIT INT TERM
+
+kill_stale_workers
+require_free_gpus
+
+dispatch() {
+  local node=$1
+  local offset=$((node * GPUS_PER_NODE))
+  local gpu_ids
+  gpu_ids=$(seq -s, 0 $((GPUS_PER_NODE - 1)))
+  local env_values=(
+    "CUDA_VISIBLE_DEVICES=${gpu_ids}"
+    "REVSI_ROOT=${REVSI_ROOT}"
+    "FRAME_BUDGET=${FRAME_BUDGET}"
+    "QA_FILE=${QA_FILE}"
+    "VIDEO_ROOT=${VIDEO_ROOT}"
+    "OUTPUT_DIR=${OUTPUT_DIR}"
+    "GLOBAL_SHARD_COUNT=${GLOBAL_SHARDS}"
+    "GLOBAL_SHARD_OFFSET=${offset}"
+    "MERGE_SHARDS=0"
+    "EXPECTED_SAMPLES=0"
+    "TASK_FILTER=${TASK_FILTER}"
+    "MAX_SAMPLES=${MAX_SAMPLES}"
+    "MEDIA_MODE=video"
+    "TP_SIZE=1"
+    "BATCH_SIZE=${REVSI_BATCH_SIZE:-16}"
+    "MAX_FRAMES=${EVAL_MAX_FRAMES}"
+    "EXACT_NFRAMES=${EVAL_EXACT_NFRAMES}"
+    "FPS=${REVSI_FPS:-2}"
+    "VIDEO_MIN_PIXELS=${REVSI_VIDEO_MIN_PIXELS:-65536}"
+    "VIDEO_MAX_PIXELS=${REVSI_VIDEO_MAX_PIXELS:-}"
+    "VIDEO_TOTAL_PIXELS=${REVSI_VIDEO_TOTAL_PIXELS:-16777216}"
+    "MAX_MODEL_LEN=${REVSI_MAX_MODEL_LEN:-32768}"
+    "MAX_NEW_TOKENS=${REVSI_MAX_NEW_TOKENS:-64}"
+    "GPU_MEM_UTIL=${REVSI_GPU_MEMORY_UTILIZATION:-0.90}"
+    "STRICT_NUMERIC_PROMPT=${REVSI_STRICT_NUMERIC_PROMPT:-0}"
+    "ENABLE_THINKING=${REVSI_ENABLE_THINKING:-false}"
+    "SCORE_LOG_INTERVAL=${REVSI_SCORE_LOG_INTERVAL:-50}"
+    "LAUNCH_DELAY=2"
+    "VLLM_BASE_PORT=49000"
+  )
+  local prefix="" value
+  for value in "${env_values[@]}"; do
+    prefix+="$(printf %q "${value}") "
+  done
+  local setup_prefix=""
+  if [[ -n "${REMOTE_SETUP}" ]]; then
+    setup_prefix="${REMOTE_SETUP} && "
+  fi
+  local command="${setup_prefix}cd $(printf %q "${PROJECT_DIR}") && ${prefix}bash $(printf %q "${RUNNER}") $(printf %q "${MODEL}")"
+  local log="${LOG_DIR}/node${node}.log"
+  echo "[revsi] node=${node}/${NUM_NODES} host=${HOSTS[node]} shards=${offset}-$((offset + GPUS_PER_NODE - 1)) log=${log}"
+  if ((node == 0)); then
+    bash -lc "${command}" 2>&1 | tee "${log}"
+  else
+    ssh "${SSH_OPTS[@]}" "${HOSTS[node]}" \
+      "bash -lc $(printf %q "${command}")" >"${log}" 2>&1 &
+    pids+=("$!")
+  fi
+}
+
+for ((node = 1; node < NUM_NODES; node++)); do
+  dispatch "${node}"
+done
+dispatch 0
+
+status=0
+for pid in "${pids[@]}"; do
+  if ! wait "${pid}"; then status=1; fi
+done
+pids=()
+((status == 0)) || {
+  echo "[FATAL] One or more ReVSI nodes failed; inspect ${LOG_DIR}." >&2
+  exit "${status}"
+}
+
+"${PYTHON_BIN}" "${MERGER}" \
+  --output-dir "${OUTPUT_DIR}" \
+  --num-shards "${GLOBAL_SHARDS}" \
+  --expected-samples "${EXPECTED_SAMPLES}"
+chmod -R a+rX "${OUTPUT_DIR}" 2>/dev/null || true
+echo "[revsi] Multi-node evaluation complete: ${OUTPUT_DIR}/summary.json"

code/eval/task/mindcube/data_utils.py

@@ -0,0 +1,254 @@
+"""Official MindCube-Tiny data loading and record normalization."""
+
+from __future__ import annotations
+
+import io
+import re
+import sys
+from collections import Counter
+from pathlib import Path
+from typing import Any, Dict, List
+
+from PIL import Image
+
+_TASK_DIR = Path(__file__).resolve().parents[1]
+if str(_TASK_DIR) not in sys.path:
+    sys.path.insert(0, str(_TASK_DIR))
+
+from canonical_data import load_json_records  # noqa: E402
+
+OFFICIAL_DATA_SOURCE = (
+    "hf://datasets/oscarqjh/MindCube_lmmseval"
+    "@7dd2725d9bd4149f2aad00a9843f72a3824da003/tiny/combined"
+)
+OFFICIAL_EXPECTED_SAMPLES = 1050
+OFFICIAL_GROUP_COUNTS = {
+    "rotation": 200,
+    "among": 600,
+    "around": 250,
+}
+HF_DATASET_PREFIX = "hf://datasets/"
+CHOICES = list("ABCDEFGH")
+
+
+def parse_hf_dataset_source(
+    source: str,
+) -> tuple[str, str | None, str, str]:
+    """Parse hf://datasets/<org>/<name>[@revision]/<config>/<split>."""
+    if not source.startswith(HF_DATASET_PREFIX):
+        raise ValueError(f"Not a Hugging Face dataset source: {source}")
+    payload = source[len(HF_DATASET_PREFIX) :]
+    try:
+        dataset_spec, config, split = payload.rsplit("/", 2)
+    except ValueError as error:
+        raise ValueError(
+            "MindCube Hugging Face source must use "
+            "hf://datasets/<org>/<name>/<config>/<split>"
+        ) from error
+    dataset_name, separator, revision = dataset_spec.rpartition("@")
+    if not separator:
+        dataset_name = dataset_spec
+        revision = None
+    if not dataset_name or "/" not in dataset_name or not config or not split:
+        raise ValueError(f"Invalid MindCube Hugging Face source: {source}")
+    return dataset_name, revision, config, split
+
+
+def mindcube_group(record: Dict[str, Any]) -> str:
+    """Return one of the official Rotation/Among/Around settings."""
+    for key in ("task", "setting"):
+        value = str(record.get(key) or "").strip().lower()
+        if value in OFFICIAL_GROUP_COUNTS:
+            return value
+
+    sample_id = str(
+        record.get("id")
+        or record.get("index")
+        or record.get("sample_id")
+        or ""
+    ).strip().lower()
+    match = re.match(r"^(rotation|among|around)(?:_|$)", sample_id)
+    return match.group(1) if match else "unknown"
+
+
+def _validate_records(
+    records: Any,
+    expected_samples: int,
+) -> None:
+    if expected_samples <= 0:
+        return
+    actual = len(records)
+    if actual != expected_samples:
+        raise RuntimeError(
+            f"Expected {expected_samples} official MindCube-Tiny samples, "
+            f"got {actual}. Refusing to evaluate a reduced or different split."
+        )
+    if expected_samples != OFFICIAL_EXPECTED_SAMPLES:
+        return
+
+    if hasattr(records, "column_names") and "id" in records.column_names:
+        groups = Counter(
+            mindcube_group({"id": sample_id}) for sample_id in records["id"]
+        )
+    else:
+        groups = Counter(mindcube_group(record) for record in records)
+    actual_groups = {
+        group: groups.get(group, 0) for group in OFFICIAL_GROUP_COUNTS
+    }
+    if actual_groups != OFFICIAL_GROUP_COUNTS or groups.get("unknown", 0):
+        raise RuntimeError(
+            "Official MindCube-Tiny must contain "
+            f"{OFFICIAL_GROUP_COUNTS}, got {dict(groups)}."
+        )
+
+
+def load_mindcube_records(
+    source: str,
+    *,
+    chunk: int = 1,
+    index: int = 0,
+    expected_samples: int = OFFICIAL_EXPECTED_SAMPLES,
+    cache_dir: str | None = None,
+) -> List[Dict[str, Any]]:
+    """Load and shard the official split before decoding its images."""
+    if chunk <= 0 or index < 0 or index >= chunk:
+        raise ValueError(f"Invalid shard index {index}/{chunk}")
+
+    if source.startswith(HF_DATASET_PREFIX):
+        from datasets import load_dataset
+
+        dataset_name, revision, config, split = parse_hf_dataset_source(
+            source
+        )
+        records = load_dataset(
+            dataset_name,
+            config,
+            split=split,
+            revision=revision,
+            cache_dir=cache_dir or None,
+        )
+        _validate_records(records, expected_samples)
+        if chunk > 1:
+            records = records.shard(
+                num_shards=chunk,
+                index=index,
+                contiguous=False,
+            )
+        return [records[row_index] for row_index in range(len(records))]
+
+    if source.endswith((".jsonl", ".json")):
+        records = load_json_records(source)
+        _validate_records(records, expected_samples)
+        return records[index::chunk]
+
+    import pandas as pd
+
+    if not source.endswith(".parquet"):
+        raise ValueError(
+            "MindCube data must be the official Hugging Face source or a "
+            f"local official parquet, got: {source}"
+        )
+    records = pd.read_parquet(source).to_dict("records")
+    _validate_records(records, expected_samples)
+    return records[index::chunk]
+
+
+def decode_hf_image(
+    value: Any,
+    min_image_bytes: int,
+) -> Image.Image | None:
+    if value is None:
+        return None
+    if isinstance(value, Image.Image):
+        return value.convert("RGB")
+
+    raw = None
+    path = None
+    if isinstance(value, dict):
+        raw = value.get("bytes")
+        path = value.get("path")
+    elif isinstance(value, (bytes, bytearray, memoryview)):
+        raw = value
+    elif isinstance(value, str):
+        path = value
+
+    if raw is None and path:
+        return Image.open(path).convert("RGB")
+    if raw is None:
+        return None
+    raw = bytes(raw)
+    if min_image_bytes > 0 and len(raw) < min_image_bytes:
+        return None
+    return Image.open(io.BytesIO(raw)).convert("RGB")
+
+
+def decode_mindcube_images(
+    record: Dict[str, Any],
+    max_images: int,
+    min_image_bytes: int,
+) -> List[Image.Image]:
+    values = record.get("images")
+    if values is not None:
+        if hasattr(values, "tolist"):
+            values = values.tolist()
+        if not isinstance(values, (list, tuple)):
+            values = [values]
+    else:
+        values = [
+            record.get(f"image_{idx}") for idx in range(max_images)
+        ]
+
+    images = []
+    for value in values[:max_images]:
+        image = decode_hf_image(value, min_image_bytes)
+        if image is not None:
+            images.append(image)
+    return images
+
+
+def normalize_choices(value: Any) -> List[str]:
+    if value is None:
+        return []
+    if hasattr(value, "tolist"):
+        value = value.tolist()
+    if isinstance(value, (list, tuple)):
+        return [str(item) for item in value]
+    return [str(value)]
+
+
+def labels_from_question(question: str) -> List[str]:
+    labels = []
+    for label in re.findall(
+        r"(?:^|[\s,])([A-H])\s*[:\).]",
+        question or "",
+    ):
+        if label not in labels:
+            labels.append(label)
+    return labels or list("ABCD")
+
+
+def mindcube_prompt(
+    record: Dict[str, Any],
+    prompt_tail: str,
+) -> tuple[str, List[str]]:
+    question = str(record.get("question") or "").strip()
+    choices = normalize_choices(record.get("choices"))
+    labels = CHOICES[: len(choices)] if choices else labels_from_question(
+        question
+    )
+
+    official_prompt = str(record.get("input_prompt") or "").strip()
+    if official_prompt:
+        return official_prompt, labels
+    if choices:
+        options = "\n".join(
+            f"{label}. {text}"
+            for label, text in zip(labels, choices)
+        )
+        return f"{question}\nOptions:\n{options}\n{prompt_tail}", labels
+    return f"{question}\n{prompt_tail}", labels
+
+
+def mindcube_answer_value(record: Dict[str, Any]) -> Any:
+    answer = record.get("gt_answer")
+    return record.get("answer") if answer is None else answer

code/eval/task/mmsi/eval_mmsi_transformers.py

@@ -0,0 +1,323 @@
+#!/usr/bin/env python3
+"""MMSI-Bench evaluation with Hugging Face Transformers.
+
+Unlike the vLLM evaluator, this backend can pass every image in a sample
+without configuring a fixed multimodal-item limit.
+"""
+
+from __future__ import annotations
+
+import argparse
+import ast
+import base64
+import io
+import json
+import logging
+import os
+import random
+import sys
+import time
+import warnings
+from pathlib import Path
+from typing import Any, Dict, List
+
+import numpy as np
+import torch
+from PIL import Image
+from qwen_vl_utils import process_vision_info
+from transformers import AutoModelForImageTextToText, AutoProcessor
+
+
+TASK_DIR = Path(__file__).resolve().parents[1]
+if str(TASK_DIR) not in sys.path:
+    sys.path.insert(0, str(TASK_DIR))
+
+from eval_image_mc_vllm import (  # noqa: E402
+    aggregate,
+    build_prompt,
+    extract_mcq_answer,
+    get_answer,
+    get_group_key,
+    load_records,
+    shard,
+)
+
+
+warnings.filterwarnings("ignore", message=".*pad_token_id.*")
+logging.getLogger("transformers").setLevel(logging.ERROR)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="MMSI-Bench evaluation via Hugging Face Transformers."
+    )
+    parser.add_argument("--model_path", required=True)
+    parser.add_argument("--processor_path", default="")
+    parser.add_argument("--data_file", required=True)
+    parser.add_argument("--output_dir", required=True)
+    parser.add_argument("--image_min_pixels", type=int, default=4096)
+    parser.add_argument("--image_max_pixels", type=int, default=262144)
+    parser.add_argument(
+        "--max_images",
+        type=int,
+        default=0,
+        help="Maximum images per sample; <=0 means use every image.",
+    )
+    parser.add_argument("--patch_size", type=int, default=16)
+    parser.add_argument("--max_new_tokens", type=int, default=1024)
+    parser.add_argument("--max_samples", type=int, default=0)
+    parser.add_argument("--category", default="")
+    parser.add_argument("--chunk", type=int, default=1)
+    parser.add_argument("--index", type=int, default=0)
+    parser.add_argument("--enable_thinking", action="store_true")
+    parser.add_argument("--seed", type=int, default=42)
+    parser.add_argument("--save_interval", type=int, default=20)
+    parser.add_argument(
+        "--attn_implementation",
+        default="flash_attention_2",
+        choices=["flash_attention_2", "sdpa", "eager"],
+    )
+    args = parser.parse_args()
+    args.bench = "mmsi"
+    args.processor_path = args.processor_path or args.model_path
+    return args
+
+
+def set_seed(seed: int) -> None:
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    if torch.cuda.is_available():
+        torch.cuda.manual_seed_all(seed)
+
+
+def decode_all_images(value: Any, max_images: int) -> List[Image.Image]:
+    values = ast.literal_eval(value) if isinstance(value, str) else value
+    if not isinstance(values, (list, tuple)):
+        values = [values]
+    if max_images > 0:
+        values = values[:max_images]
+
+    images: List[Image.Image] = []
+    for item in values:
+        if not item:
+            continue
+        path = ""
+        if isinstance(item, dict):
+            path = str(
+                item.get("path")
+                or item.get("image")
+                or item.get("image_path")
+                or ""
+            )
+        elif isinstance(item, str) and os.path.isfile(item):
+            path = item
+        if path:
+            with Image.open(path) as image:
+                images.append(image.convert("RGB"))
+            continue
+        raw = item if isinstance(item, bytes) else base64.b64decode(str(item))
+        with Image.open(io.BytesIO(raw)) as image:
+            images.append(image.convert("RGB"))
+    return images
+
+
+def prepare_inputs(
+    images: List[Image.Image],
+    prompt: str,
+    processor,
+    args: argparse.Namespace,
+):
+    content: List[Dict[str, Any]] = []
+    for image in images:
+        content.append(
+            {
+                "type": "image",
+                "image": image,
+                "min_pixels": args.image_min_pixels,
+                "max_pixels": args.image_max_pixels,
+            }
+        )
+    content.append({"type": "text", "text": prompt})
+    messages = [{"role": "user", "content": content}]
+    text = processor.apply_chat_template(
+        messages,
+        tokenize=False,
+        add_generation_prompt=True,
+        enable_thinking=args.enable_thinking,
+    )
+    image_inputs, _ = process_vision_info(
+        messages,
+        image_patch_size=args.patch_size,
+    )
+    return processor(
+        text=[text],
+        images=image_inputs,
+        padding=True,
+        return_tensors="pt",
+    )
+
+
+def sample_id(record: Dict[str, Any], fallback: int) -> str:
+    for key in ("index", "id", "sample_id"):
+        value = record.get(key)
+        if value is not None and str(value) != "":
+            return str(value)
+    return str(fallback)
+
+
+def write_progress(
+    output_path: Path,
+    summary_path: Path,
+    args: argparse.Namespace,
+    results: List[Dict[str, Any]],
+) -> None:
+    output_path.write_text(
+        json.dumps(results, ensure_ascii=False, indent=2) + "\n",
+        encoding="utf-8",
+    )
+    summary_path.write_text(
+        json.dumps(aggregate(args, results), ensure_ascii=False, indent=2) + "\n",
+        encoding="utf-8",
+    )
+
+
+def main() -> None:
+    args = parse_args()
+    if not torch.cuda.is_available():
+        raise RuntimeError("CUDA is required for MMSI Transformers evaluation.")
+    if args.chunk <= 0 or not 0 <= args.index < args.chunk:
+        raise ValueError(
+            f"Invalid shard index: index={args.index}, chunk={args.chunk}"
+        )
+
+    set_seed(args.seed)
+    output_dir = Path(args.output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+    result_path = output_dir / f"results_mmsi_shard{args.index}.json"
+    summary_path = output_dir / f"summary_shard{args.index}.json"
+
+    print(f"Model:       {args.model_path}", flush=True)
+    print(f"Processor:   {args.processor_path}", flush=True)
+    print(f"Data:        {args.data_file}", flush=True)
+    print(f"Shard:       {args.index}/{args.chunk}", flush=True)
+    print(
+        "Images:      "
+        f"{'all' if args.max_images <= 0 else args.max_images} "
+        f"(min_px={args.image_min_pixels}, max_px={args.image_max_pixels})",
+        flush=True,
+    )
+
+    print("Loading Transformers model ...", flush=True)
+    model = AutoModelForImageTextToText.from_pretrained(
+        args.model_path,
+        dtype=torch.bfloat16,
+        attn_implementation=args.attn_implementation,
+        device_map={"": "cuda:0"},
+        trust_remote_code=True,
+    ).eval()
+    processor = AutoProcessor.from_pretrained(
+        args.processor_path,
+        padding_side="left",
+        do_resize=False,
+        trust_remote_code=True,
+    )
+
+    records = load_records(args)
+    if args.category:
+        records = [
+            record
+            for record in records
+            if str(record.get("category") or "") == args.category
+        ]
+    if args.max_samples > 0:
+        records = records[: args.max_samples]
+    indexed_records = list(enumerate(records))
+    indexed_records = shard(indexed_records, args.chunk, args.index)
+    print(f"Loaded {len(indexed_records)} samples for this shard.", flush=True)
+
+    results: List[Dict[str, Any]] = []
+    started = time.time()
+    for local_index, (source_index, record) in enumerate(indexed_records, 1):
+        images: List[Image.Image] = []
+        raw_prediction = ""
+        error = ""
+        prompt, labels = build_prompt(args, record)
+        ground_truth = get_answer(args, record)
+        try:
+            images = decode_all_images(
+                record.get("images") or record.get("image"),
+                args.max_images,
+            )
+            if not images:
+                raise ValueError("sample contains no decodable images")
+            inputs = prepare_inputs(images, prompt, processor, args).to(
+                "cuda", non_blocking=True
+            )
+            input_length = inputs.input_ids.shape[-1]
+            with torch.inference_mode():
+                generated = model.generate(
+                    **inputs,
+                    do_sample=False,
+                    max_new_tokens=args.max_new_tokens,
+                    use_cache=True,
+                )
+            generated = generated[:, input_length:]
+            raw_prediction = processor.batch_decode(
+                generated,
+                skip_special_tokens=True,
+                clean_up_tokenization_spaces=False,
+            )[0]
+            del inputs, generated
+        except Exception as exc:
+            error = f"{type(exc).__name__}: {exc}"
+            torch.cuda.empty_cache()
+            print(
+                f"[warn] sample={source_index} images={len(images)}: {error}",
+                flush=True,
+            )
+        finally:
+            for image in images:
+                image.close()
+
+        prediction = extract_mcq_answer(raw_prediction, labels)
+        score = float(
+            bool(prediction)
+            and bool(ground_truth)
+            and prediction.upper() == ground_truth.upper()
+        )
+        results.append(
+            {
+                "id": sample_id(record, source_index),
+                "bench": "mmsi",
+                "group": get_group_key(args, record),
+                "question": record.get("question"),
+                "prompt": prompt,
+                "answer": ground_truth,
+                "pred_answer": prediction,
+                "raw_prediction": raw_prediction,
+                "num_images": len(images),
+                "score": score,
+                "error": error,
+            }
+        )
+
+        if (
+            local_index % args.save_interval == 0
+            or local_index == len(indexed_records)
+        ):
+            write_progress(result_path, summary_path, args, results)
+            summary = aggregate(args, results)
+            print(
+                f"[{local_index}/{len(indexed_records)}] "
+                f"{time.time() - started:.1f}s "
+                f"acc={summary['accuracy']:.2f}% "
+                f"parse={summary['parse_rate']:.2f}%",
+                flush=True,
+            )
+
+    print(json.dumps(aggregate(args, results), ensure_ascii=False, indent=2))
+
+
+if __name__ == "__main__":
+    main()

code/eval/task/mmsi/run_eval_transformers.sh

@@ -0,0 +1,214 @@
+#!/usr/bin/env bash
+# MMSI-Bench evaluation with one Transformers model replica per GPU.
+set -euo pipefail
+
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+PROJECT_DIR=$(cd -- "${SCRIPT_DIR}/../../.." && pwd)
+EVALUATOR="${SCRIPT_DIR}/eval_mmsi_transformers.py"
+
+MODEL_PATH=${1:-${MODEL_PATH:-}}
+PROCESSOR_PATH=${PROCESSOR_PATH:-${MODEL_PATH}}
+DATA_FILE=${DATA_FILE:-"${PROJECT_DIR}/data/eval/mmsi/MMSI_bench.tsv"}
+IMAGE_MIN_PIXELS=${IMAGE_MIN_PIXELS:-4096}
+IMAGE_MAX_PIXELS=${IMAGE_MAX_PIXELS:-262144}
+MAX_IMAGES=${MAX_IMAGES:-0}
+PATCH_SIZE=${PATCH_SIZE:-16}
+MAX_NEW_TOKENS=${MAX_NEW_TOKENS:-1024}
+MAX_SAMPLES=${MAX_SAMPLES:-0}
+CATEGORY=${CATEGORY:-}
+ENABLE_THINKING=${ENABLE_THINKING:-false}
+ATTN_IMPLEMENTATION=${ATTN_IMPLEMENTATION:-flash_attention_2}
+SAVE_INTERVAL=${SAVE_INTERVAL:-20}
+if [[ -n "${CATEGORY}" || "${MAX_SAMPLES}" != "0" ]]; then
+  EXPECTED_SAMPLES=${EXPECTED_SAMPLES:-0}
+else
+  EXPECTED_SAMPLES=${EXPECTED_SAMPLES:-1000}
+fi
+LAUNCH_DELAY=${LAUNCH_DELAY:-2}
+OUTPUT_ROOT=${OUTPUT_ROOT:-"${PROJECT_DIR}/outputs/mmsi"}
+
+[[ -n "${MODEL_PATH}" ]] || {
+  echo "[FATAL] MODEL_PATH is required." >&2
+  exit 1
+}
+for path in "${EVALUATOR}" "${MODEL_PATH}/config.json" "${DATA_FILE}"; do
+  [[ -f "${path}" ]] || {
+    echo "[FATAL] Required input is unavailable: ${path}" >&2
+    exit 1
+  }
+done
+compgen -G "${MODEL_PATH}/*.safetensors" >/dev/null || {
+  echo "[FATAL] No safetensors weights found under ${MODEL_PATH}." >&2
+  exit 1
+}
+
+IFS=',' read -ra GPULIST <<<"${CUDA_VISIBLE_DEVICES:-$(seq -s, 0 $(($(nvidia-smi -L | wc -l) - 1)))}"
+NUM_GPUS=${#GPULIST[@]}
+((NUM_GPUS > 0)) || {
+  echo "[FATAL] No visible GPUs." >&2
+  exit 1
+}
+unset CUDA_VISIBLE_DEVICES
+
+model_name=$(basename "${MODEL_PATH%/}")
+model_parent=$(basename "$(dirname "${MODEL_PATH%/}")")
+model_grandparent=$(basename "$(dirname "$(dirname "${MODEL_PATH%/}")")")
+if [[ "${model_name}" == "huggingface" && "${model_parent}" == "actor" && "${model_grandparent}" == global_step_* ]]; then
+  MODEL_FAMILY=$(basename "$(dirname "$(dirname "$(dirname "${MODEL_PATH%/}")")")")
+  CHECKPOINT_TAG=${model_grandparent}
+elif [[ "${model_name}" == checkpoint-* ]]; then
+  MODEL_FAMILY=$(basename "$(dirname "$(dirname "${MODEL_PATH%/}")")")
+  CHECKPOINT_TAG=${model_name}
+else
+  MODEL_FAMILY=${model_name}
+  CHECKPOINT_TAG=base
+fi
+
+DATA_TAG=$(basename "${DATA_FILE}")
+DATA_TAG=${DATA_TAG%.*}
+IMAGE_COUNT_TAG=$([[ "${MAX_IMAGES}" -le 0 ]] && echo all || echo "${MAX_IMAGES}")
+SETTING_TAG="transformers-img-min${IMAGE_MIN_PIXELS}-max${IMAGE_MAX_PIXELS}-n${IMAGE_COUNT_TAG}-all-${DATA_TAG}"
+OUTPUT_DIR=${OUTPUT_DIR:-"${OUTPUT_ROOT}/${MODEL_FAMILY}/${CHECKPOINT_TAG}/${SETTING_TAG}/$(date +%Y%m%d_%H%M%S)"}
+mkdir -p "${OUTPUT_DIR}"
+
+echo "============================================================"
+echo "MMSI-Bench Evaluation (Transformers, data-parallel)"
+echo "============================================================"
+echo "Model:       ${MODEL_PATH}"
+echo "Processor:   ${PROCESSOR_PATH}"
+echo "Data:        ${DATA_FILE}"
+echo "GPUs:        ${GPULIST[*]} (${NUM_GPUS} replicas)"
+echo "Images:      $([[ "${MAX_IMAGES}" -le 0 ]] && echo all || echo "${MAX_IMAGES}")"
+echo "Pixels:      min=${IMAGE_MIN_PIXELS} max=${IMAGE_MAX_PIXELS}"
+echo "Max new:     ${MAX_NEW_TOKENS}"
+echo "Thinking:    ${ENABLE_THINKING}"
+echo "Output:      ${OUTPUT_DIR}"
+echo "============================================================"
+
+pids=()
+cleanup() {
+  local status=$?
+  trap - EXIT INT TERM
+  for pid in "${pids[@]:-}"; do
+    [[ -n "${pid}" ]] && kill "${pid}" >/dev/null 2>&1 || true
+  done
+  wait >/dev/null 2>&1 || true
+  exit "${status}"
+}
+trap cleanup EXIT INT TERM
+
+for index in "${!GPULIST[@]}"; do
+  gpu=${GPULIST[index]}
+  args=(
+    --model_path "${MODEL_PATH}"
+    --processor_path "${PROCESSOR_PATH}"
+    --data_file "${DATA_FILE}"
+    --output_dir "${OUTPUT_DIR}"
+    --image_min_pixels "${IMAGE_MIN_PIXELS}"
+    --image_max_pixels "${IMAGE_MAX_PIXELS}"
+    --max_images "${MAX_IMAGES}"
+    --patch_size "${PATCH_SIZE}"
+    --max_new_tokens "${MAX_NEW_TOKENS}"
+    --max_samples "${MAX_SAMPLES}"
+    --category "${CATEGORY}"
+    --chunk "${NUM_GPUS}"
+    --index "${index}"
+    --attn_implementation "${ATTN_IMPLEMENTATION}"
+    --save_interval "${SAVE_INTERVAL}"
+  )
+  if [[ "${ENABLE_THINKING,,}" == "true" || "${ENABLE_THINKING}" == "1" ]]; then
+    args+=(--enable_thinking)
+  fi
+  CUDA_VISIBLE_DEVICES="${gpu}" PYTHONUNBUFFERED=1 \
+    python "${EVALUATOR}" "${args[@]}" \
+    >"${OUTPUT_DIR}/worker_${index}.log" 2>&1 &
+  pids+=("$!")
+  echo "Launched shard ${index}/${NUM_GPUS} on GPU ${gpu} (PID ${pids[-1]})"
+  if ((index + 1 < NUM_GPUS)) && ((LAUNCH_DELAY > 0)); then
+    sleep "${LAUNCH_DELAY}"
+  fi
+done
+
+failed=0
+for index in "${!pids[@]}"; do
+  if ! wait "${pids[index]}"; then
+    echo "[FAIL] shard ${index}; inspect ${OUTPUT_DIR}/worker_${index}.log" >&2
+    failed=1
+  else
+    echo "[DONE] shard ${index}"
+  fi
+done
+pids=()
+((failed == 0)) || exit 1
+
+python - "${OUTPUT_DIR}" "${NUM_GPUS}" "${EXPECTED_SAMPLES}" "${MAX_IMAGES}" <<'PY'
+import json
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+output_dir = Path(sys.argv[1])
+num_shards = int(sys.argv[2])
+expected = int(sys.argv[3])
+max_images = int(sys.argv[4])
+records_by_id = {}
+for shard in range(num_shards):
+    path = output_dir / f"results_mmsi_shard{shard}.json"
+    if not path.is_file():
+        raise RuntimeError(f"Missing MMSI result shard: {path}")
+    for record in json.loads(path.read_text(encoding="utf-8")):
+        records_by_id[str(record["id"])] = record
+
+records = list(records_by_id.values())
+if expected > 0 and len(records) != expected:
+    raise RuntimeError(
+        f"Expected {expected} unique MMSI samples, got {len(records)}"
+    )
+
+correct = sum(int(record.get("score", 0)) for record in records)
+parsed = sum(bool(record.get("pred_answer")) for record in records)
+errors = sum(bool(record.get("error")) for record in records)
+groups = defaultdict(lambda: {"correct": 0, "total": 0})
+for record in records:
+    group = str(record.get("group") or "unknown")
+    groups[group]["total"] += 1
+    groups[group]["correct"] += int(record.get("score", 0))
+
+def pct(value, total):
+    return round(100.0 * value / total, 2) if total else 0.0
+
+summary = {
+    "backend": "transformers",
+    "image_policy": "all" if max_images <= 0 else f"first_{max_images}",
+    "num_samples": len(records),
+    "correct": correct,
+    "accuracy": pct(correct, len(records)),
+    "parse_rate": pct(parsed, len(records)),
+    "errors": errors,
+    "by_category": {
+        key: {
+            "accuracy": pct(value["correct"], value["total"]),
+            "correct": value["correct"],
+            "total": value["total"],
+        }
+        for key, value in sorted(groups.items())
+    },
+}
+(output_dir / "results_mmsi.json").write_text(
+    json.dumps(records, ensure_ascii=False, indent=2) + "\n",
+    encoding="utf-8",
+)
+(output_dir / "summary.json").write_text(
+    json.dumps(summary, ensure_ascii=False, indent=2) + "\n",
+    encoding="utf-8",
+)
+print(
+    f"MMSI Transformers: n={len(records)} "
+    f"acc={summary['accuracy']:.2f}% "
+    f"parse={summary['parse_rate']:.2f}% errors={errors}"
+)
+print(f"Summary: {output_dir / 'summary.json'}")
+PY
+
+chmod -R a+rX "${OUTPUT_DIR}" 2>/dev/null || true
+echo "MMSI Transformers evaluation complete: ${OUTPUT_DIR}"

code/eval/task/qwenvl_decord_patch.py

@@ -0,0 +1,271 @@
+"""Monkey-patch qwen_vl_utils to force the decord video backend.
+
+Why this is needed
+==================
+
+* Evaluators call ``process_vision_info(..., return_video_metadata=True)``,
+  which is the **new-API** form of ``qwen_vl_utils``. New-API
+  ``fetch_video`` expects the underlying backend reader to return a
+  3-tuple ``(video_tensor, video_metadata, sample_fps)``.
+
+* Newer ``qwen_vl_utils`` (>= 0.0.10) hard-codes
+  ``VIDEO_READER_BACKENDS["torchvision"](ele)`` inside ``fetch_video``, so
+  ``FORCE_QWENVL_VIDEO_READER=decord`` no longer takes effect. The
+  torchvision reader on some machines fails with
+  ``KeyError: 'video_fps'`` because the mp4 metadata is incomplete.
+
+* Older ``_read_video_decord`` (<= 0.0.8) returns a single tensor; if we
+  just alias ``"torchvision" -> _read_video_decord``, the 3-value unpack
+  in ``fetch_video`` blows up with ``RuntimeError: Error reading <path>``.
+
+Strategy
+========
+
+Replace ``vp.fetch_video`` with our own version that uses ``decord``
+directly and returns the **new-API 3-tuple** the rest of qwen_vl_utils
+expects. This way both old- and new-API call sites work, and we never
+have to ship a patched copy of qwen_vl_utils itself.
+
+Disable the patch with ``DISABLE_QWENVL_DECORD_PATCH=1`` (e.g. when
+debugging on a machine where torchvision works fine).
+"""
+from __future__ import annotations
+
+import math
+import os
+import sys
+from typing import Any
+
+
+def _smart_nframes(ele: dict, *, total_frames: int, video_fps: float) -> int:
+    """Mirror qwen_vl_utils.vision_process.smart_nframes: pick the number
+    of frames to sample given fps / max_frames hints in the message dict.
+
+    We re-implement instead of importing because qwen_vl_utils versions
+    differ on the public name (``smart_nframes`` vs internal helper).
+    """
+    FPS = float(ele.get("fps", 2.0))
+    MIN_FRAMES = int(ele.get("min_frames", 4))
+    MAX_FRAMES = int(ele.get("max_frames", 768))
+    nframes = ele.get("nframes")
+    if nframes is not None:
+        return max(1, min(int(nframes), total_frames))
+    duration = total_frames / max(1e-6, video_fps)
+    if FPS <= 0:
+        FPS = 2.0
+    n = int(round(duration * FPS))
+    n = max(MIN_FRAMES, min(MAX_FRAMES, n))
+    n = max(1, min(n, total_frames))
+    return n
+
+
+def _resolve_video_path(video_path: str) -> str:
+    """If the input path doesn't exist, try a few fallbacks before giving up.
+
+    Benchmark annotations join a base prefix with a relative media path, so a
+    layout that nests media one level differently than the annotation expects
+    (for example ``./Evaluation/got10k/<file>.mp4`` against an on-disk
+    ``<base>/got10k/<file>.mp4``) would otherwise fail to decode.
+
+    Resolution order:
+      0. exact path (default)
+      1. drop ``/Evaluation/`` -> retry
+      2. drop leading ``Evaluation/`` from the raw component if EVAL_BASE_PREFIX_OVERRIDE
+      3. take basename + walk under override prefix (slow last resort)
+    """
+    if os.path.isfile(video_path):
+        return video_path
+
+    candidates = []
+
+    # (1) Drop the spurious "/Evaluation/" component anywhere in the path.
+    if "/Evaluation/" in video_path:
+        candidates.append(video_path.replace("/Evaluation/", "/", 1))
+
+    # (2) Override-prefix-based rewrites.
+    override = os.getenv("EVAL_BASE_PREFIX_OVERRIDE", "").rstrip("/")
+    base_name = os.path.basename(video_path)
+    parent = os.path.dirname(video_path)
+    if override:
+        rel = video_path
+        for prefix in ("Evaluation/", "evaluation/", "./Evaluation/", "./"):
+            if rel.startswith(prefix):
+                rel = rel[len(prefix):]
+                break
+        candidates.append(os.path.join(override, rel))
+        candidates.append(os.path.join(override, base_name))
+        pdir = os.path.basename(parent)
+        if pdir:
+            candidates.append(os.path.join(override, pdir, base_name))
+
+    for c in candidates:
+        if os.path.isfile(c):
+            print(f"[qwenvl_decord_patch] path-resolved {video_path!r} → {c!r}",
+                  file=sys.stderr)
+            return c
+
+    # (3) Last resort: walk under override (or the path's grandparent).
+    walk_root = override or os.path.dirname(parent) or parent
+    if walk_root and os.path.isdir(walk_root):
+        for root, dirs, files in os.walk(walk_root):
+            if base_name in files:
+                hit = os.path.join(root, base_name)
+                print(f"[qwenvl_decord_patch] walked-resolved {video_path!r} → {hit!r}",
+                      file=sys.stderr)
+                return hit
+            depth = root[len(walk_root):].count(os.sep)
+            if depth >= 4:
+                dirs.clear()
+
+    return video_path
+
+
+def _decord_fetch_video_new_api(ele: dict, *args, **kwargs):
+    """Drop-in replacement for ``qwen_vl_utils.vision_process.fetch_video``.
+
+    The exact signature of fetch_video varies across qwen_vl_utils
+    versions, but we observe the following calling conventions in the
+    wild:
+
+      v0.0.8     : fetch_video(ele, image_factor)             → tensor
+      v0.0.10+   : fetch_video(ele, image_factor=…,
+                                  return_video_sample_fps=True,
+                                  return_video_metadata=False) → (video, sample_fps)
+      v0.0.10+   : fetch_video(ele, …, return_video_metadata=True) → (video, metadata, sample_fps)
+
+    We therefore inspect ``return_video_sample_fps`` /
+    ``return_video_metadata`` kwargs and return whichever shape the
+    caller asked for. The actual decode is always decord-based.
+    """
+    import torch
+    import decord
+
+    video_path = ele["video"]
+    if isinstance(video_path, str) and video_path.startswith("file://"):
+        video_path = video_path[len("file://"):]
+
+    # Multi-tier fallback path resolution (see _resolve_video_path docstring)
+    video_path = _resolve_video_path(video_path)
+
+    if "video_start" in ele or "video_end" in ele:
+        # Trim window. decord supports ranges via numeric indices, but no
+        # evaluated benchmark sets these, so we keep it simple.
+        raise NotImplementedError(
+            "video_start/video_end not implemented in decord patch; "
+            "the evaluated benchmarks do not need this."
+        )
+
+    vr = decord.VideoReader(video_path)
+    total_frames = len(vr)
+    video_fps = float(vr.get_avg_fps())
+    if total_frames <= 0 or not math.isfinite(video_fps) or video_fps <= 0:
+        raise RuntimeError(
+            f"decord-patch: invalid video {video_path!r} "
+            f"(total_frames={total_frames}, video_fps={video_fps})"
+        )
+
+    nframes = _smart_nframes(ele, total_frames=total_frames, video_fps=video_fps)
+    idx = torch.linspace(0, total_frames - 1, nframes).round().long().tolist()
+    arr = vr.get_batch(idx).asnumpy()  # (T, H, W, C)
+    video = torch.from_numpy(arr).permute(0, 3, 1, 2).contiguous()  # (T, C, H, W)
+
+    sample_fps = nframes / (total_frames / video_fps) if total_frames > 0 else float(video_fps)
+
+    # metadata fields are chosen to be compatible with transformers'
+    # VideoMetadata dataclass (qwen3-vl image processor instantiates this
+    # from the dict we return). Field set across qwen2/qwen3-vl versions:
+    #   fps, total_num_frames, duration, frames_indices, video_backend
+    # We deliberately DO NOT include "video_fps" — that name causes
+    # ``VideoMetadata.__init__() got an unexpected keyword argument 'video_fps'``
+    # on newer transformers.
+    metadata = {
+        "fps": float(video_fps),
+        "total_num_frames": int(total_frames),
+        "duration": float(total_frames / video_fps),
+        "frames_indices": [int(i) for i in idx],
+        "video_backend": "decord",
+    }
+
+    # Decide return shape based on what the caller asked for.
+    # NOTE on shapes: qwen_vl_utils versions disagree.
+    #   * Some versions:   fetch_video(...) → (video, metadata, sample_fps)
+    #   * Other versions:  fetch_video(...) → ((video, metadata), sample_fps)
+    #                        (i.e. the wrapper bundles the metadata INSIDE the
+    #                         video return so that process_vision_info can keep
+    #                         a 2-tuple unpack.)
+    # The pinned qwen_vl_utils release uses the 2-tuple-with-bundled-metadata
+    # variant, so we default to that when both flags are set. Override with
+    # QWENVL_RETURN_SHAPE=3tuple if your stack expects the 3-element form.
+    want_metadata = bool(kwargs.get("return_video_metadata", False))
+    want_sample_fps = bool(kwargs.get("return_video_sample_fps", False))
+    return_shape = os.getenv("QWENVL_RETURN_SHAPE", "auto").strip().lower()
+
+    if want_metadata and want_sample_fps:
+        if return_shape == "3tuple":
+            return video, metadata, float(sample_fps)
+        # default "auto" / "bundled" → 2-tuple form
+        return (video, metadata), float(sample_fps)
+    if want_metadata:
+        return video, metadata
+    if want_sample_fps:
+        return video, float(sample_fps)
+    return video
+
+
+def _apply_patch() -> None:
+    if os.getenv("DISABLE_QWENVL_DECORD_PATCH", "0") == "1":
+        print("[qwenvl_decord_patch] disabled via env", file=sys.stderr)
+        return
+
+    try:
+        from qwen_vl_utils import vision_process as vp
+    except Exception as exc:  # pragma: no cover
+        print(f"[qwenvl_decord_patch] qwen_vl_utils import failed: {exc}",
+              file=sys.stderr)
+        return
+
+    # Make sure decord is importable up front so we fail loudly here, not
+    # mid-eval, if it's not installed.
+    try:
+        import decord  # noqa: F401
+    except Exception as exc:
+        print(f"[qwenvl_decord_patch] decord not installed: {exc}",
+              file=sys.stderr)
+        print("                       Install with: pip install decord",
+              file=sys.stderr)
+        return
+
+    # Belt: try the env-based switch (no-op on new versions).
+    os.environ.setdefault("FORCE_QWENVL_VIDEO_READER", "decord")
+
+    # Suspenders: replace fetch_video itself.
+    if not hasattr(vp, "fetch_video"):
+        print("[qwenvl_decord_patch] vp.fetch_video missing, abort",
+              file=sys.stderr)
+        return
+
+    # Also alias the torchvision backend to our decord function in case any
+    # downstream code re-reads VIDEO_READER_BACKENDS directly.
+    backends = getattr(vp, "VIDEO_READER_BACKENDS", None)
+    if isinstance(backends, dict):
+        # Wrap to satisfy the SINGLE-tensor return convention used by some
+        # internal callers (the older ``_read_video_torchvision`` returns
+        # just a tensor in 0.0.8 path).
+        def _single_tensor_decord(ele):
+            v, _meta, _fps = _decord_fetch_video_new_api(ele)
+            return v
+        backends["torchvision"] = _single_tensor_decord
+
+    vp.fetch_video = _decord_fetch_video_new_api
+
+    print(
+        f"[qwenvl_decord_patch] applied. "
+        f"vp.fetch_video → decord (return shape auto-adapts to "
+        f"return_video_metadata / return_video_sample_fps kwargs; "
+        f"override with QWENVL_RETURN_SHAPE=3tuple). "
+        f"file={vp.__file__}",
+        file=sys.stderr,
+    )
+
+
+_apply_patch()

code/eval/task/revsi/eval_revsi_vllm.py

@@ -0,0 +1,1025 @@
+"""ReVSI evaluation with vLLM data-parallel sharding.
+
+This keeps the existing OraRL VSI execution style (one vLLM process per GPU,
+JSONL shards, centralized merge) while using ReVSI's official prompt,
+question-type grouping, and macro-averaged metrics.
+
+Metrics:
+    * numerical tasks: MRA (mean relative accuracy)
+    * multiple-choice tasks: exact-match ACC on the first token/letter
+
+Outputs one JSONL shard per process and a summary JSON after launcher merge.
+"""
+
+import argparse
+import json
+import os
+import re
+import sys
+import time
+from collections import defaultdict
+from functools import lru_cache
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Optional, Tuple
+
+import numpy as np
+
+
+TASK_DIR = Path(__file__).resolve().parents[1]
+if str(TASK_DIR) not in sys.path:
+    sys.path.insert(0, str(TASK_DIR))
+
+from canonical_data import load_json_records  # noqa: E402
+
+
+try:
+    from tqdm import tqdm
+except ImportError:
+    def tqdm(iterable, **_kwargs):
+        return iterable
+
+
+NUMERICAL_TASKS = {
+    "object_abs_distance",
+    "object_counting_single",
+    "object_counting_multiple",
+    "object_size_estimation",
+    "room_size_estimation_single",
+    "room_size_estimation_multiple",
+}
+
+MC_TASKS = {
+    "object_rel_direction_forward_easy",
+    "object_rel_direction_backward_easy",
+    "object_rel_direction_forward_hard",
+    "object_rel_direction_backward_hard",
+    "object_rel_distance_closest",
+    "object_rel_distance_farthest",
+    "route_planning",
+}
+
+DIRECTION_PREFIXES = ("object_rel_direction",)
+
+REPORT_ORDER = [
+    ("Abs. Dist", "object_abs_distance", "MRA"),
+    ("Obj. Count", "object_counting", "MRA"),
+    ("Rel. Dir", "object_rel_direction", "ACC"),
+    ("Rel. Dis", "object_rel_distance", "ACC"),
+    ("Obj. Size", "object_size_estimation", "MRA"),
+    ("Room Size", "room_size_estimation", "MRA"),
+    ("Route Plan", "route_planning", "ACC"),
+]
+
+COMPOSITE_METRICS = {
+    "object_counting": (
+        "object_counting_single",
+        "object_counting_multiple",
+    ),
+    "object_rel_direction": (
+        "object_rel_direction_forward_easy",
+        "object_rel_direction_backward_easy",
+        "object_rel_direction_forward_hard",
+        "object_rel_direction_backward_hard",
+    ),
+    "object_rel_distance": (
+        "object_rel_distance_closest",
+        "object_rel_distance_farthest",
+    ),
+    "room_size_estimation": (
+        "room_size_estimation_single",
+        "room_size_estimation_multiple",
+    ),
+}
+
+ANSWER_RE = re.compile(r"<answer>\s*(.*?)\s*</answer>", flags=re.DOTALL | re.IGNORECASE)
+FINAL_ANSWER_RE = re.compile(
+    r"(?:the\s+)?(?:final\s+answer|answer|答案)\s*(?:is|=|是|为)?\s*[:：]?\s*"
+    r"([-+]?\d+(?:\.\d+)?|[A-Da-d])\b",
+    flags=re.IGNORECASE,
+)
+LOOSE_NUMERIC_INSTRUCTION_RE = re.compile(
+    r"\s*Please answer the question using a single word or phrase\.?\s*",
+    flags=re.IGNORECASE,
+)
+NUMERIC_STRICT_INSTRUCTION = (
+    "Output only a number. Do not output option letters, units, explanations, "
+    "or punctuation."
+)
+MC_STRICT_INSTRUCTION = "Answer with the option letter only."
+
+
+def _strip_answer_tags(text: str) -> str:
+    text = text or ""
+    matches = ANSWER_RE.findall(text)
+    if matches:
+        return matches[-1].strip()
+    return text.strip()
+
+
+def strip_think_block(text: str) -> str:
+    text = (text or "").strip()
+    if not text:
+        return ""
+    # Prefer content after the last closing tag. This handles common Qwen
+    # no-think outputs such as "</think> B" and complete think blocks.
+    parts = re.split(r"</think>", text, flags=re.IGNORECASE)
+    if len(parts) > 1:
+        text = parts[-1]
+    else:
+        text = re.sub(r"<think>.*?</think>", "", text, flags=re.DOTALL | re.IGNORECASE)
+    text = re.sub(r"</?think>", "", text, flags=re.IGNORECASE)
+    return text.strip()
+
+
+def fuzzy_matching(pred: str) -> str:
+    pred = _strip_answer_tags(strip_think_block(pred))
+    pred = pred.strip()
+    if not pred:
+        return ""
+    m = FINAL_ANSWER_RE.search(pred)
+    if m:
+        return m.group(1).rstrip(".").strip()
+    # Official VSI code uses the first whitespace-separated token and strips
+    # trailing punctuation.  After stripping dangling think tags, this recovers
+    # outputs like "</think> B" -> "B" and "</think> 2" -> "2".
+    return pred.split()[0].rstrip(".。,:：").strip()
+
+
+def extract_numeric_prediction(pred: str) -> str:
+    """Robust numeric extraction for VSI numerical tasks.
+
+    Some RL checkpoints emit option-like prefixes on numeric answers, e.g.
+    "A.2" or "A\n\n4".  For numerical tasks, recover the number while keeping
+    MC parsing unchanged.
+    """
+    text = _strip_answer_tags(strip_think_block(pred)).strip()
+    if not text:
+        return ""
+    m = FINAL_ANSWER_RE.search(text)
+    if m and re.search(r"\d", m.group(1)):
+        return m.group(1).rstrip(".").strip()
+    m = re.match(r"^\s*[A-Da-d]\s*[\.\):：\-]?\s*([-+]?\d+(?:\.\d+)?)\b", text)
+    if m:
+        return m.group(1).strip()
+    m = re.search(r"[-+]?\d+(?:\.\d+)?", text)
+    return m.group(0).strip() if m else fuzzy_matching(pred)
+
+
+def clean_prediction_for_task(question_type: str, pred: str) -> str:
+    if question_type in NUMERICAL_TASKS:
+        return extract_numeric_prediction(pred)
+    return fuzzy_matching(pred)
+
+
+def to_float(pred: Any) -> Optional[float]:
+    try:
+        return float(str(pred).strip())
+    except (ValueError, TypeError):
+        return None
+
+
+def exact_match(pred: str, target: Any) -> float:
+    return 1.0 if str(pred).strip().lower() == str(target).strip().lower() else 0.0
+
+
+def abs_dist_norm(pred: float, target: float) -> float:
+    if target == 0:
+        return float("inf")
+    return abs(pred - target) / abs(target)
+
+
+def mean_relative_accuracy(pred: float, target: float,
+                           start: float = 0.5,
+                           end: float = 0.95,
+                           interval: float = 0.05) -> float:
+    num_pts = (end - start) / interval + 2
+    conf_intervs = np.linspace(start, end, int(num_pts))
+    accuracy = abs_dist_norm(pred, target) <= 1 - conf_intervs
+    return float(accuracy.mean())
+
+
+def is_direction_task(question_type: str) -> bool:
+    return any(question_type.startswith(p) for p in DIRECTION_PREFIXES)
+
+
+def report_key(question_type: str) -> str:
+    for key, members in COMPOSITE_METRICS.items():
+        if question_type in members:
+            return key
+    return question_type
+
+
+def strengthen_prompt(prompt: str, question_type: str) -> str:
+    prompt = str(prompt or "").strip()
+    if question_type in NUMERICAL_TASKS:
+        if NUMERIC_STRICT_INSTRUCTION in prompt:
+            return prompt
+        prompt = LOOSE_NUMERIC_INSTRUCTION_RE.sub("\n", prompt).strip()
+        return prompt.rstrip() + "\n" + NUMERIC_STRICT_INSTRUCTION
+    if question_type in MC_TASKS or is_direction_task(question_type):
+        lower = prompt.lower()
+        if "option" in lower and "letter" in lower:
+            return prompt
+        return prompt.rstrip() + "\n" + MC_STRICT_INSTRUCTION
+    return prompt
+
+
+def compute_sample_score(question_type: str, prediction: str,
+                         ground_truth: Any) -> Tuple[str, float]:
+    cleaned = clean_prediction_for_task(question_type, prediction)
+    if question_type in NUMERICAL_TASKS:
+        pred_val = to_float(cleaned)
+        gt_val = to_float(ground_truth)
+        if pred_val is None or gt_val is None or gt_val == 0:
+            return "MRA", 0.0
+        return "MRA", mean_relative_accuracy(pred_val, gt_val)
+    if question_type in MC_TASKS or is_direction_task(question_type):
+        return "ACC", exact_match(cleaned, ground_truth)
+    return "UNK", 0.0
+
+
+def load_rows(path: str) -> List[Dict[str, Any]]:
+    if path.endswith(".parquet"):
+        import pandas as pd
+
+        return pd.read_parquet(path).to_dict(orient="records")
+    return load_json_records(path)
+
+
+def get_image_list(item: Dict[str, Any]) -> List[str]:
+    imgs = item.get("image_list") or item.get("images") or item.get("image") or []
+    if isinstance(imgs, str):
+        imgs = [imgs]
+    return [str(p) for p in imgs if p]
+
+
+def get_video_list(item: Dict[str, Any]) -> List[str]:
+    vids = item.get("videos") or item.get("video") or []
+    if isinstance(vids, str):
+        vids = [vids]
+    return [str(p) for p in vids if p]
+
+
+def infer_video_path(item: Dict[str, Any], video_root: str = "") -> str:
+    """Resolve ``ReVSI/<budget>_frame/<scene_id>.mp4``."""
+    vids = get_video_list(item)
+    if vids:
+        p = vids[0]
+        return os.path.join(video_root, p) if video_root and not os.path.isabs(p) else p
+
+    raw = item.get("raw") if isinstance(item.get("raw"), dict) else item
+    scene = str(
+        item.get("scene_id")
+        or item.get("scene_name")
+        or raw.get("scene_id")
+        or raw.get("scene_name")
+        or ""
+    ).strip()
+    if not scene:
+        return ""
+    frame_budget = str(item.get("num_frames") or raw.get("num_frames") or "").strip()
+    frame_budget = re.sub(r"[_-]?frame$", "", frame_budget, flags=re.IGNORECASE)
+    if not frame_budget:
+        return ""
+    filename = scene if scene.endswith(".mp4") else f"{scene}.mp4"
+    relative = os.path.join(f"{frame_budget}_frame", filename)
+    return os.path.join(video_root, relative) if video_root else relative
+
+
+def get_item_id(item: Dict[str, Any], idx: int) -> str:
+    for key in ("id", "item_id"):
+        if item.get(key) is not None:
+            return str(item[key])
+    return f"idx_{idx}"
+
+
+# ---------------------------------------------------------------------------
+# Train / Eval prompt alignment.
+#
+# We rewrite the eval prompts so they match the *new training* prompts
+# byte-for-byte. The training prompts were rewritten by
+# scripts/normalize_sft_prompts.py to:
+#
+#  - drop the verbose "Please answer the question using a single word or
+#    phrase." instruction
+#  - use a concise, format-anchored sentence on numeric tasks:
+#      abs_distance : "...? Answer with a number in meters (e.g. 2.3)."
+#      counting     : "...? Answer with an integer (e.g. 3)."
+#      size         : "...? Answer with a number (e.g. 120)."
+#      room_size    : "...? Answer with a number (e.g. 25.5)."
+#  - simplify size: "the length of the longest dimension (length, width, or
+#    height) of X, measured in centimeters" -> "the longest dimension
+#    (length, width, or height) of X in centimeters"
+#  - normalise rel_distance to the "(a, b, c, d)" + "If there are multiple
+#    instances ..." form (matches VSI-Bench eval).
+#  - keep route_planning steps on separate lines.
+#
+# All rewrites are idempotent.
+# ---------------------------------------------------------------------------
+
+# ----- numeric tasks --------------------------------------------------------
+
+_NUM_TAIL_OLD = re.compile(
+    r"\s*Please answer the question using a single word or phrase\.\s*$",
+    flags=re.IGNORECASE,
+)
+
+# abs_distance
+_ABS_DIST_NEW_TAIL = (
+    "? Answer with a number in meters within <answer>...</answer> tags. "
+    "e.g. <answer>2.3</answer>"
+)
+_ABS_DIST_RE = re.compile(
+    r"(Measuring from the closest point of each object, what is the )"
+    r"(?:direct )?(distance between the [^?]+? and the [^?]+?)"
+    r"(?:\s*\(in meters\))?\?",
+    flags=re.IGNORECASE,
+)
+
+# counting
+_COUNTING_NEW_TAIL = (
+    "? Answer with an integer within <answer>...</answer> tags. "
+    "e.g. <answer>3</answer>"
+)
+_COUNTING_RE = re.compile(
+    r"(How many [^?]+ are in this room)\?",
+    flags=re.IGNORECASE,
+)
+
+# size
+_SIZE_LONG_RE = re.compile(
+    r"What is the length of the longest dimension \(length, width, or height\) "
+    r"of (?P<obj>the [^,?]+?), measured in centimeters\?",
+    flags=re.IGNORECASE,
+)
+# Short form already used in training-rewritten samples.
+_SIZE_SHORT_RE = re.compile(
+    r"What is the longest dimension \(length, width, or height\) "
+    r"of (?P<obj>the [^?]+?) in centimeters\?",
+    flags=re.IGNORECASE,
+)
+_SIZE_NEW_TAIL = (
+    " Answer with a number in centimeters within <answer>...</answer> tags. "
+    "e.g. <answer>120</answer>"
+)
+
+# room_size
+_ROOM_SIZE_NEW = (
+    "What is the size of this room in square meters? "
+    "If multiple rooms are shown, estimate the combined size. "
+    "Answer with a number in square meters within <answer>...</answer> tags. "
+    "e.g. <answer>25.5</answer>"
+)
+_ROOM_SIZE_OLD_RE = re.compile(
+    r"What is the size of this room \(in square meters\)\?"
+    r"[ \t]*\n?[ \t]*"
+    r"(?:If multiple rooms are shown, estimate the size of the combined space\.)?",
+    flags=re.IGNORECASE,
+)
+
+# Generic detector for "already aligned with new tail" so all 4 numeric
+# rewriters short-circuit.
+_HAS_NEW_ANSWER_TAG_HINT_RE = re.compile(
+    r"<answer>\.\.\.</answer>\s*tags",
+    flags=re.IGNORECASE,
+)
+
+# MC: rewrite legacy direct-answer tails to the joint-SFT answer-only format.
+_MC_OLD_TAIL_RE = re.compile(
+    r"\s*(?:"
+    r"Answer with the option'?s letter from the given choices directly\.?|"
+    r"Answer with the option letter within <answer>\.\.\.</answer> tags\.?\s*(?:e\.g\.|Example:)?\s*<answer>A</answer>|"
+    r"Output only the option letter inside <answer>\.\.\.</answer>\.?\s*Do not explain\.?|"
+    r"Choose the best answer from the options\. Put exactly one uppercase option letter inside <answer>\.\.\.</answer>\s*Do not explain\. Example: <answer>A</answer>"
+    r")\s*$",
+    flags=re.IGNORECASE,
+)
+_MC_NEW_TAIL = (
+    "Choose the best answer from the options. "
+    "Put exactly one uppercase option letter inside <answer>...</answer> "
+    "Do not explain. Example: <answer>A</answer>"
+)
+
+# ----- rel_distance / route_planning ----------------------------------------
+
+_REL_DIST_TRIGGER_RE = re.compile(
+    r"which\s+of\s+these\s+objects\s*\([^)]*\)\s+is\s+the\s+closest\s+to\s+",
+    flags=re.IGNORECASE,
+)
+_REL_DIST_INSERT_RE = re.compile(
+    r"(\?)(\s*\n?\s*)(Options\s*:)", flags=re.IGNORECASE
+)
+_REL_DIST_MULTI_INSTANCE_LINE = (
+    "If there are multiple instances of an object category, measure to the closest."
+)
+
+_ROUTE_BLOCK_RE = re.compile(
+    r"(turn right\.'\)\s*:)\s*(.*?)(\s*You have reached the final destination\.)",
+    flags=re.IGNORECASE | re.DOTALL,
+)
+_ROUTE_STEP_FINDALL_RE = re.compile(r"\d+\.\s+\S")
+
+
+def _fix_route_planning(prompt: str) -> str:
+    def repl(m):
+        head, body, tail = m.group(1), m.group(2), m.group(3)
+        positions = [mt.start() for mt in _ROUTE_STEP_FINDALL_RE.finditer(body)]
+        if not positions:
+            return m.group(0)
+        positions.append(len(body))
+        parts = [body[positions[i]:positions[i + 1]].strip()
+                 for i in range(len(positions) - 1)]
+        parts = [p for p in parts if p]
+        if not parts:
+            return m.group(0)
+        return head + "\n" + "\n".join(parts) + "\n" + tail.lstrip()
+
+    return _ROUTE_BLOCK_RE.sub(repl, prompt, count=1)
+
+
+def _rewrite_abs_distance(prompt: str) -> str:
+    """Rewrite to "...what is the direct distance between X and Y?
+    Answer with a number in meters within <answer>...</answer> tags. ..."."""
+    if "Answer with a number in meters within <answer>" in prompt:
+        return prompt
+    # Strip any older tails (legacy "Please answer ..." or the previous
+    # format-anchored "e.g. 2.3" form).
+    prompt = _NUM_TAIL_OLD.sub("", prompt)
+    prompt = re.sub(r"\s*Answer with a number in meters \(e\.g\.[^)]*\)\.\s*$",
+                    "", prompt, flags=re.IGNORECASE)
+    new_prompt = _ABS_DIST_RE.sub(
+        lambda m: m.group(1) + "direct " + m.group(2) + _ABS_DIST_NEW_TAIL,
+        prompt,
+        count=1,
+    )
+    return new_prompt
+
+
+def _rewrite_counting(prompt: str) -> str:
+    if "Answer with an integer within <answer>" in prompt:
+        return prompt
+    prompt = _NUM_TAIL_OLD.sub("", prompt)
+    prompt = re.sub(r"\s*Answer with an integer \(e\.g\.[^)]*\)\.\s*$",
+                    "", prompt, flags=re.IGNORECASE)
+    return _COUNTING_RE.sub(lambda m: m.group(1) + _COUNTING_NEW_TAIL,
+                            prompt, count=1)
+
+
+def _rewrite_size(prompt: str) -> str:
+    if "Answer with a number in centimeters within <answer>" in prompt:
+        return prompt
+    prompt = _NUM_TAIL_OLD.sub("", prompt)
+    prompt = re.sub(r"\s*Answer with a number \(e\.g\.[^)]*\)\.\s*$",
+                    "", prompt, flags=re.IGNORECASE)
+    # Long form -> new short form with anchor.
+    new_prompt, n = _SIZE_LONG_RE.subn(
+        lambda m: ("What is the longest dimension (length, width, or height) "
+                   "of " + m.group("obj") + " in centimeters?" + _SIZE_NEW_TAIL),
+        prompt,
+        count=1,
+    )
+    if n:
+        return new_prompt
+    # Short form (no "measured in centimeters") -> just add the anchor.
+    return _SIZE_SHORT_RE.sub(
+        lambda m: ("What is the longest dimension (length, width, or height) "
+                   "of " + m.group("obj") + " in centimeters?" + _SIZE_NEW_TAIL),
+        prompt,
+        count=1,
+    )
+
+
+def _rewrite_room_size(prompt: str) -> str:
+    if "Answer with a number in square meters within <answer>" in prompt:
+        return prompt
+    prompt = _NUM_TAIL_OLD.sub("", prompt)
+    prompt = re.sub(r"\s*Answer with a number \(e\.g\.[^)]*\)\.\s*$",
+                    "", prompt, flags=re.IGNORECASE)
+    return _ROOM_SIZE_OLD_RE.sub(_ROOM_SIZE_NEW, prompt, count=1)
+
+
+def _rewrite_mc_tail(prompt: str) -> str:
+    """Replace "Answer with the option's letter from the given choices
+    directly." -> the new <answer>-tagged version."""
+    if _MC_NEW_TAIL in prompt:
+        return prompt
+    if _MC_OLD_TAIL_RE.search(prompt):
+        return _MC_OLD_TAIL_RE.sub("\n" + _MC_NEW_TAIL, prompt, count=1)
+    return prompt
+
+
+def align_prompt_to_training(question_type: str, prompt: str) -> str:
+    """Rewrite eval prompts to match the (new) training phrasing.
+
+    Idempotent: applying twice yields the same result.
+    """
+    if not prompt:
+        return prompt
+
+    if question_type == "object_abs_distance":
+        return _rewrite_abs_distance(prompt)
+
+    if question_type == "object_counting":
+        return _rewrite_counting(prompt)
+
+    if question_type == "object_size_estimation":
+        return _rewrite_size(prompt)
+
+    if question_type == "room_size_estimation":
+        return _rewrite_room_size(prompt)
+
+    if question_type == "object_rel_distance":
+        if _REL_DIST_TRIGGER_RE.search(prompt) and (
+            _REL_DIST_MULTI_INSTANCE_LINE not in prompt
+        ):
+            prompt = _REL_DIST_INSERT_RE.sub(
+                lambda m: m.group(1) + "\n" + _REL_DIST_MULTI_INSTANCE_LINE
+                + "\n" + m.group(3),
+                prompt,
+                count=1,
+            )
+        return _rewrite_mc_tail(prompt)
+
+    if question_type == "route_planning":
+        if "turn right.')" in prompt.lower() and "\n1." not in prompt:
+            prompt = _fix_route_planning(prompt)
+        return _rewrite_mc_tail(prompt)
+
+    # All other MC tasks (rel_direction_*, obj_appearance_order) only need
+    # the tail rewrite.
+    return _rewrite_mc_tail(prompt)
+
+
+# ---------------------------------------------------------------------------
+# Optional strict-numeric instruction (path A diagnostic).
+#
+# Background: a long SFT on the 87k mixture (numeric ~17k vs MC/letter ~70k)
+# pushes the model to emit single letters (mostly "B") even on numeric tasks,
+# despite the training prompt's "Please answer ... single word or phrase."
+# This helper appends an *additional* sentence that explicitly forbids
+# letters, so we can A/B test whether the model still has numeric capability
+# but is just being triggered into MC-mode.
+#
+# Toggled via the --strict_numeric_prompt CLI flag (default OFF).
+# ---------------------------------------------------------------------------
+STRICT_NUMERIC_SUFFIX = (
+    "Output ONLY a single integer or decimal number (no units, no letters, no "
+    "punctuation, no explanation)."
+)
+
+
+def apply_strict_numeric_suffix(prompt: str) -> str:
+    if not prompt:
+        return prompt
+    if STRICT_NUMERIC_SUFFIX in prompt:
+        return prompt
+    return prompt.rstrip() + "\n" + STRICT_NUMERIC_SUFFIX
+
+
+def normalise_item(item: Dict[str, Any], idx: int,
+                   strict_numeric: bool = False,
+                   video_root: str = "") -> Dict[str, Any]:
+    question = str(item.get("question") or "").strip()
+    gt = item.get("ground_truth")
+    qtype = str(item.get("question_type") or "unknown")
+    raw_options = item.get("options")
+    if raw_options is None:
+        options = []
+    elif isinstance(raw_options, np.ndarray):
+        options = raw_options.tolist()
+    elif isinstance(raw_options, (list, tuple)):
+        options = list(raw_options)
+    else:
+        options = [raw_options]
+    options = [str(option) for option in options if option is not None]
+
+    prompt_parts = ["These are frames of a video.", question]
+    if qtype in MC_TASKS:
+        prompt_parts.append("Options:\n" + "\n".join(options))
+        prompt_parts.append(
+            "Answer with the option's letter from the given choices directly."
+        )
+    elif qtype in NUMERICAL_TASKS:
+        prompt_parts.append(
+            "Answer the question using a single integer or decimal number."
+        )
+    prompt = "\n".join(part for part in prompt_parts if part).strip()
+    if strict_numeric and qtype in NUMERICAL_TASKS:
+        prompt = apply_strict_numeric_suffix(prompt)
+
+    scene_id = str(item.get("scene_id") or "").strip()
+    out = {
+        "id": get_item_id(item, idx),
+        "dataset": item.get("dataset", "revsi"),
+        "scene_id": scene_id,
+        "scene_name": scene_id,
+        "num_frames": str(item.get("num_frames") or "").strip(),
+        "question_type": qtype,
+        "question": question,
+        "prompt": prompt,
+        "ground_truth": "" if gt is None else str(gt),
+        "options": options,
+        "image_list": get_image_list(item),
+        "videos": get_video_list(item),
+        "raw": item,
+    }
+    out["video_path"] = infer_video_path(out, video_root)
+    return out
+
+
+def build_conversation(item: Dict[str, Any]) -> List[Dict[str, Any]]:
+    content = []
+    for img_path in item["image_list"]:
+        content.append({
+            "type": "image_url",
+            "image_url": {"url": f"file://{img_path}"},
+        })
+    content.append({"type": "text", "text": item["prompt"]})
+    return [
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": content},
+    ]
+
+
+@lru_cache(maxsize=4096)
+def get_video_frame_count(path: str) -> int:
+    """Read frame-count metadata without decoding the full video."""
+
+    from decord import VideoReader, cpu
+
+    return len(VideoReader(path, ctx=cpu(0), num_threads=1))
+
+
+def build_video_messages(item: Dict[str, Any], args) -> List[Dict[str, Any]]:
+    video_item: Dict[str, Any] = {
+        "type": "video",
+        "video": item["video_path"],
+        "total_pixels": args.video_total_pixels,
+    }
+    if args.exact_nframes:
+        total_frames = get_video_frame_count(item["video_path"])
+        # qwen_vl_utils requires nframes <= total_frames and a multiple of two.
+        # ReVSI all-frame videos are not uniformly long enough for a literal
+        # 128-frame request, so short videos use every available even frame.
+        nframes = min(args.max_frames, total_frames)
+        nframes -= nframes % 2
+        if nframes < 2:
+            raise ValueError(
+                f"Video has fewer than two usable frames: {item['video_path']}"
+            )
+        video_item["nframes"] = nframes
+    else:
+        video_item["max_frames"] = args.max_frames
+        video_item["fps"] = args.fps
+    if args.video_min_pixels is not None:
+        video_item["min_pixels"] = args.video_min_pixels
+    if args.video_max_pixels is not None:
+        video_item["max_pixels"] = args.video_max_pixels
+    return [
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": [video_item, {"type": "text", "text": item["prompt"]}]},
+    ]
+
+
+def prepare_video_for_vllm(messages: List[Dict[str, Any]], processor, patch_size: int,
+                           enable_thinking: bool = False) -> Dict[str, Any]:
+    """Client-side video decoding path for vLLM.
+
+    Mirrors existing STVG/Timelens eval: apply chat template, decode video via
+    qwen_vl_utils, then pass multi_modal_data to vLLM.generate.
+    """
+    from qwen_vl_utils import process_vision_info
+
+    text = processor.apply_chat_template(
+        messages,
+        tokenize=False,
+        add_generation_prompt=True,
+        enable_thinking=enable_thinking,
+    )
+    _images, video_inputs, video_kwargs = process_vision_info(
+        messages,
+        image_patch_size=patch_size,
+        return_video_kwargs=True,
+        return_video_metadata=True,
+    )
+    video_kwargs = video_kwargs or {}
+    video_kwargs["do_resize"] = False
+
+    llm_input: Dict[str, Any] = {"prompt": text}
+    if video_inputs:
+        llm_input["multi_modal_data"] = {"video": video_inputs}
+        llm_input["mm_processor_kwargs"] = video_kwargs
+    return llm_input
+
+
+def summarise(records: Iterable[Dict[str, Any]]) -> Dict[str, Any]:
+    qtype_scores = defaultdict(list)
+    for r in records:
+        qtype = r.get("question_type", "unknown")
+        score = float(r.get("score", 0.0) or 0.0)
+        qtype_scores[qtype].append(score)
+
+    task_scores = {}
+    task_means = {}
+    for display, key, metric in REPORT_ORDER:
+        members = COMPOSITE_METRICS.get(key, (key,))
+        member_scores = [
+            float(np.mean(qtype_scores[member]))
+            for member in members
+            if qtype_scores.get(member)
+        ]
+        if member_scores:
+            task_mean = float(np.mean(member_scores))
+            task_means[key] = task_mean
+            task_scores[key] = {
+                "display": display,
+                "metric": metric,
+                # ReVSI first averages each fine-grained question type, then
+                # macro-averages related types into the reported category.
+                "score": round(task_mean * 100, 2),
+                "count": sum(len(qtype_scores.get(member, [])) for member in members),
+            }
+
+    num_keys = {"object_counting", "object_abs_distance", "object_size_estimation", "room_size_estimation"}
+    mc_keys = {"object_rel_distance", "object_rel_direction", "route_planning"}
+    num_vals = [value for key, value in task_means.items() if key in num_keys]
+    mc_vals = [value for key, value in task_means.items() if key in mc_keys]
+    all_vals = list(task_means.values())
+
+    out = {
+        "total": sum(len(v) for v in qtype_scores.values()),
+        "task_scores": task_scores,
+        "question_type_scores": {
+            k: {"score": round(float(np.mean(v) * 100), 2), "count": len(v)}
+            for k, v in sorted(qtype_scores.items())
+        },
+    }
+    if num_vals:
+        out["numerical_avg"] = round(float(np.mean(num_vals) * 100), 2)
+    if mc_vals:
+        out["mc_avg"] = round(float(np.mean(mc_vals) * 100), 2)
+    if all_vals:
+        out["overall_avg"] = round(float(np.mean(all_vals) * 100), 2)
+    return out
+
+
+def print_summary(summary: Dict[str, Any], prefix: str = "") -> None:
+    print(prefix + "=" * 58, flush=True)
+    print(prefix + f"ReVSI Results  (samples={summary.get('total', 0)})", flush=True)
+    print(prefix + "=" * 58, flush=True)
+    print(prefix + f"{'Task':<16} {'Metric':<6} {'Score':>8} {'Count':>7}", flush=True)
+    print(prefix + "-" * 42, flush=True)
+    for _display, key, _metric in REPORT_ORDER:
+        if key not in summary.get("task_scores", {}):
+            continue
+        s = summary["task_scores"][key]
+        print(prefix + f"{s['display']:<16} {s['metric']:<6} {s['score']:>7.2f}% {s['count']:>7}", flush=True)
+    print(prefix + "-" * 42, flush=True)
+    if "numerical_avg" in summary:
+        print(prefix + f"{'Numerical Avg':<16} {'MRA':<6} {summary['numerical_avg']:>7.2f}%", flush=True)
+    if "mc_avg" in summary:
+        print(prefix + f"{'MC Avg':<16} {'ACC':<6} {summary['mc_avg']:>7.2f}%", flush=True)
+    if "overall_avg" in summary:
+        print(prefix + f"{'Overall Avg':<16} {'---':<6} {summary['overall_avg']:>7.2f}%", flush=True)
+    print(prefix + "=" * 58, flush=True)
+
+
+def parse_args():
+    p = argparse.ArgumentParser(description="ReVSI evaluation via vLLM")
+    p.add_argument("--model_path", required=True)
+    p.add_argument("--qa_file", required=True)
+    p.add_argument("--output_json_path", required=True)
+    p.add_argument("--resume_dir", default="", help="Existing output dir; skip all ids already present in results_shard*.jsonl before re-sharding remaining samples.")
+    p.add_argument("--task_filter", default="", help="Comma-separated question_type filter.")
+    p.add_argument("--rank", type=int, default=0)
+    p.add_argument("--world_size", type=int, default=1)
+    p.add_argument("--tensor_parallel_size", type=int, default=1)
+    p.add_argument("--max_model_len", type=int, default=32768)
+    p.add_argument("--gpu_memory_utilization", type=float, default=0.90)
+    p.add_argument("--max_new_tokens", type=int, default=64)
+    p.add_argument("--batch_size", type=int, default=16)
+    p.add_argument("--score_log_interval", type=int, default=200)
+    p.add_argument("--temperature", type=float, default=0.0)
+    p.add_argument("--top_p", type=float, default=1.0)
+    p.add_argument("--max_samples", type=int, default=0)
+    p.add_argument("--media_mode", choices=["image", "video"], default="video",
+                   help="image: use image_list; video: reconstruct/read mp4 and sample frames.")
+    p.add_argument("--video_root", default="",
+                   help="Root directory for relative video paths, e.g. VSI-590K root.")
+    p.add_argument("--max_frames", type=int, default=128)
+    p.add_argument("--fps", type=float, default=2.0)
+    p.add_argument(
+        "--exact_nframes",
+        action="store_true",
+        help="Decode exactly max_frames frames; use for fixed ReVSI subsets.",
+    )
+    p.add_argument("--video_total_pixels", type=int, default=16777216)
+    p.add_argument("--video_min_pixels", type=int, default=65536)
+    p.add_argument("--video_max_pixels", type=int, default=None)
+    p.add_argument(
+        "--strict_numeric_prompt",
+        action="store_true",
+        help="Append a strict 'output only a number' suffix to numeric-task "
+             "prompts. Useful when a long SFT has biased the model toward "
+             "single-letter outputs on numeric questions.",
+    )
+    p.add_argument(
+        "--enable_thinking",
+        type=str,
+        default="false",
+        choices=["true", "false"],
+        help="Enable Qwen3 <think> block. Default: false (no_think). "
+             "When false, vLLM is told via chat_template_kwargs to skip the "
+             "<think>...</think> reasoning prefix so the model emits the "
+             "<answer>...</answer> answer directly.",
+    )
+    return p.parse_args()
+
+
+def main():
+    args = parse_args()
+    rows = load_rows(args.qa_file)
+    items = [normalise_item(r, i, strict_numeric=args.strict_numeric_prompt,
+                            video_root=args.video_root)
+             for i, r in enumerate(rows)]
+    if args.task_filter:
+        keep = {x.strip() for x in args.task_filter.split(",") if x.strip()}
+        items = [x for x in items if x["question_type"] in keep]
+        print(f"Task filter: {sorted(keep)} -> {len(items)} samples", flush=True)
+    if args.max_samples and len(items) > args.max_samples:
+        items = items[:args.max_samples]
+
+    # Global resume mode: remove all IDs already written by any previous shard,
+    # then redistribute the remaining samples across the current world_size.
+    if args.resume_dir:
+        done_global = set()
+        for name in os.listdir(args.resume_dir) if os.path.isdir(args.resume_dir) else []:
+            if not (name.startswith("results_shard") and name.endswith(".jsonl")):
+                continue
+            p = os.path.join(args.resume_dir, name)
+            with open(p, "r", encoding="utf-8") as f:
+                for line in f:
+                    if not line.strip():
+                        continue
+                    try:
+                        done_global.add(str(json.loads(line).get("id")))
+                    except Exception:
+                        pass
+        if done_global:
+            before = len(items)
+            items = [x for x in items if str(x["id"]) not in done_global]
+            print(f"Global resume: skip {before - len(items)} done ids from {args.resume_dir}; remaining={len(items)}", flush=True)
+
+    # Contiguous sharding: matches official VSI script.
+    total = len(items)
+    chunk_size = total // args.world_size
+    remainder = total % args.world_size
+    start = args.rank * chunk_size + min(args.rank, remainder)
+    end = start + chunk_size + (1 if args.rank < remainder else 0)
+    shard_items = items[start:end]
+
+    # Resume: skip IDs already present in the shard output.
+    done = set()
+    if os.path.exists(args.output_json_path):
+        with open(args.output_json_path, "r", encoding="utf-8") as f:
+            for line in f:
+                if line.strip():
+                    try:
+                        done.add(json.loads(line)["id"])
+                    except Exception:
+                        pass
+    shard_items = [x for x in shard_items if x["id"] not in done]
+
+    if not shard_items:
+        print(f"rank={args.rank}: no remaining samples", flush=True)
+        return
+
+    if args.media_mode == "video":
+        missing = [x for x in shard_items if not x.get("video_path") or not os.path.isfile(x["video_path"])]
+        if missing:
+            ex = missing[0]
+            raise FileNotFoundError(
+                f"video_mode missing {len(missing)} videos; first id={ex['id']} "
+                f"dataset={ex['dataset']} scene={ex['scene_name']} path={ex.get('video_path')!r}"
+            )
+
+    os.makedirs(os.path.dirname(args.output_json_path), exist_ok=True)
+
+    # vLLM local-media permission root for image-url mode.
+    all_img_paths = [p for item in shard_items for p in item["image_list"]]
+    common_prefix = os.path.commonpath(all_img_paths) if all_img_paths else "/"
+    if not os.path.isdir(common_prefix):
+        common_prefix = os.path.dirname(common_prefix)
+    num_images = max((len(x["image_list"]) for x in shard_items), default=0)
+
+    from vllm import LLM, SamplingParams
+
+    print("Initializing vLLM engine...", flush=True)
+    print(f"Model: {args.model_path}", flush=True)
+    print(f"QA: {args.qa_file}", flush=True)
+    print(f"Rank: {args.rank}/{args.world_size}, samples={len(shard_items)} / total={total}", flush=True)
+    print(f"Media mode: {args.media_mode}", flush=True)
+    if args.media_mode == "video":
+        print(f"Video: root={args.video_root} max_frames={args.max_frames} "
+              f"exact_nframes={args.exact_nframes} fps={args.fps} "
+              f"total_pixels={args.video_total_pixels}", flush=True)
+    else:
+        print(f"Images per prompt max: {num_images}, allowed_media={common_prefix}", flush=True)
+    enable_thinking = (str(args.enable_thinking).lower() == "true")
+    print(f"Thinking: {enable_thinking}", flush=True)
+
+    processor = None
+    patch_size = None
+    if args.media_mode == "video":
+        from transformers import AutoProcessor, AutoTokenizer
+        processor = AutoProcessor.from_pretrained(
+            args.model_path, padding_side="left", do_resize=False,
+            trust_remote_code=True)
+        tokenizer = AutoTokenizer.from_pretrained(args.model_path, trust_remote_code=True)
+        tokenizer.padding_side = "left"
+        processor.tokenizer = tokenizer
+        patch_size = processor.image_processor.patch_size
+        print(f"Patch size: {patch_size}", flush=True)
+
+    mm_limit = {"video": 1, "image": 1} if args.media_mode == "video" else {"image": num_images}
+    llm_kwargs = dict(
+        model=args.model_path,
+        tensor_parallel_size=args.tensor_parallel_size,
+        max_model_len=args.max_model_len,
+        gpu_memory_utilization=args.gpu_memory_utilization,
+        trust_remote_code=True,
+        limit_mm_per_prompt=mm_limit,
+    )
+    if args.media_mode == "image":
+        llm_kwargs["allowed_local_media_path"] = common_prefix
+
+    llm = LLM(**llm_kwargs)
+    sampling = SamplingParams(
+        max_tokens=args.max_new_tokens,
+        temperature=args.temperature,
+        top_p=args.top_p,
+    )
+    # Passed through to the tokenizer's apply_chat_template inside vLLM.
+    # Qwen3 chat templates expose `enable_thinking` to skip the
+    # `<think>...</think>` reasoning prefix when set to False.
+    chat_template_kwargs = {"enable_thinking": enable_thinking}
+
+    t0 = time.time()
+    local_records: List[Dict[str, Any]] = []
+    with open(args.output_json_path, "a", encoding="utf-8") as f:
+        for batch_start in tqdm(range(0, len(shard_items), args.batch_size),
+                                desc=f"rank{args.rank}"):
+            batch = shard_items[batch_start:batch_start + args.batch_size]
+            if args.media_mode == "video":
+                conversations = [build_video_messages(x, args) for x in batch]
+                llm_inputs = [prepare_video_for_vllm(
+                    m, processor, patch_size, enable_thinking=enable_thinking)
+                    for m in conversations]
+                outputs = llm.generate(llm_inputs, sampling_params=sampling)
+            else:
+                conversations = [build_conversation(x) for x in batch]
+                outputs = llm.chat(
+                    conversations,
+                    sampling_params=sampling,
+                    chat_template_kwargs=chat_template_kwargs,
+                )
+            for item, output in zip(batch, outputs):
+                pred = output.outputs[0].text
+                metric, score = compute_sample_score(
+                    item["question_type"], pred, item["ground_truth"])
+                record = {
+                    "id": item["id"],
+                    "dataset": item["dataset"],
+                    "scene_id": item["scene_id"],
+                    "scene_name": item["scene_name"],
+                    "num_frames": item["num_frames"],
+                    "question_type": item["question_type"],
+                    "question": item["question"],
+                    "prompt": item["prompt"],
+                    "pred": pred,
+                    "pred_clean": clean_prediction_for_task(item["question_type"], pred),
+                    "ground_truth": item["ground_truth"],
+                    "options": item.get("options"),
+                    "image_list": item["image_list"],
+                    "video_path": item.get("video_path"),
+                    "media_mode": args.media_mode,
+                    "score": score,
+                    "metric": metric,
+                }
+                f.write(json.dumps(record, ensure_ascii=False) + "\n")
+                local_records.append(record)
+            f.flush()
+
+            done_n = min(batch_start + args.batch_size, len(shard_items))
+            if done_n % args.score_log_interval == 0 or done_n == len(shard_items):
+                elapsed = max(time.time() - t0, 1e-6)
+                print(f"[{done_n}/{len(shard_items)}] {elapsed:.1f}s "
+                      f"({done_n / elapsed:.2f} items/s)", flush=True)
+                print_summary(summarise(local_records), prefix="  ")
+
+    print("FINAL SHARD SUMMARY", flush=True)
+    print_summary(summarise(local_records))
+
+
+if __name__ == "__main__":
+    main()

code/eval/task/revsi/merge_multinode_shards.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+"""Merge, validate, and officially summarize ReVSI vLLM shards."""
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from eval_revsi_vllm import print_summary, summarise  # noqa: E402
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--output-dir", type=Path, required=True)
+    parser.add_argument("--num-shards", type=int, required=True)
+    parser.add_argument("--expected-samples", type=int, default=0)
+    args = parser.parse_args()
+
+    missing = [
+        shard
+        for shard in range(args.num_shards)
+        if not (args.output_dir / f"results_shard{shard}.jsonl").is_file()
+    ]
+    if missing:
+        raise RuntimeError(
+            f"Missing {len(missing)}/{args.num_shards} ReVSI shards: {missing}"
+        )
+
+    records_by_id = {}
+    for shard in range(args.num_shards):
+        path = args.output_dir / f"results_shard{shard}.jsonl"
+        with path.open(encoding="utf-8") as handle:
+            for line_number, line in enumerate(handle, 1):
+                if not line.strip():
+                    continue
+                record = json.loads(line)
+                sample_id = record.get("id")
+                if sample_id is None:
+                    raise RuntimeError(f"{path}:{line_number}: record has no id")
+                records_by_id[str(sample_id)] = record
+
+    records = list(records_by_id.values())
+    records.sort(
+        key=lambda row: (
+            0,
+            int(row["id"]),
+        )
+        if str(row["id"]).isdigit()
+        else (1, str(row["id"]))
+    )
+    if args.expected_samples > 0 and len(records) != args.expected_samples:
+        raise RuntimeError(
+            f"Expected {args.expected_samples} unique ReVSI samples, "
+            f"got {len(records)}"
+        )
+
+    merged_path = args.output_dir / "merged_results.jsonl"
+    with merged_path.open("w", encoding="utf-8") as handle:
+        for record in records:
+            handle.write(json.dumps(record, ensure_ascii=False) + "\n")
+
+    summary = summarise(records)
+    summary["num_shards"] = args.num_shards
+    summary["frame_budgets"] = sorted(
+        {str(record.get("num_frames") or "") for record in records}
+    )
+    summary_path = args.output_dir / "summary.json"
+    summary_path.write_text(
+        json.dumps(summary, ensure_ascii=False, indent=2) + "\n",
+        encoding="utf-8",
+    )
+
+    print_summary(summary)
+    print(f"Merged: {merged_path}")
+    print(f"Summary: {summary_path}")
+
+
+if __name__ == "__main__":
+    main()

code/eval/task/revsi/run_eval_vllm.sh

@@ -0,0 +1,321 @@
+#!/bin/bash
+# =============================================================================
+# ReVSI evaluation launcher — vLLM, data-parallel only.
+#
+# Usage:
+#   bash eval/task/revsi/run_eval_vllm.sh [MODEL_PATH]
+#
+# Env overrides:
+#   REVSI_ROOT     root containing 16_frame/32_frame/64_frame/all_frame
+#   FRAME_BUDGET   16, 32, 64, or all (default: all)
+#   QA_FILE        defaults to FRAME_BUDGET/test-00000-of-00001.parquet
+#   OUTPUT_ROOT    default: outputs/revsi
+#   OUTPUT_DIR     optional existing run dir; set this to resume an interrupted eval
+#   TASK_FILTER    comma-separated question_type list, optional
+#   MAX_SAMPLES    optional quick debug cap
+#   TP_SIZE        default 1
+#   MAX_MODEL_LEN  default 32768
+#   MAX_NEW_TOKENS default 64
+#   BATCH_SIZE     default 16
+#   GPU_MEM_UTIL   default 0.90
+#   MEDIA_MODE     image (default) or video
+#   VIDEO_ROOT     root for mp4s in video mode, e.g. VSI-590K root
+#   MAX_FRAMES/FPS/VIDEO_TOTAL_PIXELS  video sampling controls
+# =============================================================================
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+PROJECT_DIR="$(cd "$SCRIPT_DIR/../../.." && pwd)"
+
+MODEL_PATH="${1:-${MODEL_PATH:-}}"
+REVSI_ROOT="${REVSI_ROOT:-}"
+FRAME_BUDGET="${FRAME_BUDGET:-all}"
+FRAME_DIR="${REVSI_ROOT:+${REVSI_ROOT}/${FRAME_BUDGET}_frame}"
+QA_FILE="${QA_FILE:-${FRAME_DIR:+${FRAME_DIR}/test-00000-of-00001.parquet}}"
+OUTPUT_ROOT="${OUTPUT_ROOT:-${PROJECT_DIR}/outputs/revsi}"
+TASK_FILTER="${TASK_FILTER:-}"
+MAX_SAMPLES="${MAX_SAMPLES:-0}"
+STRICT_NUMERIC_PROMPT="${STRICT_NUMERIC_PROMPT:-0}"
+ENABLE_THINKING="${ENABLE_THINKING:-false}"
+MEDIA_MODE="${MEDIA_MODE:-video}"
+VIDEO_ROOT="${VIDEO_ROOT:-${REVSI_ROOT}}"
+MAX_FRAMES="${MAX_FRAMES:-${FRAME_BUDGET/all/128}}"
+EXACT_NFRAMES="${EXACT_NFRAMES:-1}"
+FPS="${FPS:-2}"
+VIDEO_TOTAL_PIXELS="${VIDEO_TOTAL_PIXELS:-16777216}"
+VIDEO_MIN_PIXELS="${VIDEO_MIN_PIXELS:-65536}"
+VIDEO_MAX_PIXELS="${VIDEO_MAX_PIXELS:-}"
+
+TP_SIZE="${TP_SIZE:-1}"
+MAX_MODEL_LEN="${MAX_MODEL_LEN:-32768}"
+MAX_NEW_TOKENS="${MAX_NEW_TOKENS:-64}"
+BATCH_SIZE="${BATCH_SIZE:-16}"
+GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.90}"
+SCORE_LOG_INTERVAL="${SCORE_LOG_INTERVAL:-200}"
+GLOBAL_SHARD_COUNT="${GLOBAL_SHARD_COUNT:-}"
+GLOBAL_SHARD_OFFSET="${GLOBAL_SHARD_OFFSET:-0}"
+MERGE_SHARDS="${MERGE_SHARDS:-1}"
+EXPECTED_SAMPLES="${EXPECTED_SAMPLES:-0}"
+VLLM_BASE_PORT="${VLLM_BASE_PORT:-}"
+LAUNCH_DELAY="${LAUNCH_DELAY:-2}"
+
+case "${FRAME_BUDGET}" in
+    16|32|64|all) ;;
+    *)
+        echo "ERROR: FRAME_BUDGET must be 16, 32, 64, or all." >&2
+        exit 2
+        ;;
+esac
+[[ -n "${MODEL_PATH}" ]] || {
+    echo "ERROR: pass MODEL_PATH as the first argument or environment variable." >&2
+    exit 2
+}
+[[ -n "${QA_FILE}" ]] || {
+    echo "ERROR: set QA_FILE or REVSI_ROOT." >&2
+    exit 2
+}
+for path in "${MODEL_PATH}/config.json" "${QA_FILE}"; do
+    [[ -f "${path}" ]] || {
+        echo "ERROR: required input is unavailable: ${path}" >&2
+        exit 1
+    }
+done
+if [[ "${QA_FILE}" == *.parquet ]]; then
+    [[ -n "${FRAME_DIR}" ]] || {
+        echo "ERROR: parquet input requires REVSI_ROOT to locate frame videos." >&2
+        exit 2
+    }
+    compgen -G "${FRAME_DIR}/*.mp4" >/dev/null || {
+        echo "ERROR: no ReVSI videos found under ${FRAME_DIR}; extract video.zip first." >&2
+        exit 1
+    }
+fi
+
+# Prefer the CUDA runtime libraries installed alongside PyTorch. In particular,
+# pip/conda CUDA 12.9 builds need their matching nvJitLink ahead of an older
+# system CUDA toolkit that may already be present in LD_LIBRARY_PATH.
+NVJITLINK_LIB="$(
+    python - <<'PY'
+import site
+from pathlib import Path
+
+roots = [*site.getsitepackages(), site.getusersitepackages()]
+for root in roots:
+    candidate = Path(root) / "nvidia" / "nvjitlink" / "lib"
+    if candidate.is_dir():
+        print(candidate)
+        break
+PY
+)"
+if [[ -n "${NVJITLINK_LIB}" ]]; then
+    export LD_LIBRARY_PATH="${NVJITLINK_LIB}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
+fi
+if ! python -c "import torch; print(f'PyTorch preflight: {torch.__version__} CUDA {torch.version.cuda}')"; then
+    echo "ERROR: PyTorch CUDA libraries cannot be loaded in the active environment." >&2
+    exit 1
+fi
+
+IFS="," read -ra GPULIST <<< "${CUDA_VISIBLE_DEVICES:-$(seq -s, 0 $(($(nvidia-smi -L | wc -l)-1)))}"
+NUM_GPUS=${#GPULIST[@]}
+# Each vLLM subprocess must inherit only its worker-specific GPU mask.
+unset CUDA_VISIBLE_DEVICES
+if (( NUM_GPUS % TP_SIZE != 0 )); then
+    echo "ERROR: NUM_GPUS=$NUM_GPUS must be divisible by TP_SIZE=$TP_SIZE"
+    exit 1
+fi
+DP_SIZE=$(( NUM_GPUS / TP_SIZE ))
+if [[ -z "${VLLM_BASE_PORT}" ]]; then
+    VLLM_BASE_PORT="$(
+        python - "$DP_SIZE" <<'PY'
+import socket
+import sys
+
+count = int(sys.argv[1])
+spacing = 16
+for base in range(48000, 64000 - spacing * count, 128):
+    sockets = []
+    try:
+        for index in range(count):
+            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            sock.bind(("127.0.0.1", base + index * spacing))
+            sockets.append(sock)
+    except OSError:
+        for sock in sockets:
+            sock.close()
+        continue
+    for sock in sockets:
+        sock.close()
+    print(base)
+    break
+else:
+    raise SystemExit("no free ReVSI vLLM port block found")
+PY
+    )"
+fi
+
+MODEL_TAG=$(basename "${MODEL_PATH%/}")
+TIMESTAMP=$(date +%Y%m%d_%H%M%S)
+RUN_TAG="${RUN_TAG:-eval_revsi_${FRAME_BUDGET}frame_vllm-${MODEL_TAG}-${TIMESTAMP}}"
+# If OUTPUT_DIR points to an existing partial run, eval_revsi_vllm.py will skip
+# IDs already present in results_shard*.jsonl and continue the remaining samples.
+OUTPUT_DIR="${OUTPUT_DIR:-${OUTPUT_ROOT}/${RUN_TAG}}"
+mkdir -p "$OUTPUT_DIR"
+export MODEL_PATH QA_FILE VIDEO_ROOT FRAME_BUDGET MAX_FRAMES EXACT_NFRAMES FPS
+export VIDEO_TOTAL_PIXELS VIDEO_MIN_PIXELS VIDEO_MAX_PIXELS
+export MAX_MODEL_LEN MAX_NEW_TOKENS BATCH_SIZE TP_SIZE MAX_SAMPLES
+export EXPECTED_SAMPLES ENABLE_THINKING VLLM_BASE_PORT
+python - "$OUTPUT_DIR/run_config.json" <<'PY'
+import json
+import os
+import sys
+
+keys = (
+    "MODEL_PATH",
+    "QA_FILE",
+    "VIDEO_ROOT",
+    "FRAME_BUDGET",
+    "MAX_FRAMES",
+    "EXACT_NFRAMES",
+    "FPS",
+    "VIDEO_TOTAL_PIXELS",
+    "VIDEO_MIN_PIXELS",
+    "VIDEO_MAX_PIXELS",
+    "MAX_MODEL_LEN",
+    "MAX_NEW_TOKENS",
+    "BATCH_SIZE",
+    "TP_SIZE",
+    "MAX_SAMPLES",
+    "EXPECTED_SAMPLES",
+    "ENABLE_THINKING",
+    "VLLM_BASE_PORT",
+)
+payload = {key.lower(): os.environ.get(key, "") for key in keys}
+with open(sys.argv[1], "w", encoding="utf-8") as handle:
+    json.dump(payload, handle, indent=2, sort_keys=True)
+    handle.write("\n")
+PY
+
+cat <<EOF
+==============================================
+ReVSI Evaluation (vLLM, data-parallel)
+==============================================
+Model:        $MODEL_PATH
+QA file:      $QA_FILE
+Output:       $OUTPUT_DIR
+GPUs:         ${GPULIST[*]} (${NUM_GPUS} total, TP=${TP_SIZE}, DP=${DP_SIZE})
+Max tokens:   new=$MAX_NEW_TOKENS model_len=$MAX_MODEL_LEN
+Batch:        $BATCH_SIZE
+Task filter:  ${TASK_FILTER:-<none>}
+Max samples:  ${MAX_SAMPLES}
+Strict num:   ${STRICT_NUMERIC_PROMPT}
+Thinking:     ${ENABLE_THINKING}
+Media mode:   ${MEDIA_MODE}
+Video root:   ${VIDEO_ROOT:-<none>}
+Base port:    ${VLLM_BASE_PORT}
+Frame budget: ${FRAME_BUDGET} (exact_nframes=${EXACT_NFRAMES})
+Video:        max_frames=${MAX_FRAMES} fps=${FPS} total_pixels=${VIDEO_TOTAL_PIXELS}
+==============================================
+EOF
+
+PIDS=()
+cleanup() {
+    echo ""; echo "Caught interrupt, killing workers ..."
+    for pid in "${PIDS[@]}"; do kill -TERM "$pid" 2>/dev/null || true; done
+    wait 2>/dev/null || true
+    exit 1
+}
+trap cleanup INT TERM
+
+EFFECTIVE_SHARD_COUNT="${GLOBAL_SHARD_COUNT:-$DP_SIZE}"
+for IDX in $(seq 0 $((DP_SIZE - 1))); do
+    START=$(( IDX * TP_SIZE ))
+    GLOBAL_IDX=$((GLOBAL_SHARD_OFFSET + IDX))
+    SHARD_PORT=$((VLLM_BASE_PORT + IDX * 16))
+    SHARD_GPUS=""
+    for j in $(seq 0 $((TP_SIZE - 1))); do
+        g=${GPULIST[$((START + j))]}
+        SHARD_GPUS="${SHARD_GPUS}${SHARD_GPUS:+,}${g}"
+    done
+
+    OUT_JSONL="${OUTPUT_DIR}/results_shard${GLOBAL_IDX}.jsonl"
+    STRICT_FLAG=""
+    if [ "$STRICT_NUMERIC_PROMPT" = "1" ] || [ "$STRICT_NUMERIC_PROMPT" = "true" ]; then
+        STRICT_FLAG="--strict_numeric_prompt"
+    fi
+    VIDEO_MAX_PIXELS_ARGS=()
+    if [ -n "$VIDEO_MAX_PIXELS" ]; then
+        VIDEO_MAX_PIXELS_ARGS=(--video_max_pixels "$VIDEO_MAX_PIXELS")
+    fi
+    EXACT_NFRAMES_ARGS=()
+    if [ "$EXACT_NFRAMES" = "1" ] || [ "$EXACT_NFRAMES" = "true" ]; then
+        EXACT_NFRAMES_ARGS=(--exact_nframes)
+    fi
+    CUDA_VISIBLE_DEVICES="$SHARD_GPUS" \
+    VLLM_PORT="$SHARD_PORT" \
+    VLLM_HOST_IP=127.0.0.1 \
+    MASTER_PORT="$SHARD_PORT" \
+    MASTER_ADDR=127.0.0.1 \
+    PYTHONUNBUFFERED=1 \
+    python "${SCRIPT_DIR}/eval_revsi_vllm.py" \
+        --output_json_path "$OUT_JSONL" \
+        --model_path "$MODEL_PATH" \
+        --qa_file "$QA_FILE" \
+        --rank "$GLOBAL_IDX" \
+        --world_size "$EFFECTIVE_SHARD_COUNT" \
+        --tensor_parallel_size "$TP_SIZE" \
+        --max_model_len "$MAX_MODEL_LEN" \
+        --gpu_memory_utilization "$GPU_MEM_UTIL" \
+        --max_new_tokens "$MAX_NEW_TOKENS" \
+        --batch_size "$BATCH_SIZE" \
+        --score_log_interval "$SCORE_LOG_INTERVAL" \
+        --task_filter "$TASK_FILTER" \
+        --max_samples "$MAX_SAMPLES" \
+        --media_mode "$MEDIA_MODE" \
+        --video_root "$VIDEO_ROOT" \
+        --max_frames "$MAX_FRAMES" \
+        --fps "$FPS" \
+        --video_total_pixels "$VIDEO_TOTAL_PIXELS" \
+        --video_min_pixels "$VIDEO_MIN_PIXELS" \
+        "${VIDEO_MAX_PIXELS_ARGS[@]}" \
+        "${EXACT_NFRAMES_ARGS[@]}" \
+        --enable_thinking "$ENABLE_THINKING" \
+        $STRICT_FLAG \
+        > "$OUTPUT_DIR/worker_${GLOBAL_IDX}.log" 2>&1 &
+    PIDS+=($!)
+    echo "Launched shard $GLOBAL_IDX/$EFFECTIVE_SHARD_COUNT on GPU $SHARD_GPUS (PID ${PIDS[-1]})"
+    if [ "$IDX" -lt $((DP_SIZE - 1)) ] && [ "$LAUNCH_DELAY" -gt 0 ]; then
+        sleep "$LAUNCH_DELAY"
+    fi
+done
+
+echo "Waiting for ${DP_SIZE} workers ..."
+FAILED=0
+for i in "${!PIDS[@]}"; do
+    RC=0
+    wait "${PIDS[$i]}" || RC=$?
+    if [ $RC -ne 0 ]; then
+        echo "[FAIL] shard $i (PID ${PIDS[$i]}) exit=$RC"
+        FAILED=1
+    else
+        echo "[DONE] shard $i (PID ${PIDS[$i]})"
+    fi
+done
+
+if [ $FAILED -ne 0 ]; then
+    echo "ERROR: some workers failed; not merging incomplete shards." >&2
+    echo "Logs: $OUTPUT_DIR/worker_*.log" >&2
+    exit 1
+fi
+
+if [ "$MERGE_SHARDS" != "1" ]; then
+    echo "Local ReVSI shard range completed; centralized merge deferred."
+    exit 0
+fi
+
+python "${SCRIPT_DIR}/merge_multinode_shards.py" \
+    --output-dir "$OUTPUT_DIR" \
+    --num-shards "$EFFECTIVE_SHARD_COUNT" \
+    --expected-samples "$EXPECTED_SAMPLES"
+echo "Done: $OUTPUT_DIR"
+exit $FAILED

code/eval/task/segmentation/eval_seg_vllm.py

@@ -0,0 +1,446 @@
+#!/usr/bin/env python3
+"""
+Segmentation prompt evaluation with vLLM.
+
+This is the inference stage only.  It reads OneThinker-eval style records,
+asks the model for SAM2 prompts (`boxes`, `positive_points`, `negative_points`,
+and optional `time`), and writes `predicted_answer_norm` for the SAM2
+post-processing stage.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import sys
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Optional
+
+
+os.environ.setdefault("VLLM_USE_V1", "1")
+os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
+os.environ.setdefault("FORCE_QWENVL_VIDEO_READER", "decord")
+os.environ.setdefault("DECORD_EOF_RETRY_MAX", "20480")
+
+_PATCH_DIRS = (
+    Path(__file__).resolve().parents[1],
+    Path(__file__).resolve().parents[2] / "Eval-onethink",
+)
+if os.environ.get("FORCE_QWENVL_VIDEO_READER") == "decord":
+    for _patch_dir in _PATCH_DIRS:
+        if not (_patch_dir / "qwenvl_decord_patch.py").is_file():
+            continue
+        sys.path.insert(0, str(_patch_dir))
+        try:
+            import qwenvl_decord_patch  # noqa: F401
+        except Exception as exc:
+            print(f"[warn] failed to import qwenvl_decord_patch: {exc}", file=sys.stderr)
+        break
+
+import torch  # noqa: E402
+from qwen_vl_utils import process_vision_info  # noqa: E402
+from tqdm import tqdm  # noqa: E402
+from transformers import AutoProcessor  # noqa: E402
+from vllm import LLM, SamplingParams  # noqa: E402
+
+
+# Prompts — single source of truth in eval/task/eval_prompt.py
+# (training tails match data/joint/sft_joint_all.jsonl seg_image / seg_video).
+_EVAL_TASK_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+if _EVAL_TASK_DIR not in sys.path:
+    sys.path.insert(0, _EVAL_TASK_DIR)
+
+from canonical_data import (  # noqa: E402
+    load_json_records,
+    repository_relative_output_path,
+)
+from eval_prompt import (  # noqa: E402
+    TRAIN_SEG_IMAGE_TAIL,
+    TRAIN_SEG_VIDEO_TAIL,
+)
+
+
+ANSWER_RE = re.compile(r"<answer>\s*(.*?)\s*</answer>", re.DOTALL | re.IGNORECASE)
+
+
+def extract_answer(text: Optional[str]) -> str:
+    if not isinstance(text, str):
+        return ""
+    match = ANSWER_RE.search(text)
+    return match.group(1).strip() if match else text.strip()
+
+
+def load_json_or_jsonl(path: Path) -> List[Dict[str, Any]]:
+    if path.suffix == ".jsonl":
+        return load_json_records(path)
+    with path.open("r", encoding="utf-8") as f:
+        payload = json.load(f)
+    if isinstance(payload, dict) and isinstance(payload.get("results"), list):
+        return payload["results"]
+    if isinstance(payload, list):
+        return payload
+    raise ValueError(f"Unsupported input JSON format: {path}")
+
+
+def resolve_dataset_path(bench_dir: str, dataset: str) -> Path:
+    raw = Path(dataset)
+    if raw.is_file():
+        return raw
+    base = Path(bench_dir)
+    candidates = [base / dataset, base / f"{dataset}.json", base / f"{dataset}.jsonl"]
+    for path in candidates:
+        if path.is_file():
+            return path
+    raise FileNotFoundError(f"Cannot find dataset {dataset!r} under {bench_dir}")
+
+
+def clean_question(problem: str) -> str:
+    return re.sub(r"^\s*<(image|video)>\s*", "", problem or "", flags=re.IGNORECASE).strip()
+
+
+def build_prompt(example: Dict[str, Any], prompt_mode: str) -> List[Dict[str, Any]]:
+    # Match the joint SFT training prompt verbatim: the question is followed
+    # immediately by the training tail.
+    question = clean_question(str(example.get("problem") or example.get("question") or ""))
+    data_type = str(example.get("data_type") or "").strip().lower()
+    tail = TRAIN_SEG_VIDEO_TAIL if data_type == "video" else TRAIN_SEG_IMAGE_TAIL
+    text = f"{question}\n{tail}"
+    return [{"role": "user", "content": [{"type": "text", "text": text}]}]
+
+
+def resolve_media_path(base_prefix: str, raw_path: str) -> tuple[str, str]:
+    """Return `(absolute_path_for_inference, relative_path_for_output)`.
+
+    OneThinker JSON often stores paths like `./Evaluation/Refcoco/...`, while
+    local mirrors may place media directly under `Refcoco/...`.
+    """
+    if os.path.isabs(raw_path):
+        return raw_path, repository_relative_output_path(raw_path, base_prefix)
+
+    rel = raw_path.lstrip("./")
+    candidates = [rel]
+    if rel.startswith("Evaluation/"):
+        candidates.append(rel[len("Evaluation/"):])
+
+    for cand in candidates:
+        full = os.path.join(base_prefix, cand)
+        if os.path.exists(full):
+            return full, f"./{cand}"
+
+    return os.path.join(base_prefix, rel), f"./{rel}"
+
+
+def canonical_output_path(base_prefix: str, raw_path: str) -> str:
+    return resolve_media_path(base_prefix, raw_path)[1]
+
+
+def build_content(example: Dict[str, Any], args: argparse.Namespace) -> List[Dict[str, Any]]:
+    data_type = str(example.get("data_type") or "").strip().lower()
+    media_path, _ = resolve_media_path(args.base_prefix, str(example.get("path") or ""))
+    if data_type == "image":
+        return [{
+            "type": "image",
+            "image": media_path,
+            "max_pixels": args.max_pixels_image,
+            "min_pixels": args.min_pixels_image,
+        }]
+    if data_type == "video":
+        item = {
+            "type": "video",
+            "video": media_path,
+            "max_pixels": args.max_pixels_video,
+            "max_frames": args.max_frames,
+            "fps": args.fps,
+        }
+        if os.getenv("EVAL_VIDEO_ITEM_ONETHINKER", "0") != "1":
+            item["min_pixels"] = args.min_pixels_video
+            item["total_pixels"] = args.total_pixels_video
+        return [item]
+    raise ValueError(f"Unsupported data_type for segmentation: {data_type!r}")
+
+
+def prepare_vllm_input(
+    messages: List[Dict[str, Any]],
+    processor: Any,
+    patch_size: Optional[int],
+    enable_thinking: bool = False,
+) -> Dict[str, Any]:
+    omit_thinking_kw = os.getenv("EVAL_OMIT_ENABLE_THINKING_KW", "0") == "1"
+    force_do_resize_false = os.getenv("EVAL_FORCE_DO_RESIZE_FALSE", "0") == "1"
+
+    if omit_thinking_kw:
+        # OneThinker eval_bench.py omits this kwarg entirely. Keep this
+        # switch for paper-parity runs with OneThinker checkpoints.
+        text = processor.apply_chat_template(
+            messages,
+            tokenize=False,
+            add_generation_prompt=True,
+        )
+    else:
+        text = processor.apply_chat_template(
+            messages,
+            tokenize=False,
+            add_generation_prompt=True,
+            enable_thinking=enable_thinking,
+        )
+    kwargs: Dict[str, Any] = {"return_video_kwargs": True, "return_video_metadata": True}
+    if patch_size is None:
+        patch_size = getattr(getattr(processor, "image_processor", None), "patch_size", None)
+    if patch_size is not None:
+        kwargs["image_patch_size"] = patch_size
+    image_inputs, video_inputs, video_kwargs = process_vision_info(messages, **kwargs)
+    video_kwargs = video_kwargs or {}
+    if force_do_resize_false:
+        # Diagnostic only. For Qwen3.5 video segmentation this can trigger
+        # processor shape mismatches when decoded frames are not patch-aligned.
+        video_kwargs["do_resize"] = False
+    mm_data: Dict[str, Any] = {}
+    if image_inputs is not None:
+        mm_data["image"] = image_inputs
+    if video_inputs is not None:
+        mm_data["video"] = video_inputs
+    return {"prompt": text, "multi_modal_data": mm_data, "mm_processor_kwargs": video_kwargs}
+
+
+def parse_json_answer(answer: str) -> Optional[Dict[str, Any]]:
+    if not answer:
+        return None
+    try:
+        obj = json.loads(answer)
+        return obj if isinstance(obj, dict) else None
+    except Exception:
+        pass
+    start = answer.find("{")
+    if start < 0:
+        return None
+    depth = 0
+    for idx in range(start, len(answer)):
+        if answer[idx] == "{":
+            depth += 1
+        elif answer[idx] == "}":
+            depth -= 1
+            if depth == 0:
+                try:
+                    obj = json.loads(answer[start:idx + 1])
+                    return obj if isinstance(obj, dict) else None
+                except Exception:
+                    return None
+    return None
+
+
+def _is_number_list(value: Any, length: int) -> bool:
+    if not isinstance(value, list) or len(value) != length:
+        return False
+    try:
+        for item in value:
+            float(item)
+        return True
+    except Exception:
+        return False
+
+
+def _is_point_list(value: Any) -> bool:
+    if not isinstance(value, list) or len(value) < 1:
+        return False
+    return all(_is_number_list(point, 2) for point in value)
+
+
+def normalize_seg_prompt(obj: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+    """Normalize common model variants to the SAM2 prompt schema.
+
+    The prompt asks for one bounding box but uses the plural key "boxes". Some
+    models naturally return a list-of-boxes (`[[x1, y1, x2, y2]]`), while the
+    downstream OneThinker SAM2 script expects the single box as a flat list.
+    """
+    if not isinstance(obj, dict):
+        return obj
+    out = dict(obj)
+    boxes = out.get("boxes")
+    if (
+        isinstance(boxes, list)
+        and len(boxes) == 1
+        and _is_number_list(boxes[0], 4)
+    ):
+        out["boxes"] = boxes[0]
+    return out
+
+
+def valid_seg_prompt(obj: Optional[Dict[str, Any]], data_type: str) -> bool:
+    if not isinstance(obj, dict):
+        return False
+    boxes = obj.get("boxes")
+    if not _is_number_list(boxes, 4):
+        return False
+    for key in ("positive_points", "negative_points"):
+        if not _is_point_list(obj.get(key)):
+            return False
+    if data_type == "video":
+        try:
+            float(obj.get("time"))
+        except Exception:
+            return False
+    return True
+
+
+def iter_filtered(data: Iterable[Dict[str, Any]], args: argparse.Namespace) -> List[Dict[str, Any]]:
+    rows = []
+    missing_media = 0
+    for rec in data:
+        if str(rec.get("problem_type", "")).strip().lower() != "segmentation":
+            continue
+        data_type = str(rec.get("data_type") or "").strip().lower()
+        if data_type not in {"image", "video"}:
+            continue
+        if args.data_type != "all" and data_type != args.data_type:
+            continue
+        media_path, _ = resolve_media_path(args.base_prefix, str(rec.get("path") or ""))
+        if args.skip_missing_media and not os.path.exists(media_path):
+            missing_media += 1
+            continue
+        rows.append(rec)
+    if missing_media:
+        print(f"[warn] skipped {missing_media} samples with missing media", file=sys.stderr)
+    if args.max_samples is not None:
+        rows = rows[:args.max_samples]
+    if args.chunk > 1:
+        rows = [r for i, r in enumerate(rows) if i % args.chunk == args.index]
+    return rows
+
+
+def evaluate_dataset(dataset: str, args: argparse.Namespace, llm: LLM, processor: Any, sampling_params: SamplingParams) -> Dict[str, Any]:
+    data_path = resolve_dataset_path(args.bench_dir, dataset)
+    rows = iter_filtered(load_json_or_jsonl(data_path), args)
+
+    outputs: List[Dict[str, Any]] = []
+    for start in tqdm(range(0, len(rows), args.batch_size), desc=f"{dataset}"):
+        batch = rows[start:start + args.batch_size]
+        inputs = []
+        for example in batch:
+            messages = build_prompt(example, args.prompt_mode)
+            user_msg = messages[-1]
+            user_msg = dict(user_msg)
+            user_msg["content"] = build_content(example, args) + user_msg["content"]
+            run_messages = messages[:-1] + [user_msg]
+            inputs.append(prepare_vllm_input(run_messages, processor, args.patch_size, args.enable_thinking))
+
+        generated = llm.generate(inputs, sampling_params=sampling_params)
+        texts = [out.outputs[0].text for out in generated]
+
+        for example, text in zip(batch, texts):
+            answer = extract_answer(text)
+            parsed = parse_json_answer(answer)
+            parsed = normalize_seg_prompt(parsed)
+            data_type = str(example.get("data_type") or "").strip().lower()
+            sample = dict(example)
+            sample["path"] = canonical_output_path(args.base_prefix, str(example.get("path") or ""))
+            sample["output"] = text
+            sample["prediction"] = answer
+            sample["predicted_answer_norm"] = (
+                json.dumps(parsed, ensure_ascii=False) if isinstance(parsed, dict) else answer
+            )
+            sample["parsed_prediction"] = parsed
+            sample["parse_ok"] = valid_seg_prompt(parsed, data_type)
+            outputs.append(sample)
+
+    total = len(outputs)
+    parsed = sum(1 for row in outputs if row.get("parse_ok"))
+    by_type: Dict[str, Dict[str, Any]] = {}
+    for data_type in ("image", "video"):
+        part = [r for r in outputs if r.get("data_type") == data_type]
+        if part:
+            ok = sum(1 for r in part if r.get("parse_ok"))
+            by_type[data_type] = {
+                "num_samples": len(part),
+                "parse_rate": round(ok / len(part) * 100.0, 2),
+            }
+    return {
+        "dataset": dataset,
+        "input_file": str(data_path),
+        "results": outputs,
+        "metrics": {
+            "num_samples": total,
+            "parse_ok": parsed,
+            "parse_rate": round(parsed / total * 100.0, 2) if total else 0.0,
+            "by_data_type": by_type,
+        },
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model_path", required=True)
+    parser.add_argument("--processor_path", default=None)
+    parser.add_argument("--bench_dir", required=True)
+    parser.add_argument("--datasets", required=True, help="Comma-separated dataset names or JSON/JSONL paths.")
+    parser.add_argument("--output_dir", required=True)
+    parser.add_argument("--base_prefix", default="")
+    parser.add_argument("--data_type", choices=["all", "image", "video"], default="all")
+    parser.add_argument("--prompt_mode", choices=["think", "no_think", "bare", "onethink_system", "train_seg"], default="no_think")
+    parser.add_argument("--enable_thinking", action="store_true", default=False)
+    parser.add_argument("--max_samples", type=int, default=None)
+    parser.add_argument("--skip_missing_media", action="store_true")
+    parser.add_argument("--chunk", type=int, default=1)
+    parser.add_argument("--index", type=int, default=0)
+    parser.add_argument("--batch_size", type=int, default=16)
+    parser.add_argument("--temperature", type=float, default=0.0)
+    parser.add_argument("--top_k", type=int, default=-1)
+    parser.add_argument("--max_new_tokens", type=int, default=1024)
+    parser.add_argument("--tensor_parallel_size", type=int, default=1)
+    parser.add_argument("--gpu_memory_utilization", type=float, default=0.85)
+    parser.add_argument("--max_model_len", type=int, default=32768)
+    parser.add_argument("--seed", type=int, default=42)
+    parser.add_argument("--max_pixels_image", type=int, default=1024 * 32 * 32)
+    parser.add_argument("--min_pixels_image", type=int, default=4 * 32 * 32)
+    parser.add_argument("--max_pixels_video", type=int, default=256 * 32 * 32)
+    parser.add_argument("--min_pixels_video", type=int, default=4 * 32 * 32)
+    parser.add_argument("--total_pixels_video", type=int, default=256 * 64 * 32 * 32)
+    parser.add_argument("--max_frames", type=int, default=128)
+    parser.add_argument("--fps", type=int, default=2)
+    parser.add_argument("--patch_size", type=int, default=None)
+    args = parser.parse_args()
+
+    if args.chunk < 1 or not (0 <= args.index < args.chunk):
+        raise ValueError("--chunk must be >=1 and --index must be in [0, chunk)")
+
+    os.makedirs(args.output_dir, exist_ok=True)
+    torch.manual_seed(args.seed)
+
+    processor = AutoProcessor.from_pretrained(args.processor_path or args.model_path)
+    llm = LLM(
+        model=args.model_path,
+        tensor_parallel_size=args.tensor_parallel_size,
+        max_model_len=args.max_model_len,
+        gpu_memory_utilization=args.gpu_memory_utilization,
+        mm_encoder_tp_mode="data",
+        seed=args.seed,
+    )
+    sampling_params = SamplingParams(
+        temperature=args.temperature,
+        max_tokens=args.max_new_tokens,
+        top_k=args.top_k,
+        stop_token_ids=[],
+    )
+
+    summary: Dict[str, Any] = {}
+    for dataset in [x.strip() for x in args.datasets.split(",") if x.strip()]:
+        payload = evaluate_dataset(dataset, args, llm, processor, sampling_params)
+        dataset_key = Path(dataset).stem if Path(dataset).suffix else dataset
+        suffix = f"_shard{args.index}" if args.chunk > 1 else ""
+        out_path = Path(args.output_dir) / f"results_{dataset_key}{suffix}.json"
+        with out_path.open("w", encoding="utf-8") as f:
+            json.dump({"results": payload["results"], "metrics": payload["metrics"]}, f, ensure_ascii=False, indent=2)
+        summary[dataset_key] = payload["metrics"]
+        print(
+            f"{dataset_key}: n={payload['metrics']['num_samples']} "
+            f"parse={payload['metrics']['parse_rate']:.2f}% -> {out_path}"
+        )
+
+    summary_name = f"summary_shard{args.index}.json" if args.chunk > 1 else "summary.json"
+    with (Path(args.output_dir) / summary_name).open("w", encoding="utf-8") as f:
+        json.dump(summary, f, ensure_ascii=False, indent=2)
+
+
+if __name__ == "__main__":
+    main()

code/eval/task/segmentation/post_sam2.py

@@ -0,0 +1,491 @@
+#!/usr/bin/env python3
+"""
+Thin project-local wrapper around OneThinker's SAM2 segmentation post-process.
+
+The OneThinker script contains the full image/video mask generation and metric
+logic.  This wrapper only injects paths from CLI args so evaluation can be run
+from this repository without editing the third-party checkout.
+"""
+
+from __future__ import annotations
+
+import argparse
+import importlib.util
+import json
+import multiprocessing.spawn as mp_spawn
+import os
+import sys
+import tempfile
+from pathlib import Path
+from typing import Any
+
+
+DEFAULT_ONETHINKER_SCRIPT = str(
+    Path(__file__).resolve().parents[3]
+    / "third_party"
+    / "OneThinker"
+    / "Evaluation"
+    / "Eval"
+    / "seg_post_sam2.py"
+)
+
+
+def sam2_config_name(
+    value: str,
+    package_root: str | Path | None = None,
+) -> str:
+    """Convert an installed SAM2 YAML path to Hydra's package-relative name."""
+
+    path = Path(value).expanduser()
+    if not path.is_absolute():
+        return path.as_posix()
+    if package_root is None:
+        try:
+            import sam2
+        except ImportError:
+            return str(path)
+        package_root = Path(sam2.__file__).resolve().parent
+    try:
+        return path.resolve().relative_to(Path(package_root).resolve()).as_posix()
+    except ValueError:
+        return str(path)
+
+
+def load_module(script_path: str):
+    path = Path(script_path)
+    if not path.is_file():
+        raise FileNotFoundError(f"OneThinker SAM2 script not found: {path}")
+    # Use the real filename as the module name so multiprocessing "spawn"
+    # workers can import it from path.parent when unpickling worker_run.
+    module_name = path.stem
+    spec = importlib.util.spec_from_file_location(module_name, path)
+    if spec is None or spec.loader is None:
+        raise RuntimeError(f"Failed to load module spec from {path}")
+    module = importlib.util.module_from_spec(spec)
+    sys.path.insert(0, str(path.parent))
+    # multiprocessing pickles functions by module name; dynamic modules must be
+    # registered so child processes can resolve worker_run.
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+def _results(payload: Any) -> list[dict[str, Any]]:
+    if isinstance(payload, dict) and isinstance(payload.get("results"), list):
+        return payload["results"]
+    if isinstance(payload, list):
+        return payload
+    raise ValueError("SAM2 input must be a list or an object containing `results`.")
+
+
+def _segmentation_output(sample: dict[str, Any]) -> dict[str, Any] | None:
+    output = sample.get("segmentation_output")
+    if isinstance(output, dict):
+        return output
+    task_payload = sample.get("task_payload")
+    if isinstance(task_payload, dict):
+        output = task_payload.get("segmentation_output")
+        if isinstance(output, dict):
+            sample["segmentation_output"] = output
+            return output
+    return None
+
+
+def _compressed_rle(
+    size: Any,
+    counts: list[Any],
+    cocomask: Any,
+) -> dict[str, Any] | None:
+    if (
+        not isinstance(size, (list, tuple))
+        or len(size) != 2
+        or not all(isinstance(value, (int, float)) for value in size)
+    ):
+        return None
+    height, width = (int(size[0]), int(size[1]))
+    if height <= 0 or width <= 0:
+        return None
+    encoded = cocomask.frPyObjects(
+        {"size": [height, width], "counts": counts},
+        height,
+        width,
+    )
+    if isinstance(encoded, list):
+        encoded = cocomask.merge(encoded)
+    encoded_counts = encoded["counts"]
+    if isinstance(encoded_counts, bytes):
+        encoded_counts = encoded_counts.decode("utf-8")
+    return {"size": [height, width], "counts": encoded_counts}
+
+
+def normalize_missing_rle_counts(payload: Any, cocomask: Any) -> int:
+    """Repair mask annotations that encode empty masks or polygons without RLE counts."""
+
+    repaired = 0
+    for sample in _results(payload):
+        segmentation = _segmentation_output(sample)
+        if segmentation is None:
+            continue
+        rle = segmentation.get("segmentation_rle")
+
+        if sample.get("data_type") == "video":
+            if not isinstance(rle, dict):
+                continue
+            for frame, frame_rle in list(rle.items()):
+                if not isinstance(frame_rle, dict):
+                    continue
+                counts = frame_rle.get("counts")
+                if counts is not None and not isinstance(counts, list):
+                    continue
+                size = frame_rle.get("size")
+                if (
+                    counts is None
+                    and (
+                        not isinstance(size, (list, tuple))
+                        or len(size) != 2
+                    )
+                ):
+                    continue
+                replacement = _compressed_rle(
+                    size,
+                    counts
+                    if isinstance(counts, list)
+                    else [int(size[0]) * int(size[1])],
+                    cocomask,
+                )
+                if replacement is not None:
+                    rle[frame] = replacement
+                    repaired += 1
+            continue
+
+        if not isinstance(rle, dict):
+            continue
+
+        size = rle.get("size")
+        counts = rle.get("counts")
+        if isinstance(counts, list):
+            replacement = _compressed_rle(size, counts, cocomask)
+            if replacement is not None:
+                segmentation["segmentation_rle"] = replacement
+                repaired += 1
+            continue
+        if counts is not None:
+            continue
+
+        polygons = segmentation.get("segmentation_polygon")
+        if polygons:
+            if isinstance(polygons, list) and polygons and isinstance(
+                polygons[0], (int, float)
+            ):
+                polygons = [polygons]
+            if (
+                isinstance(size, (list, tuple))
+                and len(size) == 2
+                and polygons
+            ):
+                height, width = int(size[0]), int(size[1])
+                encoded = cocomask.frPyObjects(polygons, height, width)
+                if isinstance(encoded, list):
+                    encoded = cocomask.merge(encoded)
+                counts = encoded["counts"]
+                if isinstance(counts, bytes):
+                    counts = counts.decode("utf-8")
+                segmentation["segmentation_rle"] = {
+                    "size": [height, width],
+                    "counts": counts,
+                }
+                repaired += 1
+                continue
+
+        replacement = None
+        if isinstance(size, (list, tuple)) and len(size) == 2:
+            replacement = _compressed_rle(
+                size,
+                [int(size[0]) * int(size[1])],
+                cocomask,
+            )
+        if replacement is not None:
+            segmentation["segmentation_rle"] = replacement
+            repaired += 1
+
+    return repaired
+
+
+def _sam2_output_path(input_json: str | Path) -> Path:
+    path = Path(input_json)
+    return path.with_name(f"{path.stem}_sam2.json")
+
+
+def _aggregate_rewards(payload: dict[str, Any]) -> None:
+    ok_items = [
+        row
+        for row in _results(payload)
+        if row.get("status") == "ok" and isinstance(row.get("reward"), dict)
+    ]
+    video_j: list[float] = []
+    video_f: list[float] = []
+    video_jf: list[float] = []
+    image_iou: list[float] = []
+    total_inter = 0
+    total_union = 0
+
+    for row in ok_items:
+        reward = row["reward"]
+        if all(key in reward for key in ("J", "F", "J&F")):
+            video_j.append(float(reward["J"]))
+            video_f.append(float(reward["F"]))
+            video_jf.append(float(reward["J&F"]))
+        if "IoU" in reward:
+            image_iou.append(float(reward["IoU"]))
+            total_inter += int(reward.get("inter", 0))
+            total_union += int(reward.get("union", 0))
+
+    averages: dict[str, float] = {}
+    if video_j:
+        averages.update(
+            {
+                "video/J": sum(video_j) / len(video_j),
+                "video/F": sum(video_f) / len(video_f),
+                "video/J&F": sum(video_jf) / len(video_jf),
+            }
+        )
+    if image_iou:
+        giou = sum(image_iou) / len(image_iou)
+        averages.update(
+            {
+                "image/IoU": giou,
+                "image/gIoU": giou,
+                "image/cIoU": (
+                    float(total_inter) / float(total_union) if total_union else 0.0
+                ),
+            }
+        )
+    payload["avg_rewards"] = averages
+
+
+def _write_json_atomic(path: Path, payload: dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd, temporary = tempfile.mkstemp(
+        dir=path.parent,
+        prefix=f".{path.name}.",
+        suffix=".tmp",
+    )
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as stream:
+            json.dump(payload, stream, ensure_ascii=False, indent=2)
+        os.replace(temporary, path)
+    except BaseException:
+        try:
+            os.unlink(temporary)
+        except FileNotFoundError:
+            pass
+        raise
+
+
+def _run_upstream(
+    module: Any,
+    *,
+    input_json: Path,
+    viz_ratio: float,
+    normalize: bool,
+) -> Path:
+    original_load = module.json.load
+
+    if normalize:
+        def load_and_normalize(*args: Any, **kwargs: Any) -> Any:
+            payload = original_load(*args, **kwargs)
+            repaired = normalize_missing_rle_counts(payload, module.cocomask)
+            if repaired:
+                print(f"[INFO] normalized {repaired} mask RLE entries without counts")
+            return payload
+
+        module.json.load = load_and_normalize
+
+    try:
+        sys.argv = [
+            str(Path(module.__file__).name),
+            "--input_json",
+            str(input_json),
+            "--viz_ratio",
+            str(viz_ratio),
+        ]
+        module.main()
+    finally:
+        module.json.load = original_load
+    return _sam2_output_path(input_json)
+
+
+def _retry_errors(
+    module: Any,
+    *,
+    output_path: Path,
+    attempts: int,
+    viz_ratio: float,
+) -> tuple[dict[str, Any], int]:
+    with output_path.open(encoding="utf-8") as stream:
+        payload = json.load(stream)
+
+    retried = 0
+    result_fields = (
+        "reward",
+        "status",
+        "viz_key_pred",
+        "viz_key_gt",
+        "viz_pred_video",
+        "viz_gt_video",
+    )
+    module.WORKERS_PER_GPU = 1
+
+    for attempt in range(1, attempts + 1):
+        failed = [
+            row
+            for row in _results(payload)
+            if str(row.get("status", "")).startswith("error:")
+        ]
+        if not failed:
+            break
+        retried += len(failed)
+        retry_payload = {"results": failed}
+        repaired = normalize_missing_rle_counts(retry_payload, module.cocomask)
+        if repaired:
+            print(f"[INFO] normalized {repaired} mask RLE entries without counts")
+        print(
+            f"[INFO] retrying {len(failed)} SAM2 errors "
+            f"(attempt {attempt}/{attempts}, workers_per_gpu=1)"
+        )
+
+        with tempfile.TemporaryDirectory(
+            dir=output_path.parent,
+            prefix=".sam2-retry-",
+        ) as temporary_dir:
+            retry_input = Path(temporary_dir) / "retry.json"
+            with retry_input.open("w", encoding="utf-8") as stream:
+                json.dump(retry_payload, stream, ensure_ascii=False)
+            retry_output = _run_upstream(
+                module,
+                input_json=retry_input,
+                viz_ratio=viz_ratio,
+                normalize=False,
+            )
+            with retry_output.open(encoding="utf-8") as stream:
+                retry_result = json.load(stream)
+
+        retried_by_id = {
+            row.get("problem_id"): row for row in _results(retry_result)
+        }
+        for row in _results(payload):
+            retry_row = retried_by_id.get(row.get("problem_id"))
+            if retry_row is not None:
+                for field in result_fields:
+                    row[field] = retry_row.get(field)
+
+    _aggregate_rewards(payload)
+    _write_json_atomic(output_path, payload)
+    return payload, retried
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--input_json", required=True, help="Prediction JSON from eval_seg_vllm.py.")
+    parser.add_argument("--data_root", required=True, help="Root joined with each record's relative `path`.")
+    parser.add_argument("--sam2_ckpt", required=True)
+    parser.add_argument("--sam2_cfg", required=True)
+    parser.add_argument("--onethinker_script", default=DEFAULT_ONETHINKER_SCRIPT)
+    parser.add_argument("--num_gpus", type=int, default=None)
+    parser.add_argument("--workers_per_gpu", type=int, default=None)
+    parser.add_argument("--pre_extract_threads", type=int, default=None)
+    parser.add_argument("--epoch_size", type=int, default=None)
+    parser.add_argument("--viz_ratio", type=float, default=0.0)
+    parser.add_argument(
+        "--retry_attempts",
+        type=int,
+        default=1,
+        help="Retry per-sample SAM2 errors with one worker per GPU.",
+    )
+    parser.add_argument(
+        "--retry_existing_errors",
+        action="store_true",
+        help="Retry errors in an existing sibling *_sam2.json without rerunning all samples.",
+    )
+    args = parser.parse_args()
+
+    # Python's spawn start method calls os.getcwd() for every new worker pool.
+    # If the command was launched from a transient directory, later epochs can
+    # fail with FileNotFoundError. Pin cwd to the project root.
+    project_dir = Path(__file__).resolve().parents[3]
+    os.chdir(project_dir)
+    original_get_preparation_data = mp_spawn.get_preparation_data
+
+    def get_preparation_data(name):
+        try:
+            return original_get_preparation_data(name)
+        except FileNotFoundError:
+            os.chdir(project_dir)
+            return original_get_preparation_data(name)
+
+    mp_spawn.get_preparation_data = get_preparation_data
+
+    config_name = sam2_config_name(args.sam2_cfg)
+    os.environ["SAM2_DATA_ROOT"] = args.data_root
+    os.environ["SAM2_CKPT"] = args.sam2_ckpt
+    os.environ["SAM2_CFG"] = config_name
+    module = load_module(args.onethinker_script)
+    module.DATA_ROOT = args.data_root
+    module.SAM2_CKPT = args.sam2_ckpt
+    module.SAM2_CFG = config_name
+    if args.num_gpus is not None:
+        module.NUM_GPUS = args.num_gpus
+    if args.workers_per_gpu is not None:
+        module.WORKERS_PER_GPU = args.workers_per_gpu
+    if args.pre_extract_threads is not None:
+        module.PRE_EXTRACT_THREADS = args.pre_extract_threads
+    if args.epoch_size is not None:
+        module.EPOCH_SIZE = args.epoch_size
+
+    sys.argv = [
+        str(Path(args.onethinker_script).name),
+        "--input_json",
+        args.input_json,
+        "--viz_ratio",
+        str(args.viz_ratio),
+    ]
+    os.environ.setdefault(
+        "PYTORCH_CUDA_ALLOC_CONF",
+        "expandable_segments:True,max_split_size_mb:64,garbage_collection_threshold:0.8",
+    )
+    module.mp.set_start_method("spawn", force=True)
+
+    input_path = Path(args.input_json).resolve()
+    output_path = _sam2_output_path(input_path)
+    if not args.retry_existing_errors:
+        output_path = _run_upstream(
+            module,
+            input_json=input_path,
+            viz_ratio=args.viz_ratio,
+            normalize=True,
+        )
+    elif not output_path.is_file():
+        raise FileNotFoundError(f"Existing SAM2 result not found: {output_path}")
+
+    payload, retried = _retry_errors(
+        module,
+        output_path=output_path,
+        attempts=max(0, args.retry_attempts),
+        viz_ratio=args.viz_ratio,
+    )
+    failed = [row for row in _results(payload) if row.get("status") != "ok"]
+    print(
+        f"[INFO] final SAM2 status: ok={len(_results(payload)) - len(failed)} "
+        f"/ total={len(_results(payload))}; retried={retried}"
+    )
+    if failed:
+        examples = ", ".join(
+            f"{row.get('problem_id')}={row.get('status')}" for row in failed[:5]
+        )
+        raise RuntimeError(
+            f"SAM2 post-processing left {len(failed)} failed samples: {examples}"
+        )
+
+
+if __name__ == "__main__":
+    main()

code/eval/task/segmentation/run_eval_vllm.sh

@@ -0,0 +1,500 @@
+#!/bin/bash
+# =============================================================================
+# Segmentation evaluation — vLLM inference + optional SAM2 post-processing.
+#
+# Usage:
+#   bash eval/task/segmentation/run_eval_vllm.sh [MODEL_PATH] [OUTPUT_DIR]
+#
+# Common overrides:
+#   BENCH_DIR=/path/to/OneThinker-eval DATASETS=reasonseg-val DATA_ROOT=/path/to/OneThinker-eval \
+#   RUN_SAM2=true SAM2_CKPT=/path/to/sam2.1_hiera_large.pt SAM2_CFG=/path/to/sam2.1_hiera_l.yaml \
+#   bash eval/task/segmentation/run_eval_vllm.sh /path/to/ckpt
+# =============================================================================
+
+set -eo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+PROJECT_DIR="$(cd "$SCRIPT_DIR/../../.." && pwd)"
+
+# ---------- paths ----------
+MODEL_PATH="${1:-}"
+PROCESSOR_PATH="${PROCESSOR_PATH:-$MODEL_PATH}"
+BENCH_DIR="${BENCH_DIR:-${PROJECT_DIR}/data/eval/segmentation}"
+DATA_ROOT="${DATA_ROOT:-$BENCH_DIR}"
+DATASETS="${DATASETS:-eval_seg_refcoco,eval_seg_refcocop,eval_seg_refcocog,eval_seg_mevis,eval_seg_reasonvos}"
+
+if [[ -z "$MODEL_PATH" || ! -d "$MODEL_PATH" ]]; then
+    echo "ERROR: MODEL_PATH must name an existing model directory." >&2
+    exit 2
+fi
+if [[ ! -d "$BENCH_DIR" || ! -d "$DATA_ROOT" ]]; then
+    echo "ERROR: BENCH_DIR and DATA_ROOT must name existing directories." >&2
+    exit 2
+fi
+
+# ---------- eval settings ----------
+DATA_TYPE="${DATA_TYPE:-all}"              # all | image | video
+PROMPT_MODE="${PROMPT_MODE:-train_seg}"   # think | no_think | bare | onethink_system | train_seg
+ENABLE_THINKING="${ENABLE_THINKING:-false}"
+MAX_SAMPLES="${MAX_SAMPLES:-}"
+BATCH_SIZE="${BATCH_SIZE:-16}"
+MAX_NEW_TOKENS="${MAX_NEW_TOKENS:-1024}"
+MAX_PIXELS_IMAGE="${MAX_PIXELS_IMAGE:-1048576}"
+MIN_PIXELS_IMAGE="${MIN_PIXELS_IMAGE:-4096}"
+# Accept both the tracking-style VIDEO_* names and older MAX_PIXELS_VIDEO names.
+VIDEO_MAX_PIXELS="${VIDEO_MAX_PIXELS:-${MAX_PIXELS_VIDEO:-262144}}"
+VIDEO_MIN_PIXELS="${VIDEO_MIN_PIXELS:-${MIN_PIXELS_VIDEO:-4096}}"
+VIDEO_TOTAL_PIXELS="${VIDEO_TOTAL_PIXELS:-${TOTAL_PIXELS_VIDEO:-16777216}}"
+MAX_FRAMES="${MAX_FRAMES:-128}"
+FPS="${FPS:-2}"
+PATCH_SIZE="${PATCH_SIZE:-}"
+VIDEO_READER="${VIDEO_READER:-decord}"
+
+case "$VIDEO_READER" in
+    decord|torchcodec|torchvision) ;;
+    *)
+        echo "ERROR: VIDEO_READER must be decord, torchcodec, or torchvision; got '$VIDEO_READER'" >&2
+        exit 1
+        ;;
+esac
+export FORCE_QWENVL_VIDEO_READER="$VIDEO_READER"
+
+require_int() {
+    local name="$1"
+    local value="$2"
+    if ! [[ "$value" =~ ^[0-9]+$ ]]; then
+        echo "ERROR: $name must be a non-empty integer, got '$value'"
+        exit 1
+    fi
+}
+
+require_int MAX_PIXELS_IMAGE "$MAX_PIXELS_IMAGE"
+require_int MIN_PIXELS_IMAGE "$MIN_PIXELS_IMAGE"
+require_int VIDEO_MAX_PIXELS "$VIDEO_MAX_PIXELS"
+require_int VIDEO_MIN_PIXELS "$VIDEO_MIN_PIXELS"
+require_int VIDEO_TOTAL_PIXELS "$VIDEO_TOTAL_PIXELS"
+require_int MAX_FRAMES "$MAX_FRAMES"
+require_int FPS "$FPS"
+require_int BATCH_SIZE "$BATCH_SIZE"
+require_int MAX_NEW_TOKENS "$MAX_NEW_TOKENS"
+
+# ---------- vLLM ----------
+TP_SIZE="${TP_SIZE:-1}"
+GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.85}"
+MAX_MODEL_LEN="${MAX_MODEL_LEN:-32768}"
+SEED="${SEED:-42}"
+VLLM_BASE_PORT="${VLLM_BASE_PORT:-}"
+RESUME_SHARDS="${RESUME_SHARDS:-false}"
+RETRY_FAILED_SHARDS="${RETRY_FAILED_SHARDS:-true}"
+
+# ---------- SAM2 ----------
+RUN_SAM2="${RUN_SAM2:-false}"
+SAM2_CKPT="${SAM2_CKPT:-}"
+SAM2_CFG="${SAM2_CFG:-}"
+ONETHINKER_SEG_POST="${ONETHINKER_SEG_POST:-${PROJECT_DIR}/third_party/OneThinker/Evaluation/Eval/seg_post_sam2.py}"
+SAM2_NUM_GPUS="${SAM2_NUM_GPUS:-}"
+SAM2_WORKERS_PER_GPU="${SAM2_WORKERS_PER_GPU:-}"
+PRE_EXTRACT_THREADS="${PRE_EXTRACT_THREADS:-4}"
+# Each SAM2 epoch spawns `world_size` worker processes that EACH reload the SAM2
+# model and then handle only their slice of the epoch. Total model loads =
+# world_size * num_epochs. A small epoch with many workers means tiny slices and
+# constant model reloading (the real bottleneck). Keep the epoch large so there
+# is effectively one epoch and each worker amortizes its model load over a big
+# contiguous slice.
+SAM2_EPOCH_SIZE="${SAM2_EPOCH_SIZE:-100000}"
+VIZ_RATIO="${VIZ_RATIO:-0.0}"
+
+# ---------- output ----------
+MODEL_TAG=$(basename "${MODEL_PATH%/}")
+TIMESTAMP=$(date +%Y%m%d_%H%M%S)
+OUTPUT_DIR="${2:-${PROJECT_DIR}/outputs/segmentation/eval_seg_vllm-${MODEL_TAG}-${TIMESTAMP}}"
+
+# ---------- hardware ----------
+NVJITLINK_LIB="$(
+    python - <<'PY'
+import site
+from pathlib import Path
+
+roots = [*site.getsitepackages(), site.getusersitepackages()]
+for root in roots:
+    candidate = Path(root) / "nvidia" / "nvjitlink" / "lib"
+    if candidate.is_dir():
+        print(candidate)
+        break
+PY
+)"
+if [[ -n "${NVJITLINK_LIB}" ]]; then
+    export LD_LIBRARY_PATH="${NVJITLINK_LIB}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
+fi
+if ! python -c "import torch; print(f'PyTorch preflight: {torch.__version__} CUDA {torch.version.cuda}')"; then
+    echo "ERROR: PyTorch CUDA libraries cannot be loaded in the active environment." >&2
+    exit 1
+fi
+
+if [ -n "${CUDA_VISIBLE_DEVICES:-}" ]; then
+    IFS="," read -ra GPULIST <<< "$CUDA_VISIBLE_DEVICES"
+else
+    IFS="," read -ra GPULIST <<< "$(seq -s, 0 $(($(nvidia-smi -L | wc -l)-1)))"
+fi
+NUM_GPUS=${#GPULIST[@]}
+if (( NUM_GPUS % TP_SIZE != 0 )); then
+    echo "ERROR: NUM_GPUS ($NUM_GPUS) must be divisible by TP_SIZE ($TP_SIZE)"
+    exit 1
+fi
+DP_SIZE=$(( NUM_GPUS / TP_SIZE ))
+if [[ -z "${VLLM_BASE_PORT}" ]]; then
+    VLLM_BASE_PORT="$(
+        python - "$DP_SIZE" <<'PY'
+import socket
+import sys
+
+count = int(sys.argv[1])
+spacing = 16
+for base in range(45000, 64000 - spacing * count, 128):
+    sockets = []
+    try:
+        for index in range(count):
+            for offset in (0, 1):
+                sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+                sock.bind(("127.0.0.1", base + index * spacing + offset))
+                sockets.append(sock)
+    except OSError:
+        for sock in sockets:
+            sock.close()
+        continue
+    for sock in sockets:
+        sock.close()
+    print(base)
+    break
+else:
+    raise SystemExit("no free segmentation vLLM port block found")
+PY
+    )"
+fi
+
+echo "=============================================="
+echo "Segmentation Evaluation (vLLM)"
+echo "=============================================="
+echo "Model:       $MODEL_PATH"
+echo "Processor:   $PROCESSOR_PATH"
+echo "Bench dir:   $BENCH_DIR"
+echo "Data root:   $DATA_ROOT"
+echo "Datasets:    $DATASETS"
+echo "Data type:   $DATA_TYPE"
+echo "Prompt:      $PROMPT_MODE (enable_thinking=$ENABLE_THINKING)"
+echo "Video pix:   min=$VIDEO_MIN_PIXELS max=$VIDEO_MAX_PIXELS total=$VIDEO_TOTAL_PIXELS frames=$MAX_FRAMES fps=$FPS"
+echo "Video reader:$VIDEO_READER"
+echo "GPUs:        ${GPULIST[*]} (${NUM_GPUS} total, TP=${TP_SIZE}, DP=${DP_SIZE})"
+echo "Base port:   $VLLM_BASE_PORT"
+echo "Output:      $OUTPUT_DIR"
+echo "SAM2:        $RUN_SAM2"
+echo "=============================================="
+
+mkdir -p "$OUTPUT_DIR"
+
+PY_ARGS=(
+    --model_path "$MODEL_PATH"
+    --processor_path "$PROCESSOR_PATH"
+    --bench_dir "$BENCH_DIR"
+    --datasets "$DATASETS"
+    --output_dir "$OUTPUT_DIR"
+    --base_prefix "$DATA_ROOT"
+    --data_type "$DATA_TYPE"
+    --prompt_mode "$PROMPT_MODE"
+    --batch_size "$BATCH_SIZE"
+    --max_new_tokens "$MAX_NEW_TOKENS"
+    --tensor_parallel_size "$TP_SIZE"
+    --gpu_memory_utilization "$GPU_MEM_UTIL"
+    --max_model_len "$MAX_MODEL_LEN"
+    --seed "$SEED"
+    --max_pixels_image "$MAX_PIXELS_IMAGE"
+    --min_pixels_image "$MIN_PIXELS_IMAGE"
+    --max_pixels_video "$VIDEO_MAX_PIXELS"
+    --min_pixels_video "$VIDEO_MIN_PIXELS"
+    --total_pixels_video "$VIDEO_TOTAL_PIXELS"
+    --max_frames "$MAX_FRAMES"
+    --fps "$FPS"
+    --skip_missing_media
+)
+
+if [ -n "$MAX_SAMPLES" ]; then
+    PY_ARGS+=(--max_samples "$MAX_SAMPLES")
+fi
+if [ -n "$PATCH_SIZE" ]; then
+    PY_ARGS+=(--patch_size "$PATCH_SIZE")
+fi
+if [ "$ENABLE_THINKING" = "true" ]; then
+    PY_ARGS+=(--enable_thinking)
+fi
+
+PIDS=()
+PID_SHARDS=()
+cleanup() {
+    echo ""
+    echo "Caught interrupt, killing workers ..."
+    for pid in "${PIDS[@]}"; do
+        kill -TERM "$pid" 2>/dev/null || true
+    done
+    wait 2>/dev/null || true
+    exit 1
+}
+trap cleanup INT TERM
+
+dataset_key() {
+    python -c \
+        "from pathlib import Path; import sys; p=sys.argv[1]; print(Path(p).stem if Path(p).suffix else p)" \
+        "$1"
+}
+
+shard_is_complete() {
+    local shard_id="$1"
+    local dataset
+    local key
+    local datasets=()
+    IFS=',' read -ra datasets <<< "$DATASETS"
+    for dataset in "${datasets[@]}"; do
+        key="$(dataset_key "$dataset")"
+        if [[ ! -s "$OUTPUT_DIR/results_${key}_shard${shard_id}.json" ]]; then
+            return 1
+        fi
+    done
+    return 0
+}
+
+if [ "$DP_SIZE" -eq 1 ]; then
+    CUDA_VISIBLE_DEVICES=$(IFS=,; echo "${GPULIST[*]}") \
+    VLLM_PORT="$VLLM_BASE_PORT" \
+    VLLM_HOST_IP=127.0.0.1 \
+    MASTER_PORT="$((VLLM_BASE_PORT + 1))" \
+    MASTER_ADDR=127.0.0.1 \
+    PYTHONUNBUFFERED=1 python "$SCRIPT_DIR/eval_seg_vllm.py" "${PY_ARGS[@]}" \
+        2>&1 | tee "$OUTPUT_DIR/run.log"
+else
+    echo ""
+    echo ">>> Launching $DP_SIZE vLLM workers (TP=$TP_SIZE each) ..."
+    for IDX in $(seq 0 $((DP_SIZE - 1))); do
+        if [[ "$RESUME_SHARDS" == "true" ]] && shard_is_complete "$IDX"; then
+            echo "  Reusing completed shard $IDX"
+            continue
+        fi
+        START=$(( IDX * TP_SIZE ))
+        SHARD_PORT=$((VLLM_BASE_PORT + IDX * 16))
+        SHARD_MASTER_PORT=$((SHARD_PORT + 1))
+        SHARD_GPUS=""
+        for j in $(seq 0 $((TP_SIZE - 1))); do
+            g=${GPULIST[$((START + j))]}
+            SHARD_GPUS="${SHARD_GPUS}${SHARD_GPUS:+,}${g}"
+        done
+
+        CUDA_VISIBLE_DEVICES="$SHARD_GPUS" \
+        VLLM_PORT="$SHARD_PORT" \
+        VLLM_HOST_IP=127.0.0.1 \
+        MASTER_PORT="$SHARD_MASTER_PORT" \
+        MASTER_ADDR=127.0.0.1 \
+        PYTHONUNBUFFERED=1 \
+        python "$SCRIPT_DIR/eval_seg_vllm.py" "${PY_ARGS[@]}" \
+            --chunk "$DP_SIZE" --index "$IDX" \
+            > "$OUTPUT_DIR/worker_shard${IDX}.log" 2>&1 &
+        PIDS+=($!)
+        PID_SHARDS+=("$IDX")
+        echo "  Launched shard $IDX on GPU $SHARD_GPUS (PID ${PIDS[-1]})"
+    done
+
+    FAILED_SHARDS=()
+    for i in "${!PIDS[@]}"; do
+        if ! wait "${PIDS[$i]}"; then
+            FAILED_SHARDS+=("${PID_SHARDS[$i]}")
+        fi
+    done
+
+    if (( ${#FAILED_SHARDS[@]} > 0 )) && [[ "$RETRY_FAILED_SHARDS" == "true" ]]; then
+        echo ""
+        echo ">>> Retrying failed shards sequentially with fresh ports ..."
+        RETRY_FAILURES=()
+        for IDX in "${FAILED_SHARDS[@]}"; do
+            START=$(( IDX * TP_SIZE ))
+            SHARD_GPUS=""
+            for j in $(seq 0 $((TP_SIZE - 1))); do
+                g=${GPULIST[$((START + j))]}
+                SHARD_GPUS="${SHARD_GPUS}${SHARD_GPUS:+,}${g}"
+            done
+            RETRY_PORT="$(
+                python - <<'PY'
+import socket
+
+for base in range(52000, 64000, 8):
+    sockets = []
+    try:
+        for offset in (0, 1):
+            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            sock.bind(("127.0.0.1", base + offset))
+            sockets.append(sock)
+    except OSError:
+        for sock in sockets:
+            sock.close()
+        continue
+    for sock in sockets:
+        sock.close()
+    print(base)
+    break
+else:
+    raise SystemExit("no free vLLM retry ports found")
+PY
+            )"
+            echo "  Retrying shard $IDX on GPU $SHARD_GPUS (ports $RETRY_PORT/$((RETRY_PORT + 1)))"
+            if ! CUDA_VISIBLE_DEVICES="$SHARD_GPUS" \
+                VLLM_PORT="$RETRY_PORT" \
+                VLLM_HOST_IP=127.0.0.1 \
+                MASTER_PORT="$((RETRY_PORT + 1))" \
+                MASTER_ADDR=127.0.0.1 \
+                PYTHONUNBUFFERED=1 \
+                python "$SCRIPT_DIR/eval_seg_vllm.py" "${PY_ARGS[@]}" \
+                    --chunk "$DP_SIZE" --index "$IDX" \
+                    > "$OUTPUT_DIR/worker_shard${IDX}_retry.log" 2>&1; then
+                RETRY_FAILURES+=("$IDX")
+            fi
+        done
+        FAILED_SHARDS=("${RETRY_FAILURES[@]}")
+    fi
+
+    if (( ${#FAILED_SHARDS[@]} > 0 )); then
+        echo "ERROR: Failed shards: ${FAILED_SHARDS[*]}; not merging or running SAM2."
+        exit 1
+    fi
+
+    echo ""
+    echo ">>> Merging shard results ..."
+    IFS=',' read -ra DATASET_LIST <<< "$DATASETS"
+    for DATASET in "${DATASET_LIST[@]}"; do
+        DATASET_KEY=$(python -c "from pathlib import Path; import sys; p=sys.argv[1]; print(Path(p).stem if Path(p).suffix else p)" "$DATASET")
+        python - "$OUTPUT_DIR" "$DATASET_KEY" "$DP_SIZE" <<'PY'
+import json
+import os
+import sys
+
+out_dir, dataset, num_shards = sys.argv[1], sys.argv[2], int(sys.argv[3])
+all_samples = []
+for sid in range(num_shards):
+    path = os.path.join(out_dir, f"results_{dataset}_shard{sid}.json")
+    if os.path.isfile(path):
+        with open(path, encoding="utf-8") as f:
+            payload = json.load(f)
+        all_samples.extend(payload.get("results", []))
+
+n = len(all_samples)
+if n == 0:
+    print(f"  No results for {dataset}")
+    sys.exit(0)
+
+parse_ok = sum(1 for row in all_samples if row.get("parse_ok"))
+summary = {
+    "num_samples": n,
+    "parse_ok": parse_ok,
+    "parse_rate": round(parse_ok / n * 100.0, 2),
+    "by_data_type": {},
+}
+for data_type in ("image", "video"):
+    part = [r for r in all_samples if r.get("data_type") == data_type]
+    if part:
+        ok = sum(1 for r in part if r.get("parse_ok"))
+        summary["by_data_type"][data_type] = {
+            "num_samples": len(part),
+            "parse_rate": round(ok / len(part) * 100.0, 2),
+        }
+
+with open(os.path.join(out_dir, f"results_{dataset}.json"), "w", encoding="utf-8") as f:
+    json.dump({"results": all_samples, "metrics": summary}, f, ensure_ascii=False, indent=2)
+
+summary_path = os.path.join(out_dir, "summary.json")
+full_summary = json.load(open(summary_path, encoding="utf-8")) if os.path.isfile(summary_path) else {}
+full_summary[dataset] = summary
+with open(summary_path, "w", encoding="utf-8") as f:
+    json.dump(full_summary, f, ensure_ascii=False, indent=2)
+
+print(f"  {dataset}: n={n} parse={summary['parse_rate']:.2f}%")
+PY
+    done
+fi
+
+if [ "$RUN_SAM2" = "true" ]; then
+    if [ -z "$SAM2_CKPT" ] || [ -z "$SAM2_CFG" ]; then
+        echo "ERROR: RUN_SAM2=true requires SAM2_CKPT and SAM2_CFG."
+        exit 1
+    fi
+    echo ""
+    echo ">>> Running SAM2 post-processing ..."
+    # SAM2 needs visible GPUs. The DP>1 inference path sets CUDA_VISIBLE_DEVICES
+    # only inside per-worker subshells, so the parent env may be empty here; an
+    # empty CUDA_VISIBLE_DEVICES makes torch.cuda.is_available() False and SAM2
+    # falls back to slow CPU serial. Re-derive a non-empty device list.
+    export CUDA_VISIBLE_DEVICES="$(IFS=,; echo "${GPULIST[*]}")"
+    # Default SAM2 GPU count to all visible GPUs unless caller overrides.
+    if [ -z "$SAM2_NUM_GPUS" ]; then
+        SAM2_NUM_GPUS="$NUM_GPUS"
+    fi
+    echo "    SAM2 GPUs: $CUDA_VISIBLE_DEVICES (num_gpus=$SAM2_NUM_GPUS)"
+    IFS=',' read -ra DATASET_LIST <<< "$DATASETS"
+    for DATASET in "${DATASET_LIST[@]}"; do
+        DATASET_KEY=$(python -c "from pathlib import Path; import sys; p=sys.argv[1]; print(Path(p).stem if Path(p).suffix else p)" "$DATASET")
+        RESULT_JSON="$OUTPUT_DIR/results_${DATASET_KEY}.json"
+        if [ ! -f "$RESULT_JSON" ]; then
+            echo "  Skip $DATASET_KEY: missing $RESULT_JSON"
+            continue
+        fi
+        SAM2_ARGS=(
+            --input_json "$RESULT_JSON"
+            --data_root "$DATA_ROOT"
+            --sam2_ckpt "$SAM2_CKPT"
+            --sam2_cfg "$SAM2_CFG"
+            --onethinker_script "$ONETHINKER_SEG_POST"
+            --pre_extract_threads "$PRE_EXTRACT_THREADS"
+            --epoch_size "$SAM2_EPOCH_SIZE"
+            --viz_ratio "$VIZ_RATIO"
+        )
+        if [ -n "$SAM2_NUM_GPUS" ]; then
+            SAM2_ARGS+=(--num_gpus "$SAM2_NUM_GPUS")
+        fi
+        if [ -n "$SAM2_WORKERS_PER_GPU" ]; then
+            SAM2_ARGS+=(--workers_per_gpu "$SAM2_WORKERS_PER_GPU")
+        fi
+        PYTHONUNBUFFERED=1 python "$SCRIPT_DIR/post_sam2.py" "${SAM2_ARGS[@]}" \
+            2>&1 | tee "$OUTPUT_DIR/sam2_${DATASET_KEY}.log"
+
+        # Merge SAM2 metrics (cIoU / gIoU / J&F) back into summary.json.
+        SAM2_JSON="$OUTPUT_DIR/results_${DATASET_KEY}_sam2.json"
+        if [ -f "$SAM2_JSON" ]; then
+            python - "$OUTPUT_DIR" "$DATASET_KEY" "$SAM2_JSON" <<'PY'
+import json
+import os
+import sys
+
+out_dir, dataset, sam2_json = sys.argv[1], sys.argv[2], sys.argv[3]
+with open(sam2_json, encoding="utf-8") as f:
+    payload = json.load(f)
+
+metrics = payload.get("metrics", {})
+avg_rewards = payload.get("avg_rewards", {})
+
+summary_path = os.path.join(out_dir, "summary.json")
+full_summary = json.load(open(summary_path, encoding="utf-8")) if os.path.isfile(summary_path) else {}
+entry = full_summary.get(dataset, {})
+if isinstance(metrics, dict):
+    for k in ("num_samples", "parse_ok", "parse_rate"):
+        if k in metrics:
+            entry[k] = metrics[k]
+entry["avg_rewards"] = avg_rewards
+full_summary[dataset] = entry
+with open(summary_path, "w", encoding="utf-8") as f:
+    json.dump(full_summary, f, ensure_ascii=False, indent=2)
+
+parts = ", ".join(f"{k}={v:.4f}" for k, v in avg_rewards.items())
+print(f"  {dataset}: {parts}")
+PY
+        fi
+    done
+fi
+
+echo ""
+echo "=============================================="
+echo "Done. Summary: $OUTPUT_DIR/summary.json"
+echo "=============================================="

code/eval/task/spatial_grounding/_grounding_utils.py

@@ -0,0 +1,386 @@
+"""Shared helpers for RefCOCO / RefCOCO+ / RefCOCOg evaluation.
+
+Pure-Python (no torch / vLLM / HF dependencies) so it can be imported by both
+the inference script (``eval_refcoco_vllm.py``) and the offline rescorer
+(``rescore_results.py``).
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import sys
+from typing import Any, Dict, List, Optional, Tuple
+
+
+# ---------------------------------------------------------------------------
+# Prompts — single source of truth in eval/task/eval_prompt.py
+# (qwen_native matches data/joint/sft_joint_all.jsonl spatial grounding samples).
+# ---------------------------------------------------------------------------
+_EVAL_TASK_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+if _EVAL_TASK_DIR not in sys.path:
+    sys.path.insert(0, _EVAL_TASK_DIR)
+
+from canonical_data import load_json_records  # noqa: E402
+from eval_prompt import QWEN_NATIVE_PROMPT_SG  # noqa: E402
+
+
+def _with_terminal_period(expression: str) -> str:
+    expression = (expression or "").strip()
+    if expression and expression[-1] not in ".?!":
+        expression += "."
+    return expression
+
+
+def build_qwen_native_prompt(expression: str) -> str:
+    # The 13k iou<0.95 training set uses the qwen-native JSON bbox format,
+    # wrapped in <answer> tags for consistency with the joint SFT mixture:
+    #   <image>Locate "expr." in the image. Output its bounding box in JSON
+    #   format within <answer>...</answer> tags. Example: ...
+    # Keep eval byte-for-byte aligned on the user text after the <image> token.
+    return QWEN_NATIVE_PROMPT_SG.format(_with_terminal_period(expression))
+
+
+# Alternate grounding prompt styles below are NOT used by the eval pipeline
+# (eval.sh always runs qwen_native). They are retained only for the offline
+# analysis / data-selection tools in this folder (compute_difficulty.py,
+# select_train_data.py), so they live here rather than in eval_prompt.py.
+_QWEN_OFFICIAL_PROMPT_SG = (
+    'Locate every object that matches the description "{}" in the image. '
+    "Report bbox coordinates in JSON format."
+)
+_EVAL_BENCH_SG_PROMPT = (
+    'Locate "{}" in the image. Output its bounding box in JSON format '
+    'within <answer>...</answer> tags. '
+    'Example: <answer>[{{"bbox_2d": [123, 30, 404, 846]}}]</answer>'
+)
+
+
+def build_qwen_official_prompt(expression: str) -> str:
+    return _QWEN_OFFICIAL_PROMPT_SG.format(expression or "")
+
+
+def build_eval_bench_prompt(expression: str) -> str:
+    return _EVAL_BENCH_SG_PROMPT.format(expression or "")
+
+
+# ---------------------------------------------------------------------------
+# Data loading
+# ---------------------------------------------------------------------------
+
+DATASET_CONFIGS = {
+    "refcoco-val": {"anno": "refcoco_val.json"},
+    "refcoco-testA": {"anno": "refcoco_testA.json"},
+    "refcoco-testB": {"anno": "refcoco_testB.json"},
+    "refcoco-train": {"anno": "refcoco_train.json"},
+    "refcoco+-val": {"anno": "refcocop_val.json"},
+    "refcoco+-testA": {"anno": "refcocop_testA.json"},
+    "refcoco+-testB": {"anno": "refcocop_testB.json"},
+    "refcoco+-train": {"anno": "refcocop_train.json"},
+    "refcocop-val": {"anno": "refcocop_val.json"},
+    "refcocop-testA": {"anno": "refcocop_testA.json"},
+    "refcocop-testB": {"anno": "refcocop_testB.json"},
+    "refcocop-train": {"anno": "refcocop_train.json"},
+    "refcocog-val": {"anno": "refcocog_val.json"},
+    "refcocog-test": {"anno": "refcocog_test.json"},
+    "refcocog-train": {"anno": "refcocog_train.json"},
+}
+CANONICAL_DATASET_NAMES = {
+    "refcoco-val": "refcoco_val",
+    "refcoco-testA": "refcoco_test_a",
+    "refcoco-testB": "refcoco_test_b",
+    "refcoco+-val": "refcocop_val",
+    "refcoco+-testA": "refcocop_test_a",
+    "refcoco+-testB": "refcocop_test_b",
+    "refcocog-val": "refcocog_val",
+    "refcocog-test": "refcocog_test",
+}
+for _canonical_name in CANONICAL_DATASET_NAMES.values():
+    DATASET_CONFIGS.setdefault(
+        _canonical_name,
+        {"anno": f"{_canonical_name}.jsonl"},
+    )
+
+
+def _first_existing_path(paths: List[str]) -> Optional[str]:
+    for path in paths:
+        if path and os.path.isfile(path):
+            return path
+    return None
+
+
+def _resolve_annotation_path(bench_dir: str, anno_name: str, dataset_name: str) -> str:
+    here = os.path.dirname(os.path.abspath(__file__))
+    canonical_name = CANONICAL_DATASET_NAMES.get(dataset_name, dataset_name)
+    candidates = [
+        os.path.join(bench_dir, f"{canonical_name}.jsonl"),
+        os.path.join(bench_dir, "annotations", f"{canonical_name}.jsonl"),
+        os.path.join(bench_dir, anno_name),
+        os.path.join(bench_dir, "rec_jsons_processed", anno_name),
+        os.path.join(bench_dir, "annotations", anno_name),
+        os.path.join(bench_dir, "annotations", "rec_jsons_processed", anno_name),
+        os.path.join(bench_dir, f"{dataset_name}.json"),
+        os.path.join(bench_dir, "rec_jsons_processed", f"{dataset_name}.json"),
+        os.path.join(here, "rec_jsons_processed", anno_name),
+    ]
+    anno_path = _first_existing_path(candidates)
+    if anno_path is None:
+        raise FileNotFoundError(
+            f"Could not find annotations for {dataset_name!r}. Tried: "
+            + ", ".join(candidates)
+        )
+    return anno_path
+
+
+def _resolve_image_root(bench_dir: str, anno_path: str, sample_image_ref: str) -> str:
+    if not sample_image_ref or os.path.isabs(sample_image_ref):
+        return ""
+
+    override = os.getenv("SPATIAL_GROUNDING_IMAGE_ROOT", "").strip()
+    if override:
+        return override
+
+    anno_dir = os.path.dirname(os.path.abspath(anno_path))
+    bench_parent = os.path.dirname(os.path.abspath(bench_dir))
+    rel_dir = os.path.dirname(sample_image_ref)
+    candidates = [
+        bench_dir,
+        os.path.join(bench_dir, "images"),
+        os.path.join(bench_dir, "Spatial-Grounding"),
+        os.path.join(bench_parent, "Spatial-Grounding"),
+        anno_dir,
+        os.path.dirname(anno_dir),
+    ]
+    for root in candidates:
+        probe = os.path.join(root, rel_dir) if rel_dir else root
+        if os.path.isdir(probe):
+            return root
+    return bench_dir
+
+
+def _resolve_image_path(image_ref: str, image_root: str) -> str:
+    if not image_ref:
+        return ""
+    if os.path.isabs(image_ref):
+        return image_ref
+    direct = os.path.join(image_root, image_ref)
+    if os.path.exists(direct):
+        return direct
+    basename_path = os.path.join(image_root, os.path.basename(image_ref))
+    if os.path.exists(basename_path):
+        return basename_path
+    return direct
+
+
+def _as_box(value: Any) -> Optional[List[float]]:
+    if not isinstance(value, (list, tuple)) or len(value) != 4:
+        return None
+    try:
+        return [float(x) for x in value]
+    except (TypeError, ValueError):
+        return None
+
+
+def load_annotations(bench_dir: str, dataset_name: str) -> List[Dict[str, Any]]:
+    cfg = DATASET_CONFIGS.get(
+        dataset_name,
+        {"anno": f"{dataset_name}.jsonl"},
+    )
+    anno_path = _resolve_annotation_path(bench_dir, cfg["anno"], dataset_name)
+    if anno_path.endswith(".jsonl"):
+        raw = load_json_records(anno_path)
+    else:
+        with open(anno_path, "r") as f:
+            raw = json.load(f)
+
+    rows = raw if isinstance(raw, list) else list(raw.values())
+    first_image_ref = ""
+    for item in rows:
+        first_image_ref = item.get("image_path") or item.get("image") or item.get("path") or ""
+        if first_image_ref:
+            break
+    image_root = _resolve_image_root(bench_dir, anno_path, first_image_ref)
+
+    annos: List[Dict[str, Any]] = []
+    for idx, item in enumerate(rows):
+        expression = (
+            item.get("expression")
+            or item.get("normal_caption")
+            or item.get("caption")
+            or item.get("query")
+            or item.get("problem")
+            or ""
+        )
+        bbox = (
+            _as_box(item.get("bbox"))
+            or _as_box(item.get("normalized_solution"))
+            or _as_box(item.get("gt_bbox"))
+            or _as_box(item.get("solution"))
+        )
+        image_ref = item.get("image_path") or item.get("image") or item.get("path") or ""
+        if not expression or bbox is None or not image_ref:
+            continue
+
+        annos.append({
+            "problem_id": item.get("problem_id", item.get("id", idx)),
+            "image_path": _resolve_image_path(image_ref, image_root),
+            "expression": re.sub(r"\s+", " ", str(expression)).strip().strip("."),
+            "bbox": bbox,
+            "width": item.get("width"),
+            "height": item.get("height"),
+        })
+    return annos
+
+
+# ---------------------------------------------------------------------------
+# Bounding-box extraction / scoring
+# ---------------------------------------------------------------------------
+
+ANSWER_RE = re.compile(r"<answer>\s*(.*?)\s*</answer>", re.DOTALL | re.IGNORECASE)
+NUMBER_RE = r"-?\d+(?:\.\d+)?"
+
+
+def _clean_jsonish(text: str) -> str:
+    text = (text or "").strip()
+    if text.startswith("```"):
+        text = re.sub(r"^```(?:json)?\s*", "", text, flags=re.IGNORECASE)
+        text = re.sub(r"\s*```$", "", text)
+    return text.strip()
+
+
+def _iter_json_values(text: str) -> List[Any]:
+    decoder = json.JSONDecoder()
+    values: List[Any] = []
+    for i, ch in enumerate(text):
+        if ch not in "[{":
+            continue
+        try:
+            value, _ = decoder.raw_decode(text[i:])
+        except json.JSONDecodeError:
+            continue
+        values.append(value)
+    return values
+
+
+def _append_candidate(
+    out: List[Dict[str, Any]],
+    box: Any,
+    label: Optional[str] = None,
+) -> None:
+    parsed = _as_box(box)
+    if parsed is not None:
+        out.append({"bbox": parsed, "label": label})
+
+
+def _extract_from_json_value(value: Any, out: List[Dict[str, Any]],
+                             label: Optional[str] = None) -> None:
+    if isinstance(value, dict):
+        local_label = value.get("label") or value.get("name") or value.get("category") or label
+        for key in ("bbox", "bbox_2d", "box", "boxes", "bboxes"):
+            if key not in value:
+                continue
+            box_value = value[key]
+            if _as_box(box_value) is not None:
+                _append_candidate(out, box_value, local_label)
+            elif isinstance(box_value, list):
+                for entry in box_value:
+                    if _as_box(entry) is not None:
+                        _append_candidate(out, entry, local_label)
+                    else:
+                        _extract_from_json_value(entry, out, local_label)
+        for key in ("objects", "items", "regions", "detections", "predictions"):
+            if key in value:
+                _extract_from_json_value(value[key], out, local_label)
+    elif isinstance(value, list):
+        if _as_box(value) is not None:
+            _append_candidate(out, value, label)
+        else:
+            for entry in value:
+                _extract_from_json_value(entry, out, label)
+
+
+def extract_bboxes(text: str) -> List[Dict[str, Any]]:
+    """Extract candidate bboxes from Qwen JSON and legacy R1 answer formats."""
+    if not text:
+        return []
+
+    candidates: List[Dict[str, Any]] = []
+    chunks = [m.group(1) for m in ANSWER_RE.finditer(text)] or [text]
+    for chunk in chunks:
+        cleaned = _clean_jsonish(chunk)
+        for value in _iter_json_values(cleaned):
+            _extract_from_json_value(value, candidates)
+
+        pair_pat = re.compile(
+            rf"\(({NUMBER_RE})\s*,\s*({NUMBER_RE})\)\s*,?\s*"
+            rf"\(({NUMBER_RE})\s*,\s*({NUMBER_RE})\)"
+        )
+        for match in pair_pat.finditer(cleaned):
+            _append_candidate(candidates, [float(x) for x in match.groups()])
+
+        bracket_pat = re.compile(
+            rf"\[\s*({NUMBER_RE})\s*,\s*({NUMBER_RE})\s*,\s*"
+            rf"({NUMBER_RE})\s*,\s*({NUMBER_RE})\s*\]"
+        )
+        for match in bracket_pat.finditer(cleaned):
+            _append_candidate(candidates, [float(x) for x in match.groups()])
+
+    if candidates:
+        return _dedupe_candidates(candidates)
+
+    nums = re.findall(NUMBER_RE, ANSWER_RE.sub(r"\1", text))
+    if len(nums) >= 4:
+        _append_candidate(candidates, [float(x) for x in nums[:4]])
+    return _dedupe_candidates(candidates)
+
+
+def _dedupe_candidates(candidates: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    seen = set()
+    unique: List[Dict[str, Any]] = []
+    for cand in candidates:
+        bbox = cand.get("bbox")
+        if bbox is None:
+            continue
+        key = tuple(round(float(x), 6) for x in bbox)
+        if key in seen:
+            continue
+        seen.add(key)
+        unique.append(cand)
+    return unique
+
+
+def compute_iou_2d(box1: Any, box2: Any) -> float:
+    b1 = _as_box(box1)
+    b2 = _as_box(box2)
+    if b1 is None or b2 is None:
+        return 0.0
+
+    x1 = max(b1[0], b2[0])
+    y1 = max(b1[1], b2[1])
+    x2 = min(b1[2], b2[2])
+    y2 = min(b1[3], b2[3])
+    inter = max(0.0, x2 - x1) * max(0.0, y2 - y1)
+    area1 = max(0.0, b1[2] - b1[0]) * max(0.0, b1[3] - b1[1])
+    area2 = max(0.0, b2[2] - b2[0]) * max(0.0, b2[3] - b2[1])
+    union = area1 + area2 - inter
+    return inter / union if union > 1e-12 else 0.0
+
+
+def select_pred_bbox(candidates: List[Dict[str, Any]], gt_box: Any,
+                     mode: str = "first") -> Tuple[Optional[List[float]], Optional[str]]:
+    if not candidates:
+        return None, None
+    if mode == "best_iou":
+        best = max(candidates, key=lambda c: compute_iou_2d(gt_box, c.get("bbox")))
+    else:
+        best = candidates[0]
+    return best.get("bbox"), best.get("label")
+
+
+def sanitize_video_kwargs(video_kwargs: Any, has_video: bool = False) -> Dict[str, Any]:
+    if not has_video:
+        return {}
+    if not isinstance(video_kwargs, dict):
+        return {}
+    return {k: v for k, v in video_kwargs.items() if v is not None}