diff --git a/.gitattributes b/.gitattributes index 0fd17d2f7c330afce8bc2b472393143fa66f68a1..b6d6069c9fed01931405110b12cba17f1da5b723 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,43 @@ *.safetensors filter=lfs diff=lfs merge=lfs -text tokenizer.json filter=lfs diff=lfs merge=lfs -text +code/assets/orarl-data-scaling.gif filter=lfs diff=lfs merge=lfs -text +code/assets/orarl-hero.gif filter=lfs diff=lfs merge=lfs -text +code/assets/orarl-method.gif filter=lfs diff=lfs merge=lfs -text +code/assets/orarl-model-scaling.gif filter=lfs diff=lfs merge=lfs -text +code/assets/orarl-teaser.mp4 filter=lfs diff=lfs merge=lfs -text +code/assets/paper-data-scaling.png filter=lfs diff=lfs merge=lfs -text +code/assets/paper-framework.png filter=lfs diff=lfs merge=lfs -text +code/assets/paper-model-scaling.png filter=lfs diff=lfs merge=lfs -text +code/assets/paper-results.png filter=lfs diff=lfs merge=lfs -text +code/runs/smoke-training-9b-20260825_175949/grpo/global_step_1/actor/model_world_size_8_rank_0.pt filter=lfs diff=lfs merge=lfs -text +code/runs/smoke-training-9b-20260825_175949/grpo/global_step_1/actor/model_world_size_8_rank_1.pt filter=lfs diff=lfs merge=lfs -text +code/runs/smoke-training-9b-20260825_175949/grpo/global_step_1/actor/model_world_size_8_rank_2.pt filter=lfs diff=lfs merge=lfs -text +code/runs/smoke-training-9b-20260825_175949/grpo/global_step_1/actor/model_world_size_8_rank_3.pt filter=lfs diff=lfs merge=lfs -text +code/runs/smoke-training-9b-20260825_175949/grpo/global_step_1/actor/model_world_size_8_rank_4.pt filter=lfs diff=lfs merge=lfs -text +code/runs/smoke-training-9b-20260825_175949/grpo/global_step_1/actor/model_world_size_8_rank_5.pt filter=lfs diff=lfs merge=lfs -text +code/runs/smoke-training-9b-20260825_175949/grpo/global_step_1/actor/model_world_size_8_rank_6.pt filter=lfs diff=lfs merge=lfs -text +code/runs/smoke-training-9b-20260825_175949/grpo/global_step_1/actor/model_world_size_8_rank_7.pt filter=lfs diff=lfs merge=lfs -text +code/runs/smoke-training-9b-20260825_175949/grpo/global_step_1/actor/optim_world_size_8_rank_0.pt filter=lfs diff=lfs merge=lfs -text +code/runs/smoke-training-9b-20260825_175949/grpo/global_step_1/actor/optim_world_size_8_rank_1.pt filter=lfs diff=lfs merge=lfs -text +code/runs/smoke-training-9b-20260825_175949/grpo/global_step_1/actor/optim_world_size_8_rank_2.pt filter=lfs diff=lfs merge=lfs -text +code/runs/smoke-training-9b-20260825_175949/grpo/global_step_1/actor/optim_world_size_8_rank_3.pt filter=lfs diff=lfs merge=lfs -text +code/runs/smoke-training-9b-20260825_175949/grpo/global_step_1/actor/optim_world_size_8_rank_4.pt filter=lfs diff=lfs merge=lfs -text +code/runs/smoke-training-9b-20260825_175949/grpo/global_step_1/actor/optim_world_size_8_rank_5.pt filter=lfs diff=lfs merge=lfs -text +code/runs/smoke-training-9b-20260825_175949/grpo/global_step_1/actor/optim_world_size_8_rank_6.pt filter=lfs diff=lfs merge=lfs -text +code/runs/smoke-training-9b-20260825_175949/grpo/global_step_1/actor/optim_world_size_8_rank_7.pt filter=lfs diff=lfs merge=lfs -text +code/runs/smoke-training-9b-20260825_175949/orarl/global_step_1/actor/model_world_size_8_rank_0.pt filter=lfs diff=lfs merge=lfs -text +code/runs/smoke-training-9b-20260825_175949/orarl/global_step_1/actor/model_world_size_8_rank_1.pt filter=lfs diff=lfs merge=lfs -text +code/runs/smoke-training-9b-20260825_175949/orarl/global_step_1/actor/model_world_size_8_rank_2.pt filter=lfs diff=lfs merge=lfs -text +code/runs/smoke-training-9b-20260825_175949/orarl/global_step_1/actor/model_world_size_8_rank_3.pt filter=lfs diff=lfs merge=lfs -text +code/runs/smoke-training-9b-20260825_175949/orarl/global_step_1/actor/model_world_size_8_rank_4.pt filter=lfs diff=lfs merge=lfs -text +code/runs/smoke-training-9b-20260825_175949/orarl/global_step_1/actor/model_world_size_8_rank_5.pt filter=lfs diff=lfs merge=lfs -text +code/runs/smoke-training-9b-20260825_175949/orarl/global_step_1/actor/model_world_size_8_rank_6.pt filter=lfs diff=lfs merge=lfs -text +code/runs/smoke-training-9b-20260825_175949/orarl/global_step_1/actor/model_world_size_8_rank_7.pt filter=lfs diff=lfs merge=lfs -text +code/runs/smoke-training-9b-20260825_175949/orarl/global_step_1/actor/optim_world_size_8_rank_0.pt filter=lfs diff=lfs merge=lfs -text +code/runs/smoke-training-9b-20260825_175949/orarl/global_step_1/actor/optim_world_size_8_rank_1.pt filter=lfs diff=lfs merge=lfs -text +code/runs/smoke-training-9b-20260825_175949/orarl/global_step_1/actor/optim_world_size_8_rank_2.pt filter=lfs diff=lfs merge=lfs -text +code/runs/smoke-training-9b-20260825_175949/orarl/global_step_1/actor/optim_world_size_8_rank_3.pt filter=lfs diff=lfs merge=lfs -text +code/runs/smoke-training-9b-20260825_175949/orarl/global_step_1/actor/optim_world_size_8_rank_4.pt filter=lfs diff=lfs merge=lfs -text +code/runs/smoke-training-9b-20260825_175949/orarl/global_step_1/actor/optim_world_size_8_rank_5.pt filter=lfs diff=lfs merge=lfs -text +code/runs/smoke-training-9b-20260825_175949/orarl/global_step_1/actor/optim_world_size_8_rank_6.pt filter=lfs diff=lfs merge=lfs -text +code/runs/smoke-training-9b-20260825_175949/orarl/global_step_1/actor/optim_world_size_8_rank_7.pt filter=lfs diff=lfs merge=lfs -text diff --git a/code/.gitignore b/code/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..26b0d7c7ee0d350fcb5620bae3df66190df74965 --- /dev/null +++ b/code/.gitignore @@ -0,0 +1,80 @@ +# Python and test state +__pycache__/ +*.py[cod] +*.egg-info/ +.pytest_cache/ +.ruff_cache/ +.mypy_cache/ +.nox/ +.tox/ +.coverage +htmlcov/ +build/ +dist/ +.venv/ +venv/ + +# Local inputs and generated runs +/artifacts/ +/cache/ +/checkpoint/ +/checkpoints/ +/data/ +/local_data/ +/logs/ +/models/ +/outputs/ +/prepared/ +/ray_results/ +/runs/ +/wandb/ +/orarl-eval-summary.json +nohup.out +core.* + +# Model, media, and generated dataset payloads +*.arrow +*.avi +*.bin +*.ckpt +*.jsonl +*.mkv +*.mov +*.mp4 +*.npy +*.npz +*.parquet +*.pt +*.pth +*.safetensors +*.webm +*.log + +# Only the small evaluation profile manifest is release source. Canonical +# annotations, media, subtitles, and processed artifacts live on Hugging Face. +!/data/ +/data/* +!/data/eval/ +/data/eval/* +!/data/eval/datasets.jsonl +!/data/eval/README.md + +# README media shipped with the repository +!/assets/orarl-teaser.mp4 + +# Editor and operating-system state +.DS_Store +.idea/ +.ipynb_checkpoints/ +.vscode/ +*~ + +# Local configuration +.env +.env.* +*.local.yaml +*.local.yml +*.key +*.pem +credentials.json +token diff --git a/code/LICENSE b/code/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..c77c560e3c3592b45f6b463007e701e099c4fc05 --- /dev/null +++ b/code/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2026 The OraRL Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/code/MANIFEST.in b/code/MANIFEST.in new file mode 100644 index 0000000000000000000000000000000000000000..fd11102cd1593b8341e9739e00c43f2531ddb036 --- /dev/null +++ b/code/MANIFEST.in @@ -0,0 +1,20 @@ +include LICENSE +include NOTICE +include README.md +include environment.yml +include pyproject.toml +include requirements-cu129.txt +recursive-include configs *.yaml +recursive-include data/eval *.jsonl *.md +recursive-include docs *.md +recursive-include eval *.md *.py *.sh +recursive-include scripts *.py *.sh +recursive-include tests *.py +recursive-include verl *.py +prune build +prune dist +prune .pytest_cache +prune .ruff_cache +prune checkpoint +prune checkpoints +global-exclude __pycache__ *.py[cod] .DS_Store diff --git a/code/NOTICE b/code/NOTICE new file mode 100644 index 0000000000000000000000000000000000000000..ffe9c141d8619701b27b001e1745bfb4ec935218 --- /dev/null +++ b/code/NOTICE @@ -0,0 +1,7 @@ +OraRL +Copyright 2026 The OraRL Authors + +Portions of this software are derived from veRL and other Apache-2.0 upstream +projects. The derived portions retain their original copyright notices, +including Copyright 2024 ByteDance Ltd. and/or its affiliates, and are provided +under the Apache License, Version 2.0. diff --git a/code/README.md b/code/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b98a5405f22f9458f284d8fcff9bf8b373387505 --- /dev/null +++ b/code/README.md @@ -0,0 +1,201 @@ +

简体中文

+ +
+ +# OraRL + +### Annotations as Rollouts + +**Efficient and scalable reinforcement learning for unified video MLLMs** + +Yunheng Li · Guohong Mu · Hao Li · Shengsheng Qian · Dingwen Zhang · +Qibin Hou · Ming-Ming Cheng + +

+ 📄 Paper +   ·   + 🌐 Project Page +   ·   + 🤗 Models (4B / 9B) +

+

+ ⚙️ Environment +   ·   + 🚀 Training +   ·   + 📊 Evaluation +   ·   + ⚖️ License +

+ + + Animated OraRL method preview + + +**▶ Click the image to watch the 1:38 project overview.** + +
+ +## Why OraRL + +- **Annotation-as-rollout:** annotations become reliable positive rollouts while + policy samples retain an on-policy baseline. +- **Seven task families:** one update rule covers temporal and spatial grounding, + segmentation, tracking, spatial-temporal grounding, video QA, and spatial + intelligence. +- **Efficient training (4B):** sign-balanced pruning delivers **1.48× faster + updates** (**92.5 → 62.4 s/step**) while reducing peak per-GPU memory from + **62.4 to 50.9 GB**. +- **Efficient inference:** on one H20 with vLLM in BF16, weight loading occupies + **8.6 GiB (4B)** and **17.6 GiB (9B)**. On ten-minute, 2-fps videos, + answer-only decoding cuts median post-TTFT latency from **4.78 s to 130 ms** + and total latency from **29.03 to 24.30 s**. +- **Multimodal veRL infrastructure:** a unified video contract carries cached + artifacts, raw paths, or inline frame tensors through vLLM rollouts and FSDP + updates, with decode-once frame reuse, temporal metadata, task-grouped + batching, asynchronous Ray rewards, and safe hybrid-engine cache handling. + +## OraRL in One Update + +

+ Animated OraRL framework +

+ +An OraRL update separates reliable annotation guidance from on-policy +normalization: + +1. **Build the group:** append one serialized annotation rollout to the policy + samples generated for the same prompt. +2. **Keep the baseline on-policy:** estimate the group baseline from policy + rewards only. +3. **Guide and select:** convert the annotation-policy reward gap into a + correction, then retain a sign-balanced subset for the update. + +This design uses task-native annotations directly and requires no +chain-of-thought supervision or decoding. + +## Video-ORA Results + +

+ Video-ORA-9B results across seven task families +

+ +### Dataset-Level Results + + + + + Dataset-level benchmark matrix comparing Video-ORA with multimodal baselines + + +Video-ORA-9B leads the matched seven-family comparison without CoT decoding. +Best and second-best values are highlighted per row; `†` denotes an +original-report value whose frame, prompt, split, or decoding settings may +differ. Averages require complete family coverage. + + + +### Model Scaling + +

+ Animated Video-ORA model scaling from 0.8B to 9B +

+ +### Data Scaling + +

+ Animated OraRL data scaling and reward dynamics +

+ +## Models + +| Model | Backbone | Released recipe | Weights | +| --- | --- | --- | --- | +| **Video-ORA-9B** | Qwen3.5-9B | `orarl_9b.yaml` | [Hugging Face](https://huggingface.co/OraRL/Video-ORA-9B) | +| **Video-ORA-4B** | Qwen3.5-4B | `orarl_4b.yaml` | Hugging Face (coming soon) | + +### vLLM Serving + +Both Video-ORA checkpoints load directly with **vLLM 0.19.1** for +OpenAI-compatible serving: + +```bash +MODEL=OraRL/Video-ORA-9B + +vllm serve "$MODEL" \ + --served-model-name Video-ORA-9B \ + --trust-remote-code \ + --dtype bfloat16 \ + --tensor-parallel-size 1 \ + --max-model-len 131072 \ + --limit-mm-per-prompt '{"image": 1, "video": 1}' +``` + +Set `--tensor-parallel-size` to the GPU count for multi-GPU deployment and +lower `--max-model-len` on smaller-memory devices. Use +`enable_thinking=false` in the chat template for answer-only inference. + +## Use OraRL + +The release is organized around three user-facing workflows: + +1. **[Environment](docs/environment.md):** install the pinned CUDA stack that + covers both the bundled trainer and the evaluators. +2. **[Training](docs/training.md):** prepare licensed local training data and + launch GRPO or OraRL on one or multiple nodes. +3. **[Evaluation](docs/evaluation.md):** download Video-ORA and OraRL-Data, + then run a smoke test or the complete paper suite. + +Training and evaluation are dry runs by default; inspect the resolved command +before adding `--run`. Checkpoints and evaluation media are hosted under the +[OraRL Hugging Face organization](https://huggingface.co/OraRL). + +## Acknowledgements + +OraRL is built on [veRL](https://github.com/volcengine/verl) — a +high-performance RL framework with HybridEngine. We thank its authors and +contributors for open-sourcing the training infrastructure. + +## License + +OraRL source is released under [Apache-2.0](LICENSE). Datasets, models, +benchmarks, and optional dependencies retain their original licenses; see +[NOTICE](NOTICE). + +## Citation + +If you find OraRL useful, please consider giving this repository a ⭐ and +citing our [paper](https://arxiv.org/abs/2608.20492). + +```bibtex +@article{li2026orarl, + title = {Annotations as Rollouts: Efficient and Scalable + Reinforcement Learning for Video MLLMs}, + author = {Li, Yunheng and Mu, Guohong and Li, Hao and + Qian, Shengsheng and Zhang, Dingwen and Hou, Qibin + and Cheng, Ming-Ming}, + journal = {arXiv preprint arXiv:2608.20492}, + year = {2026}, + url = {https://arxiv.org/abs/2608.20492} +} +``` diff --git a/code/README_zh.md b/code/README_zh.md new file mode 100644 index 0000000000000000000000000000000000000000..5c7824cc6a7ec68d004956ea1a5f6fda7bef22c9 --- /dev/null +++ b/code/README_zh.md @@ -0,0 +1,193 @@ +

English

+ +
+ +# OraRL + +### 将标注作为 Rollout + +**面向统一视频多模态大模型的高效、可扩展强化学习** + +Yunheng Li · Guohong Mu · Hao Li · Shengsheng Qian · Dingwen Zhang · +Qibin Hou · Ming-Ming Cheng + +

+ 📄 论文 +   ·   + 🌐 项目主页 +   ·   + 🤗 模型（4B / 9B） +

+

+ ⚙️ 环境 +   ·   + 🚀 训练 +   ·   + 📊 评测 +   ·   + ⚖️ 许可证 +

+ + + OraRL 方法动态预览 + + +**▶ 点击图片观看 1 分 38 秒的项目介绍视频。** + +
+ +## 为什么选择 OraRL + +- **将标注作为 Rollout：** 将标注转化为可靠的正样本 rollout，同时策略样本 + 仍使用纯 on-policy 基线。 +- **七类任务统一训练：** 同一套更新规则覆盖时序定位、空间定位、分割、跟踪、 + 时空定位、视频问答和空间智能。 +- **高效训练（4B）：** 符号均衡剪枝带来 **1.48× 的更新加速** + （**92.5 → 62.4 秒/步**），同时将单卡峰值显存从 **62.4 GB 降至 + 50.9 GB**。 +- **高效推理：** 在单张 H20 上使用 vLLM 和 BF16 时，权重加载占用分别为 + **8.6 GiB（4B）**和 **17.6 GiB（9B）**。对于 2 fps 采样的十分钟视频， + 仅生成答案将首 token 后的中位延迟从 **4.78 秒降至 130 毫秒**，并将总 + 中位延迟从 **29.03 秒降至 24.30 秒**。 +- **多模态 veRL 基础设施：** 统一的视频数据契约支持缓存产物、原始路径和内联 + 帧张量，并贯通 vLLM rollout 与 FSDP 更新；同时加入单次解码帧复用、时序 + 元数据、按任务分组的批处理、异步 Ray 奖励和安全的混合引擎缓存处理。 + +## 一次 OraRL 更新 + +

+ OraRL 方法框架动画 +

+ +一次 OraRL 更新将可靠的标注引导与 on-policy 归一化明确分开： + +1. **构造样本组：** 在同一 prompt 的策略样本后加入一个序列化标注 rollout。 +2. **保持 on-policy 基线：** 仅使用策略样本的奖励估计组内基线。 +3. **引导并筛选：** 将标注与策略样本的奖励差转化为修正量，再选择符号均衡的 + 子集执行更新。 + +该设计直接使用任务原生标注，不需要思维链监督或解码。 + +## Video-ORA 结果 + +

+ Video-ORA-9B 在七类任务上的结果 +

+ +### 数据集级结果 + + + + + Video-ORA 与多模态基线的数据集级对比矩阵 + + +Video-ORA-9B 在不使用思维链解码的情况下，在统一的七类任务对比中取得领先。 +每行的最佳和次佳结果均已突出显示；`†` 表示该数值来自原始报告，其帧数、提示词、 +数据划分或解码设置可能不同。仅在完整覆盖该任务类别时才计算类别平均值。 + + + +### 模型规模扩展 + +

+ Video-ORA 从 0.8B 到 9B 的模型规模扩展动画 +

+ +### 数据规模扩展 + +

+ OraRL 数据规模扩展与奖励变化动画 +

+ + + +## 模型 + +| 模型 | 基座模型 | 发布配置 | 权重 | +| --- | --- | --- | --- | +| **Video-ORA-9B** | Qwen3.5-9B | `orarl_9b.yaml` | [Hugging Face](https://huggingface.co/OraRL/Video-ORA-9B) | +| **Video-ORA-4B** | Qwen3.5-4B | `orarl_4b.yaml` | Hugging Face（即将发布） | + +### vLLM 部署 + +两个 Video-ORA 检查点均可通过 **vLLM 0.19.1** 直接提供兼容 OpenAI 的 +推理服务： + +```bash +MODEL=OraRL/Video-ORA-9B + +vllm serve "$MODEL" \ + --served-model-name Video-ORA-9B \ + --trust-remote-code \ + --dtype bfloat16 \ + --tensor-parallel-size 1 \ + --max-model-len 131072 \ + --limit-mm-per-prompt '{"image": 1, "video": 1}' +``` + +多卡部署时，将 `--tensor-parallel-size` 设为 GPU 数量；显存较小时可降低 +`--max-model-len`。如需仅生成答案，请在聊天模板中设置 +`enable_thinking=false`。 + +## 使用 OraRL + +本仓库围绕三个面向用户的工作流组织： + +1. **[环境](docs/environment_zh.md)：** 安装同时覆盖自带训练运行时与评测运行时的 + 固定 CUDA 软件栈。 +2. **[训练](docs/training_zh.md)：** 准备已获许可的本地训练数据，并启动单节点或 + 多节点 GRPO/OraRL。 +3. **[评测](docs/evaluation_zh.md)：** 下载 Video-ORA 和 OraRL-Data，运行 + smoke test 或完整论文评测。 + +训练和评测默认仅执行 dry run；检查解析后的命令后再添加 `--run`。Checkpoint +和评测媒体托管在 [OraRL Hugging Face 组织](https://huggingface.co/OraRL)下。 + +## 致谢 + +OraRL 基于 [veRL](https://github.com/volcengine/verl) 构建——这是一个采用 +HybridEngine 的高性能强化学习框架。感谢其作者和贡献者开源训练基础设施。 + +## 许可证 + +OraRL 源代码采用 [Apache-2.0](LICENSE) 许可证发布。数据集、模型、基准和可选 +依赖仍遵循各自的原始许可证，详情请参阅 [NOTICE](NOTICE)。 + +## 引用 + +如果 OraRL 对您有所帮助，欢迎为本仓库点亮 ⭐，并引用我们的 +[论文](https://arxiv.org/abs/2608.20492)。 + +```bibtex +@article{li2026orarl, + title = {Annotations as Rollouts: Efficient and Scalable + Reinforcement Learning for Video MLLMs}, + author = {Li, Yunheng and Mu, Guohong and Li, Hao and + Qian, Shengsheng and Zhang, Dingwen and Hou, Qibin + and Cheng, Ming-Ming}, + journal = {arXiv preprint arXiv:2608.20492}, + year = {2026}, + url = {https://arxiv.org/abs/2608.20492} +} +``` diff --git a/code/assets/orarl-data-scaling.gif b/code/assets/orarl-data-scaling.gif new file mode 100644 index 0000000000000000000000000000000000000000..5d35e3acf10b0b51348257040629e851dbb908dc --- /dev/null +++ b/code/assets/orarl-data-scaling.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f63cbb854c1c9f3867dce9300b44f01b9d6d27a018947478fb927ac9c932e57e +size 240092 diff --git a/code/assets/orarl-hero.gif b/code/assets/orarl-hero.gif new file mode 100644 index 0000000000000000000000000000000000000000..855733cec9ded23a97e76c63b2018f6937544a45 --- /dev/null +++ b/code/assets/orarl-hero.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9445d37d764a433e6743df9595d497819a994d27e4d074e875f23cae44e5ef8 +size 338665 diff --git a/code/assets/orarl-method.gif b/code/assets/orarl-method.gif new file mode 100644 index 0000000000000000000000000000000000000000..940052c9111303753fc078fa034870220f3f0f2b --- /dev/null +++ b/code/assets/orarl-method.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e43a40e4ea1d241e5ee230090ca39f84ce972587808b7ac11cef8ffb7c6ccd7 +size 2247059 diff --git a/code/assets/orarl-model-scaling.gif b/code/assets/orarl-model-scaling.gif new file mode 100644 index 0000000000000000000000000000000000000000..ed2e1758b24e49cb5bf597fc6d4483aad241007f --- /dev/null +++ b/code/assets/orarl-model-scaling.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5463916e2ec28a459bee1bed561867ad6de98b8a37ad9dee3944e43d43761595 +size 267523 diff --git a/code/assets/orarl-teaser.mp4 b/code/assets/orarl-teaser.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..708816d849b8a8b02760297b81d8b48a57945fcd --- /dev/null +++ b/code/assets/orarl-teaser.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7915acbb2207376ef20467792c3e5b2b86b7a7a292d7f10c1bf3f44817c28e0b +size 5901910 diff --git a/code/assets/paper-data-scaling.png b/code/assets/paper-data-scaling.png new file mode 100644 index 0000000000000000000000000000000000000000..86daf7bf8a20a98c8db069bcb206c4860fc76c20 --- /dev/null +++ b/code/assets/paper-data-scaling.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f9243c616b35fea4bc5aa25b86b6c7e501d1766ba10f3db493aa1fcdf08b4ec +size 144665 diff --git a/code/assets/paper-framework.png b/code/assets/paper-framework.png new file mode 100644 index 0000000000000000000000000000000000000000..b0f79ae9ee820add2296e8576f0de7050a2aa550 --- /dev/null +++ b/code/assets/paper-framework.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa39c05c95b1b9c6fac5615d4babe206c96c103d6c3d9c70d69bb3002a3fbe17 +size 186111 diff --git a/code/assets/paper-model-scaling.png b/code/assets/paper-model-scaling.png new file mode 100644 index 0000000000000000000000000000000000000000..606ee8d446a3bca10228b10f7b9ca4f0dd0a6afe --- /dev/null +++ b/code/assets/paper-model-scaling.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c79ab0337be5ee65a4e203e8060d7515b394660409b4fd256230c5a5b32a2dc5 +size 138189 diff --git a/code/assets/paper-results.png b/code/assets/paper-results.png new file mode 100644 index 0000000000000000000000000000000000000000..81db6a0878decdba6b49e1f8c519aab691897701 --- /dev/null +++ b/code/assets/paper-results.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fd05c8ff68f5a864d1d113eb5060c48e5c482e5d8ff387e0308b80a87752d0c +size 317400 diff --git a/code/assets/video_ora_benchmark_matrix_dark.svg b/code/assets/video_ora_benchmark_matrix_dark.svg new file mode 100644 index 0000000000000000000000000000000000000000..ddcff7738e89bb9c993317a8f07a2209282d7f3b --- /dev/null +++ b/code/assets/video_ora_benchmark_matrix_dark.svg @@ -0,0 +1,362 @@ + + +Benchmark comparison of Video-ORA across seven task families +Dataset-level matrix comparing Video-ORA with Qwen3.5, InternVideo3, VideoChat3, Qwen3-VL, and LLaVA-OneVision-2, including the latest ReVSI result. +Primary source: OraRL paper dated 2026-08-21, Tables 1–8 and 20. Dagger values: original LLaVA-OneVision-2, VideoChat3, OneThinker, and ReVSI reports. + +Table 1: +Benchmark comparison of Video-ORA. + +Best + +Second-best +† Original-report value + + +Task family +Benchmark +Video-ORA9B +Video-ORA4B +Qwen3.59B +InternVideo38B +VideoChat34B +Qwen3-VL8B +LLaVA-OV28B + +TEMPORAL +GROUNDING +Charades (mIoU) + +61.8 + +56.8 +50.6 +53.2 +56.1 +48.3 +52.6 + +ActivityNet (mIoU) + +63.6 + +57.3 +52.2 +46.3 +54.6 +46.8 +52.4 +QVHighlights (mIoU) + +72.5 + +67.5 +65.8 +59.2 +67.0 +59.4 +65.7 + + + +Average + +66.0 + +60.5 +56.2 +52.9 +59.2 +51.5 +56.9 + +SPATIAL +GROUNDING +RefCOCO (R@0.5, 3 splits) + +92.9 +91.4 + +91.6 +— +— +89.1† +— + +RefCOCO+ (R@0.5, 3 splits) + +88.9 +86.3 + +86.4 +— +— +84.0† +— +RefCOCOg (R@0.5, 2 splits) + +90.9 + +89.9 +88.6 +— +— +86.8† +— + + + +Average + +90.9 + +89.1 +88.9 +— +— +87.9† +— + +SEGMENTATION +RefCOCO (cIoU) + +79.4 + +76.7 +75.2 +— +— +73.8 +— + +RefCOCO+ (cIoU) + +72.6 + +68.7 +65.6 +— +— +65.1 +— +RefCOCOg (cIoU) + +75.1 + +73.0 +70.0 +— +— +70.1 +— + +MeViS (J&F) + +61.3 + +57.5 +32.1 +— +— +22.9 +— +ReasonVOS (J&F) + +63.7 + +63.8 +21.5 +— +— +19.6 +— + + + +Average + +70.4 + +67.9 +52.9 +— +— +50.3 +— + +VISUAL +TRACKING +GOT-10k (AO) + +78.2 + +67.5 +46.0 +— +— +33.7 +— + + +SPATIAL–TEMP. +GROUNDING +STVG (tIoU / sIoU avg.) + +35.0 + +32.4 +27.9 +— +— +19.5 +— + +VIDEO QA +VideoMME + +76.7 +69.8 +71.1 + +73.8 +70.1† +71.4† +71.9 + +VideoMME-v2 + +32.0 +26.1 + +29.9 +27.6 +28.7† +18.2† +19.9 +MV-Bench + +72.5 +67.2 +68.2 + +75.0 +— +69.0† +66.2 + +MMVU (MC) + +77.0 +65.4 + +72.5 +— +56.4† +65.6† +— +VideoHolmes + +65.5 + +61.9 +50.3 +— +— +40.9† +— + +LongVideoBench +66.2 +60.2 +64.6 +66.8 +— + +68.0† + +66.9 +MLVU-MC + +77.7 +73.2 +76.7 +77.3 +— + +78.1† +76.6 + + + +Average + +66.8 +60.5 + +61.9 +— +— +58.7† +— + +SPATIAL +INTELLIGENCE +VSI-Bench + +73.1 +66.2 +57.9 +— +— +57.9 + +70.9 + +ReVSI (Avg., native frames) + +58.2 +— + +51.6† +— +— +49.1† +— +MMSI-Bench + +37.9 + +31.9 +31.7 +— +— +31.1 +— + +MindCube-Tiny + +57.4 + +48.6 +41.2 +— +— +29.4 +— + + +Paper Avg. (3 datasets) + +56.1 + +48.9 +43.6 +— +— +39.5 +— + +† Original-report value; frame, prompt, split, or decoding settings may differ. +OraRL paper (Aug. 21, 2026), Tables 1–8 & 20. Paper Average excludes ReVSI. +↑ higher is better + diff --git a/code/assets/video_ora_benchmark_matrix_light.svg b/code/assets/video_ora_benchmark_matrix_light.svg new file mode 100644 index 0000000000000000000000000000000000000000..cc0e19d95333bdc5e1d655803a03a4d5e3ae254d --- /dev/null +++ b/code/assets/video_ora_benchmark_matrix_light.svg @@ -0,0 +1,362 @@ + + +Benchmark comparison of Video-ORA across seven task families +Dataset-level matrix comparing Video-ORA with Qwen3.5, InternVideo3, VideoChat3, Qwen3-VL, and LLaVA-OneVision-2, including the latest ReVSI result. +Primary source: OraRL paper dated 2026-08-21, Tables 1–8 and 20. Dagger values: original LLaVA-OneVision-2, VideoChat3, OneThinker, and ReVSI reports. + +Table 1: +Benchmark comparison of Video-ORA. + +Best + +Second-best +† Original-report value + + +Task family +Benchmark +Video-ORA9B +Video-ORA4B +Qwen3.59B +InternVideo38B +VideoChat34B +Qwen3-VL8B +LLaVA-OV28B + +TEMPORAL +GROUNDING +Charades (mIoU) + +61.8 + +56.8 +50.6 +53.2 +56.1 +48.3 +52.6 + +ActivityNet (mIoU) + +63.6 + +57.3 +52.2 +46.3 +54.6 +46.8 +52.4 +QVHighlights (mIoU) + +72.5 + +67.5 +65.8 +59.2 +67.0 +59.4 +65.7 + + + +Average + +66.0 + +60.5 +56.2 +52.9 +59.2 +51.5 +56.9 + +SPATIAL +GROUNDING +RefCOCO (R@0.5, 3 splits) + +92.9 +91.4 + +91.6 +— +— +89.1† +— + +RefCOCO+ (R@0.5, 3 splits) + +88.9 +86.3 + +86.4 +— +— +84.0† +— +RefCOCOg (R@0.5, 2 splits) + +90.9 + +89.9 +88.6 +— +— +86.8† +— + + + +Average + +90.9 + +89.1 +88.9 +— +— +87.9† +— + +SEGMENTATION +RefCOCO (cIoU) + +79.4 + +76.7 +75.2 +— +— +73.8 +— + +RefCOCO+ (cIoU) + +72.6 + +68.7 +65.6 +— +— +65.1 +— +RefCOCOg (cIoU) + +75.1 + +73.0 +70.0 +— +— +70.1 +— + +MeViS (J&F) + +61.3 + +57.5 +32.1 +— +— +22.9 +— +ReasonVOS (J&F) + +63.7 + +63.8 +21.5 +— +— +19.6 +— + + + +Average + +70.4 + +67.9 +52.9 +— +— +50.3 +— + +VISUAL +TRACKING +GOT-10k (AO) + +78.2 + +67.5 +46.0 +— +— +33.7 +— + + +SPATIAL–TEMP. +GROUNDING +STVG (tIoU / sIoU avg.) + +35.0 + +32.4 +27.9 +— +— +19.5 +— + +VIDEO QA +VideoMME + +76.7 +69.8 +71.1 + +73.8 +70.1† +71.4† +71.9 + +VideoMME-v2 + +32.0 +26.1 + +29.9 +27.6 +28.7† +18.2† +19.9 +MV-Bench + +72.5 +67.2 +68.2 + +75.0 +— +69.0† +66.2 + +MMVU (MC) + +77.0 +65.4 + +72.5 +— +56.4† +65.6† +— +VideoHolmes + +65.5 + +61.9 +50.3 +— +— +40.9† +— + +LongVideoBench +66.2 +60.2 +64.6 +66.8 +— + +68.0† + +66.9 +MLVU-MC + +77.7 +73.2 +76.7 +77.3 +— + +78.1† +76.6 + + + +Average + +66.8 +60.5 + +61.9 +— +— +58.7† +— + +SPATIAL +INTELLIGENCE +VSI-Bench + +73.1 +66.2 +57.9 +— +— +57.9 + +70.9 + +ReVSI (Avg., native frames) + +58.2 +— + +51.6† +— +— +49.1† +— +MMSI-Bench + +37.9 + +31.9 +31.7 +— +— +31.1 +— + +MindCube-Tiny + +57.4 + +48.6 +41.2 +— +— +29.4 +— + + +Paper Avg. (3 datasets) + +56.1 + +48.9 +43.6 +— +— +39.5 +— + +† Original-report value; frame, prompt, split, or decoding settings may differ. +OraRL paper (Aug. 21, 2026), Tables 1–8 & 20. Paper Average excludes ReVSI. +↑ higher is better + diff --git a/code/configs/data_sources.example.yaml b/code/configs/data_sources.example.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c04f47416a055e7d6e996f81635f59d2a3e4e8cd --- /dev/null +++ b/code/configs/data_sources.example.yaml @@ -0,0 +1,90 @@ +# Copy this file, then replace every ../local_data placeholder. +# +# OraRL never downloads data or media. You must obtain each dataset under its +# license, keep the required attribution, and point input/media_root at your +# own local, licensed copy. The optional license and url values are audit +# metadata only and are never opened by the builder. +# +# The seven quotas reproduce the final paper mixture: 100,032 train prompts +# (1,563 batches of 64), with 62,656 structured prompts and 37,376 answer-only +# prompts. The canary is +# additional held-out data and is not subtracted from the train target. +version: 1 +seed: 42 +target: 100032 +canary_size: 512 +max_prompts_per_media: 2 +require_media: true +allow_shortfall: false + +# Supply JSON/JSONL benchmark records before a release build. Any candidate +# sharing a normalized prompt identity or media identity is removed. +benchmark_excludes: + - ../local_data/exclusions/public_benchmarks.jsonl + +sources: + - name: temporal_grounding_train + input: ../local_data/annotations/temporal_grounding.jsonl + task: temporal grounding + family: temporal + quota: 20096 + media_root: ../local_data/media/temporal + license: REPLACE_WITH_DATASET_LICENSE + url: REPLACE_WITH_DATASET_HOME_PAGE + + - name: tracking_train + input: ../local_data/annotations/tracking.jsonl + task: tracking + family: tracking + quota: 13952 + media_root: ../local_data/media/tracking + license: REPLACE_WITH_DATASET_LICENSE + url: REPLACE_WITH_DATASET_HOME_PAGE + + - name: segmentation_train + input: ../local_data/annotations/segmentation.jsonl + task: segmentation + family: segmentation + quota: 12032 + media_root: ../local_data/media/segmentation + license: REPLACE_WITH_DATASET_LICENSE + url: REPLACE_WITH_DATASET_HOME_PAGE + + - name: spatial_grounding_train + input: ../local_data/annotations/spatial_grounding.jsonl + task: spatial grounding + family: spatial + quota: 7040 + media_root: ../local_data/media/spatial + license: REPLACE_WITH_DATASET_LICENSE + url: REPLACE_WITH_DATASET_HOME_PAGE + + - name: spatial_temporal_grounding_train + input: ../local_data/annotations/spatial_temporal_grounding.jsonl + task: spatial-temporal grounding + family: stvg + quota: 9536 + media_root: ../local_data/media/spatial_temporal + license: REPLACE_WITH_DATASET_LICENSE + url: REPLACE_WITH_DATASET_HOME_PAGE + + - name: video_qa_train + input: ../local_data/annotations/video_qa.jsonl + task: video_qa_mc + family: video_qa + quota: 20288 + media_root: ../local_data/media/video_qa + license: REPLACE_WITH_DATASET_LICENSE + url: REPLACE_WITH_DATASET_HOME_PAGE + + - name: spatial_intelligence_train + input: ../local_data/annotations/spatial_intelligence.jsonl + task: spatial intelligence + family: spatial_intelligence + # Keep subtype labels such as object_counting and object_rel_direction; + # the reward adapter uses them to select the official scoring rule. + preserve_problem_type: true + quota: 17088 + media_root: ../local_data/media/spatial_intelligence + license: REPLACE_WITH_DATASET_LICENSE + url: REPLACE_WITH_DATASET_HOME_PAGE diff --git a/code/configs/grpo_4b.yaml b/code/configs/grpo_4b.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a7c8b8190758cefddf156b95fa5dd6bcdf8e58de --- /dev/null +++ b/code/configs/grpo_4b.yaml @@ -0,0 +1,139 @@ +# Complete 4B GRPO baseline. Required paths are supplied by the CLI or environment. +data: + train_files: ${oc.env:ORARL_TRAIN_DATA} + val_files: ${oc.env:ORARL_VAL_DATA} + prompt_key: problem + answer_key: answer + image_key: images + video_key: videos + image_dir: ${oc.env:ORARL_MEDIA_ROOT,null} + dataloader_num_workers: ${oc.env:ORARL_DATALOADER_WORKERS,4} + video_max_pixels: ${oc.env:ORARL_VIDEO_MAX_PIXELS,786432} + video_min_pixels: ${oc.env:ORARL_VIDEO_MIN_PIXELS,4096} + video_max_frames: ${oc.env:ORARL_VIDEO_MAX_FRAMES,128} + video_fps: ${oc.env:ORARL_VIDEO_FPS,2} + video_total_pixels: ${oc.env:ORARL_VIDEO_TOTAL_PIXELS,8388608} + min_pixels: ${oc.env:ORARL_IMAGE_MIN_PIXELS,4096} + image_max_pixels: ${oc.env:ORARL_IMAGE_MAX_PIXELS,1048576} + use_preprocessed_videos: false + video_source_mode: ${oc.env:ORARL_VIDEO_SOURCE_MODE,realtime_only} + preprocessed_video_dir: ${oc.env:ORARL_PREPROCESSED_DIR,null} + inline_video_tensors: ${oc.env:ORARL_INLINE_VIDEO_TENSORS,false} + max_prompt_length: ${oc.env:ORARL_MAX_PROMPT_LENGTH,16384} + max_response_length: ${oc.env:ORARL_MAX_RESPONSE_LENGTH,4096} + rollout_batch_size: ${oc.env:ORARL_ROLLOUT_BATCH_SIZE,64} + mini_rollout_batch_size: null + val_batch_size: ${oc.env:ORARL_VAL_BATCH_SIZE,16} + format_prompt: null + override_chat_template: null + enable_thinking: ${oc.env:ORARL_ENABLE_THINKING,false} + response_prefix: "" + shuffle: true + seed: ${oc.env:ORARL_SEED,42} + group_by_task: true + group_by_task_key: problem_type + filter_overlong_prompts: false + +algorithm: + name: grpo + adv_estimator: grpo + scale_rewards: true + disable_kl: true + use_kl_loss: false + kl_penalty: low_var_kl + kl_coef: 0.0 + +worker: + actor: + global_batch_size: ${oc.env:ORARL_GLOBAL_BATCH_SIZE,64} + micro_batch_size_per_device_for_update: ${oc.env:ORARL_UPDATE_MICRO_BATCH,2} + micro_batch_size_per_device_for_experience: ${oc.env:ORARL_EXPERIENCE_MICRO_BATCH,1} + ppo_epochs: 1 + max_grad_norm: 1.0 + clip_ratio_low: 0.2 + clip_ratio_high: 0.2 + loss_avg_mode: token + padding_free: true + dynamic_batching: true + max_token_len_per_gpu: ${oc.env:ORARL_MAX_TOKEN_LEN_PER_GPU,24576} + ulysses_size: 1 + use_torch_compile: false + model: + model_path: ${oc.env:ORARL_MODEL_PATH} + tokenizer_path: ${oc.env:ORARL_TOKENIZER_PATH,${oc.env:ORARL_MODEL_PATH}} + enable_gradient_checkpointing: true + trust_remote_code: false + freeze_vision_tower: true + train_vision_merger: false + optim: + lr: ${oc.env:ORARL_LEARNING_RATE,2.0e-6} + weight_decay: 0.0 + strategy: adamw + lr_warmup_ratio: 0.0 + lr_scheduler_type: constant + fsdp: + torch_dtype: bf16 + enable_full_shard: true + enable_cpu_offload: false + enable_rank0_init: true + use_orig_params: false + offload: + offload_params: false + offload_optimizer: false + + rollout: + name: ${oc.env:ORARL_ROLLOUT_BACKEND,vllm} + n: 8 + seed: ${oc.env:ORARL_SEED,42} + temperature: 1.0 + top_p: 0.85 + limit_images: 10 + gpu_memory_utilization: 0.5 + enforce_eager: false + enable_chunked_prefill: true + tensor_parallel_size: 1 + max_num_batched_tokens: 32768 + disable_tqdm: true + calculate_log_probs: false + collect_seq_logprob_for_filter: false + val_override_config: + temperature: 0.7 + top_p: 0.95 + n: 1 + + ref: + fsdp: + torch_dtype: bf16 + enable_full_shard: true + enable_cpu_offload: false + enable_rank0_init: true + offload: + offload_params: false + + reward: + reward_function: orarl.rewards:compute_score + reward_function_kwargs: {} + +trainer: + total_epochs: 1 + max_steps: null + project_name: OraRL + experiment_name: grpo-4b + logger: ["console"] + nnodes: 1 + n_gpus_per_node: 8 + val_freq: -1 + val_before_train: false + val_only: false + val_generations_to_log: 0 + save_freq: 50 + save_limit: 3 + save_model_only: false + keep_optim_only_latest: true + save_checkpoint_path: ${oc.env:ORARL_OUTPUT_DIR} + load_checkpoint_path: null + find_last_checkpoint: false + keep_best_train_ckpt: false + best_train_metric_key: reward/overall + best_train_smooth_window: 5 + best_train_min_step: 10 diff --git a/code/configs/grpo_9b.yaml b/code/configs/grpo_9b.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a6cc6be5e47836a2658a8a15d8f03112a0654446 --- /dev/null +++ b/code/configs/grpo_9b.yaml @@ -0,0 +1,139 @@ +# Complete 9B GRPO baseline. Required paths are supplied by the CLI or environment. +data: + train_files: ${oc.env:ORARL_TRAIN_DATA} + val_files: ${oc.env:ORARL_VAL_DATA} + prompt_key: problem + answer_key: answer + image_key: images + video_key: videos + image_dir: ${oc.env:ORARL_MEDIA_ROOT,null} + dataloader_num_workers: ${oc.env:ORARL_DATALOADER_WORKERS,4} + video_max_pixels: ${oc.env:ORARL_VIDEO_MAX_PIXELS,786432} + video_min_pixels: ${oc.env:ORARL_VIDEO_MIN_PIXELS,4096} + video_max_frames: ${oc.env:ORARL_VIDEO_MAX_FRAMES,128} + video_fps: ${oc.env:ORARL_VIDEO_FPS,2} + video_total_pixels: ${oc.env:ORARL_VIDEO_TOTAL_PIXELS,8388608} + min_pixels: ${oc.env:ORARL_IMAGE_MIN_PIXELS,4096} + image_max_pixels: ${oc.env:ORARL_IMAGE_MAX_PIXELS,1048576} + use_preprocessed_videos: false + video_source_mode: ${oc.env:ORARL_VIDEO_SOURCE_MODE,realtime_only} + preprocessed_video_dir: ${oc.env:ORARL_PREPROCESSED_DIR,null} + inline_video_tensors: ${oc.env:ORARL_INLINE_VIDEO_TENSORS,false} + max_prompt_length: ${oc.env:ORARL_MAX_PROMPT_LENGTH,24576} + max_response_length: ${oc.env:ORARL_MAX_RESPONSE_LENGTH,4096} + rollout_batch_size: ${oc.env:ORARL_ROLLOUT_BATCH_SIZE,64} + mini_rollout_batch_size: null + val_batch_size: ${oc.env:ORARL_VAL_BATCH_SIZE,16} + format_prompt: null + override_chat_template: null + enable_thinking: ${oc.env:ORARL_ENABLE_THINKING,false} + response_prefix: "" + shuffle: true + seed: ${oc.env:ORARL_SEED,42} + group_by_task: true + group_by_task_key: problem_type + filter_overlong_prompts: false + +algorithm: + name: grpo + adv_estimator: grpo + scale_rewards: true + disable_kl: true + use_kl_loss: false + kl_penalty: low_var_kl + kl_coef: 0.0 + +worker: + actor: + global_batch_size: ${oc.env:ORARL_GLOBAL_BATCH_SIZE,64} + micro_batch_size_per_device_for_update: ${oc.env:ORARL_UPDATE_MICRO_BATCH,1} + micro_batch_size_per_device_for_experience: ${oc.env:ORARL_EXPERIENCE_MICRO_BATCH,1} + ppo_epochs: 1 + max_grad_norm: 1.0 + clip_ratio_low: 0.2 + clip_ratio_high: 0.2 + loss_avg_mode: token + padding_free: true + dynamic_batching: true + max_token_len_per_gpu: ${oc.env:ORARL_MAX_TOKEN_LEN_PER_GPU,32768} + ulysses_size: 1 + use_torch_compile: false + model: + model_path: ${oc.env:ORARL_MODEL_PATH} + tokenizer_path: ${oc.env:ORARL_TOKENIZER_PATH,${oc.env:ORARL_MODEL_PATH}} + enable_gradient_checkpointing: true + trust_remote_code: false + freeze_vision_tower: true + train_vision_merger: false + optim: + lr: ${oc.env:ORARL_LEARNING_RATE,1.0e-6} + weight_decay: 0.0 + strategy: adamw + lr_warmup_ratio: 0.0 + lr_scheduler_type: constant + fsdp: + torch_dtype: bf16 + enable_full_shard: true + enable_cpu_offload: false + enable_rank0_init: true + use_orig_params: false + offload: + offload_params: false + offload_optimizer: false + + rollout: + name: ${oc.env:ORARL_ROLLOUT_BACKEND,vllm} + n: 8 + seed: ${oc.env:ORARL_SEED,42} + temperature: 1.0 + top_p: 0.85 + limit_images: 10 + gpu_memory_utilization: 0.45 + enforce_eager: false + enable_chunked_prefill: true + tensor_parallel_size: 1 + max_num_batched_tokens: 32768 + disable_tqdm: true + calculate_log_probs: false + collect_seq_logprob_for_filter: false + val_override_config: + temperature: 0.7 + top_p: 0.95 + n: 1 + + ref: + fsdp: + torch_dtype: bf16 + enable_full_shard: true + enable_cpu_offload: false + enable_rank0_init: true + offload: + offload_params: false + + reward: + reward_function: orarl.rewards:compute_score + reward_function_kwargs: {} + +trainer: + total_epochs: 1 + max_steps: null + project_name: OraRL + experiment_name: grpo-9b + logger: ["console"] + nnodes: 1 + n_gpus_per_node: 8 + val_freq: -1 + val_before_train: false + val_only: false + val_generations_to_log: 0 + save_freq: 50 + save_limit: 3 + save_model_only: false + keep_optim_only_latest: true + save_checkpoint_path: ${oc.env:ORARL_OUTPUT_DIR} + load_checkpoint_path: null + find_last_checkpoint: false + keep_best_train_ckpt: false + best_train_metric_key: reward/overall + best_train_smooth_window: 5 + best_train_min_step: 10 diff --git a/code/configs/orarl_4b.yaml b/code/configs/orarl_4b.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c11299402ec34940d976dbaef352327a7c5ca856 --- /dev/null +++ b/code/configs/orarl_4b.yaml @@ -0,0 +1,162 @@ +# Complete 4B OraRL recipe. Required paths are supplied by the CLI or environment. +data: + train_files: ${oc.env:ORARL_TRAIN_DATA} + val_files: ${oc.env:ORARL_VAL_DATA} + prompt_key: problem + answer_key: answer + image_key: images + video_key: videos + image_dir: ${oc.env:ORARL_MEDIA_ROOT,null} + dataloader_num_workers: ${oc.env:ORARL_DATALOADER_WORKERS,4} + video_max_pixels: ${oc.env:ORARL_VIDEO_MAX_PIXELS,786432} + video_min_pixels: ${oc.env:ORARL_VIDEO_MIN_PIXELS,4096} + video_max_frames: ${oc.env:ORARL_VIDEO_MAX_FRAMES,128} + video_fps: ${oc.env:ORARL_VIDEO_FPS,2} + video_total_pixels: ${oc.env:ORARL_VIDEO_TOTAL_PIXELS,8388608} + min_pixels: ${oc.env:ORARL_IMAGE_MIN_PIXELS,4096} + image_max_pixels: ${oc.env:ORARL_IMAGE_MAX_PIXELS,1048576} + use_preprocessed_videos: false + video_source_mode: ${oc.env:ORARL_VIDEO_SOURCE_MODE,realtime_only} + preprocessed_video_dir: ${oc.env:ORARL_PREPROCESSED_DIR,null} + inline_video_tensors: ${oc.env:ORARL_INLINE_VIDEO_TENSORS,false} + max_prompt_length: ${oc.env:ORARL_MAX_PROMPT_LENGTH,16384} + max_response_length: ${oc.env:ORARL_MAX_RESPONSE_LENGTH,4096} + rollout_batch_size: ${oc.env:ORARL_ROLLOUT_BATCH_SIZE,64} + mini_rollout_batch_size: null + val_batch_size: ${oc.env:ORARL_VAL_BATCH_SIZE,16} + format_prompt: null + override_chat_template: null + enable_thinking: ${oc.env:ORARL_ENABLE_THINKING,false} + response_prefix: "" + shuffle: true + seed: ${oc.env:ORARL_SEED,42} + group_by_task: true + group_by_task_key: problem_type + filter_overlong_prompts: false + +algorithm: + name: orarl + adv_estimator: grpo + scale_rewards: false + oracle_injection: true + oracle_injection_mode: append + oracle_builder: orarl.rewards:build_oracle_response_from_ground_truth + oracle_log_exclude: true + directional_gain: true + directional_gain_gamma: 0.25 + directional_gain_positive_only: true + directional_gain_recenter: true + detached_oracle_advantage: true + detached_oracle_advantage_scale: 2.0 + detached_oracle_use_directional_gain: false + detached_oracle_match_best_ratio: 1.2 + detached_oracle_match_best_min: 0.05 + detached_oracle_match_best_max: 1.0 + oracle_reward_gate_beta: 2.0 + selection_prune_ratio: 0.5 + selection_keep_oracle: true + selection_positive_quota: 1 + selection_negative_quota: 2 + selection_strict_sign_balance: true + post_selection_recenter: true + post_selection_rms_match: true + post_selection_rms_min_scale: 0.25 + disable_kl: true + use_kl_loss: false + kl_penalty: low_var_kl + kl_coef: 0.0 + +worker: + actor: + global_batch_size: ${oc.env:ORARL_GLOBAL_BATCH_SIZE,64} + micro_batch_size_per_device_for_update: ${oc.env:ORARL_UPDATE_MICRO_BATCH,2} + micro_batch_size_per_device_for_experience: ${oc.env:ORARL_EXPERIENCE_MICRO_BATCH,1} + ppo_epochs: 1 + max_grad_norm: 1.0 + clip_ratio_low: 0.2 + clip_ratio_high: 0.2 + loss_avg_mode: token + padding_free: true + dynamic_batching: true + max_token_len_per_gpu: ${oc.env:ORARL_MAX_TOKEN_LEN_PER_GPU,24576} + ulysses_size: 1 + use_torch_compile: false + model: + model_path: ${oc.env:ORARL_MODEL_PATH} + tokenizer_path: ${oc.env:ORARL_TOKENIZER_PATH,${oc.env:ORARL_MODEL_PATH}} + enable_gradient_checkpointing: true + trust_remote_code: false + freeze_vision_tower: true + train_vision_merger: false + optim: + lr: ${oc.env:ORARL_LEARNING_RATE,2.0e-6} + weight_decay: 0.0 + strategy: adamw + lr_warmup_ratio: 0.0 + lr_scheduler_type: constant + fsdp: + torch_dtype: bf16 + enable_full_shard: true + enable_cpu_offload: false + enable_rank0_init: true + use_orig_params: false + offload: + offload_params: false + offload_optimizer: false + + rollout: + name: ${oc.env:ORARL_ROLLOUT_BACKEND,vllm} + n: 8 + seed: ${oc.env:ORARL_SEED,42} + temperature: 1.0 + top_p: 0.85 + limit_images: 10 + gpu_memory_utilization: 0.5 + enforce_eager: false + enable_chunked_prefill: true + tensor_parallel_size: 1 + max_num_batched_tokens: 32768 + disable_tqdm: true + calculate_log_probs: false + collect_seq_logprob_for_filter: false + val_override_config: + temperature: 0.7 + top_p: 0.95 + n: 1 + + ref: + fsdp: + torch_dtype: bf16 + enable_full_shard: true + enable_cpu_offload: false + enable_rank0_init: true + offload: + offload_params: false + + reward: + reward_function: orarl.rewards:compute_score + reward_function_kwargs: {} + +trainer: + total_epochs: 1 + max_steps: null + project_name: OraRL + experiment_name: orarl-4b + logger: ["console"] + nnodes: 1 + n_gpus_per_node: 8 + val_freq: -1 + val_before_train: false + val_only: false + val_generations_to_log: 0 + save_freq: 50 + save_limit: 3 + save_model_only: false + keep_optim_only_latest: true + save_checkpoint_path: ${oc.env:ORARL_OUTPUT_DIR} + load_checkpoint_path: null + find_last_checkpoint: false + keep_best_train_ckpt: false + best_train_metric_key: reward/overall + best_train_smooth_window: 5 + best_train_min_step: 10 diff --git a/code/configs/orarl_9b.yaml b/code/configs/orarl_9b.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4da8dcba7ce449c5880daf80c96dac4a8043379f --- /dev/null +++ b/code/configs/orarl_9b.yaml @@ -0,0 +1,162 @@ +# Complete 9B OraRL recipe. Required paths are supplied by the CLI or environment. +data: + train_files: ${oc.env:ORARL_TRAIN_DATA} + val_files: ${oc.env:ORARL_VAL_DATA} + prompt_key: problem + answer_key: answer + image_key: images + video_key: videos + image_dir: ${oc.env:ORARL_MEDIA_ROOT,null} + dataloader_num_workers: ${oc.env:ORARL_DATALOADER_WORKERS,4} + video_max_pixels: ${oc.env:ORARL_VIDEO_MAX_PIXELS,786432} + video_min_pixels: ${oc.env:ORARL_VIDEO_MIN_PIXELS,4096} + video_max_frames: ${oc.env:ORARL_VIDEO_MAX_FRAMES,128} + video_fps: ${oc.env:ORARL_VIDEO_FPS,2} + video_total_pixels: ${oc.env:ORARL_VIDEO_TOTAL_PIXELS,8388608} + min_pixels: ${oc.env:ORARL_IMAGE_MIN_PIXELS,4096} + image_max_pixels: ${oc.env:ORARL_IMAGE_MAX_PIXELS,1048576} + use_preprocessed_videos: false + video_source_mode: ${oc.env:ORARL_VIDEO_SOURCE_MODE,realtime_only} + preprocessed_video_dir: ${oc.env:ORARL_PREPROCESSED_DIR,null} + inline_video_tensors: ${oc.env:ORARL_INLINE_VIDEO_TENSORS,false} + max_prompt_length: ${oc.env:ORARL_MAX_PROMPT_LENGTH,24576} + max_response_length: ${oc.env:ORARL_MAX_RESPONSE_LENGTH,4096} + rollout_batch_size: ${oc.env:ORARL_ROLLOUT_BATCH_SIZE,64} + mini_rollout_batch_size: null + val_batch_size: ${oc.env:ORARL_VAL_BATCH_SIZE,16} + format_prompt: null + override_chat_template: null + enable_thinking: ${oc.env:ORARL_ENABLE_THINKING,false} + response_prefix: "" + shuffle: true + seed: ${oc.env:ORARL_SEED,42} + group_by_task: true + group_by_task_key: problem_type + filter_overlong_prompts: false + +algorithm: + name: orarl + adv_estimator: grpo + scale_rewards: false + oracle_injection: true + oracle_injection_mode: append + oracle_builder: orarl.rewards:build_oracle_response_from_ground_truth + oracle_log_exclude: true + directional_gain: true + directional_gain_gamma: 0.25 + directional_gain_positive_only: true + directional_gain_recenter: true + detached_oracle_advantage: true + detached_oracle_advantage_scale: 2.0 + detached_oracle_use_directional_gain: false + detached_oracle_match_best_ratio: 1.2 + detached_oracle_match_best_min: 0.05 + detached_oracle_match_best_max: 1.0 + oracle_reward_gate_beta: 2.0 + selection_prune_ratio: 0.5 + selection_keep_oracle: true + selection_positive_quota: 1 + selection_negative_quota: 2 + selection_strict_sign_balance: true + post_selection_recenter: true + post_selection_rms_match: true + post_selection_rms_min_scale: 0.25 + disable_kl: true + use_kl_loss: false + kl_penalty: low_var_kl + kl_coef: 0.0 + +worker: + actor: + global_batch_size: ${oc.env:ORARL_GLOBAL_BATCH_SIZE,64} + micro_batch_size_per_device_for_update: ${oc.env:ORARL_UPDATE_MICRO_BATCH,1} + micro_batch_size_per_device_for_experience: ${oc.env:ORARL_EXPERIENCE_MICRO_BATCH,1} + ppo_epochs: 1 + max_grad_norm: 1.0 + clip_ratio_low: 0.2 + clip_ratio_high: 0.2 + loss_avg_mode: token + padding_free: true + dynamic_batching: true + max_token_len_per_gpu: ${oc.env:ORARL_MAX_TOKEN_LEN_PER_GPU,32768} + ulysses_size: 1 + use_torch_compile: false + model: + model_path: ${oc.env:ORARL_MODEL_PATH} + tokenizer_path: ${oc.env:ORARL_TOKENIZER_PATH,${oc.env:ORARL_MODEL_PATH}} + enable_gradient_checkpointing: true + trust_remote_code: false + freeze_vision_tower: true + train_vision_merger: false + optim: + lr: ${oc.env:ORARL_LEARNING_RATE,1.0e-6} + weight_decay: 0.0 + strategy: adamw + lr_warmup_ratio: 0.0 + lr_scheduler_type: constant + fsdp: + torch_dtype: bf16 + enable_full_shard: true + enable_cpu_offload: false + enable_rank0_init: true + use_orig_params: false + offload: + offload_params: false + offload_optimizer: false + + rollout: + name: ${oc.env:ORARL_ROLLOUT_BACKEND,vllm} + n: 8 + seed: ${oc.env:ORARL_SEED,42} + temperature: 1.0 + top_p: 0.85 + limit_images: 10 + gpu_memory_utilization: 0.45 + enforce_eager: false + enable_chunked_prefill: true + tensor_parallel_size: 1 + max_num_batched_tokens: 32768 + disable_tqdm: true + calculate_log_probs: false + collect_seq_logprob_for_filter: false + val_override_config: + temperature: 0.7 + top_p: 0.95 + n: 1 + + ref: + fsdp: + torch_dtype: bf16 + enable_full_shard: true + enable_cpu_offload: false + enable_rank0_init: true + offload: + offload_params: false + + reward: + reward_function: orarl.rewards:compute_score + reward_function_kwargs: {} + +trainer: + total_epochs: 1 + max_steps: null + project_name: OraRL + experiment_name: orarl-9b + logger: ["console"] + nnodes: 1 + n_gpus_per_node: 8 + val_freq: -1 + val_before_train: false + val_only: false + val_generations_to_log: 0 + save_freq: 50 + save_limit: 3 + save_model_only: false + keep_optim_only_latest: true + save_checkpoint_path: ${oc.env:ORARL_OUTPUT_DIR} + load_checkpoint_path: null + find_last_checkpoint: false + keep_best_train_ckpt: false + best_train_metric_key: reward/overall + best_train_smooth_window: 5 + best_train_min_step: 10 diff --git a/code/data/eval/README.md b/code/data/eval/README.md new file mode 100644 index 0000000000000000000000000000000000000000..102e06d9fa8b4fa724cc00da1ac31f84f93b4ab0 --- /dev/null +++ b/code/data/eval/README.md @@ -0,0 +1,25 @@ +# Evaluation profiles + +This directory contains only the small, reviewable `datasets.jsonl` manifest +that pins the paper-suite splits, prompts, preprocessing settings, parsers, and +metrics. + +Evaluation annotations and raw media are hosted separately at +[`OraRL/OraRL-Data`](https://huggingface.co/datasets/OraRL/OraRL-Data). +Model checkpoints are hosted under the +[`OraRL`](https://huggingface.co/OraRL) organization. They are intentionally not +duplicated in this Git repository. + +Use the Hugging Face dataset ID or a complete local snapshot when evaluating: + +```bash +orarl-eval \ + --model /path/to/Video-ORA-9B \ + --tasks paper \ + --dataset OraRL/OraRL-Data \ + --summary ./evaluation.json \ + --run +``` + +The manifest in this directory is release metadata, not a standalone dataset +root; it does not include `assets.jsonl` or media payloads. diff --git a/code/data/eval/datasets.jsonl b/code/data/eval/datasets.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..a286784ba3ae7b0d304f7b00b2316c265526fb7a --- /dev/null +++ b/code/data/eval/datasets.jsonl @@ -0,0 +1,29 @@ +{"annotation_path":"annotations/video_qa/longvideobench/val.jsonl","artifact_paths":[],"benchmark":"longvideobench","evaluation":{"metric_profile":"accuracy","parser_profile":"multiple_choice","prompt_profile":"longvideobench_default"},"expected_count":1337,"family":"video_qa","legacy_environment":{"LONGVIDEOBENCH_BATCH_SIZE":1,"LONGVIDEOBENCH_ENABLE_THINKING":false,"LONGVIDEOBENCH_FPS":2,"LONGVIDEOBENCH_GPU_MEMORY_UTILIZATION":0.9,"LONGVIDEOBENCH_MAX_FRAMES":384,"LONGVIDEOBENCH_MAX_MODEL_LEN":65536,"LONGVIDEOBENCH_MAX_NEW_TOKENS":128,"LONGVIDEOBENCH_MAX_NUM_BATCHED_TOKENS":65536,"LONGVIDEOBENCH_MIN_P":0.0,"LONGVIDEOBENCH_PRESENCE_PENALTY":0.0,"LONGVIDEOBENCH_PROMPT_MODE":"default","LONGVIDEOBENCH_TEMPERATURE":0.0,"LONGVIDEOBENCH_TOP_K":-1,"LONGVIDEOBENCH_TOP_P":1.0,"LONGVIDEOBENCH_VIDEO_MAX_PIXELS":262144,"LONGVIDEOBENCH_VIDEO_MIN_PIXELS":4096,"LONGVIDEOBENCH_VIDEO_TOTAL_PIXELS":0},"license":"See the upstream benchmark terms","media_paths":["media/video_qa/longvideobench/subtitles","media/video_qa/longvideobench/subtitles/val","media/video_qa/longvideobench/videos","media/video_qa/longvideobench/videos/val"],"preprocessing":{"batch_size":1,"enable_thinking":false,"fps":2,"gpu_memory_utilization":0.9,"max_frames":384,"max_model_len":65536,"max_new_tokens":128,"max_num_batched_tokens":65536,"min_p":0.0,"presence_penalty":0.0,"prompt_mode":"default","temperature":0.0,"top_k":-1,"top_p":1.0,"video_max_pixels":262144,"video_min_pixels":4096,"video_total_pixels":0},"redistribution_authorized":true,"schema_version":1,"source_url":"https://github.com/longvideobench/LongVideoBench","split":"val","task":"longvideobench"} +{"annotation_path":"annotations/spatial_intelligence/mindcube/test.jsonl","artifact_paths":[],"benchmark":"mindcube","evaluation":{"aggregation":"micro","expected_group_counts":{"among":600,"around":250,"rotation":200},"metric_profile":"micro_accuracy","parser_profile":"multiple_choice","prompt_profile":"mindcube_official"},"expected_count":1050,"family":"spatial_intelligence","legacy_environment":{"MINDCUBE_BATCH_SIZE":16,"MINDCUBE_MAX_MODEL_LEN":32768,"MINDCUBE_MAX_NEW_TOKENS":1024},"license":"See the upstream benchmark terms","media_paths":["media/spatial_intelligence/mindcube/images","media/spatial_intelligence/mindcube/images/test"],"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/oscarqjh/MindCube_lmmseval/tree/7dd2725d9bd4149f2aad00a9843f72a3824da003","split":"test","task":"mindcube"} +{"annotation_path":"annotations/video_qa/mlvu/dev.jsonl","artifact_paths":[],"benchmark":"mlvu","evaluation":{"metric_profile":"macro_accuracy","parser_profile":"multiple_choice","prompt_profile":"mlvu_default"},"expected_count":2174,"family":"video_qa","legacy_environment":{"MLVU_BATCH_SIZE":1,"MLVU_ENABLE_THINKING":false,"MLVU_FPS":2,"MLVU_GPU_MEMORY_UTILIZATION":0.9,"MLVU_MAX_FRAMES":384,"MLVU_MAX_MODEL_LEN":65536,"MLVU_MAX_NEW_TOKENS":128,"MLVU_MAX_NUM_BATCHED_TOKENS":65536,"MLVU_MIN_P":0.0,"MLVU_PRESENCE_PENALTY":0.0,"MLVU_PROMPT_MODE":"default","MLVU_TEMPERATURE":0.0,"MLVU_TOP_K":-1,"MLVU_TOP_P":1.0,"MLVU_VIDEO_MAX_PIXELS":262144,"MLVU_VIDEO_MIN_PIXELS":4096,"MLVU_VIDEO_TOTAL_PIXELS":0},"license":"See the upstream benchmark terms","media_paths":["media/video_qa/mlvu/videos","media/video_qa/mlvu/videos/dev"],"preprocessing":{"batch_size":1,"enable_thinking":false,"fps":2,"gpu_memory_utilization":0.9,"max_frames":384,"max_model_len":65536,"max_new_tokens":128,"max_num_batched_tokens":65536,"min_p":0.0,"presence_penalty":0.0,"prompt_mode":"default","temperature":0.0,"top_k":-1,"top_p":1.0,"video_max_pixels":262144,"video_min_pixels":4096,"video_total_pixels":0},"redistribution_authorized":true,"schema_version":1,"source_url":"https://github.com/JUNJIE99/MLVU","split":"dev","task":"mlvu"} +{"annotation_path":"annotations/spatial_intelligence/mmsi/test.jsonl","artifact_paths":[],"benchmark":"mmsi","evaluation":{"metric_profile":"accuracy","parser_profile":"multiple_choice","prompt_profile":"mmsi_default"},"expected_count":1000,"family":"spatial_intelligence","legacy_environment":{"MMSI_BACKEND":"transformers","MMSI_ENABLE_THINKING":false,"MMSI_IMAGE_MAX_PIXELS":262144,"MMSI_IMAGE_MIN_PIXELS":4096},"license":"See the upstream benchmark terms","media_paths":["media/spatial_intelligence/mmsi/images/test"],"preprocessing":{"image_max_pixels":262144,"image_min_pixels":4096},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/RunsenXu/MMSI-Bench","split":"test","task":"mmsi"} +{"annotation_path":"annotations/video_qa/mmvu/test.jsonl","artifact_paths":[],"benchmark":"mmvu","evaluation":{"metric_profile":"accuracy","parser_profile":"multiple_choice","prompt_profile":"mmvu_default"},"expected_count":625,"family":"video_qa","legacy_environment":{"MMVU_BATCH_SIZE":1,"MMVU_ENABLE_THINKING":false,"MMVU_FPS":2,"MMVU_GPU_MEMORY_UTILIZATION":0.9,"MMVU_MAX_FRAMES":384,"MMVU_MAX_MODEL_LEN":65536,"MMVU_MAX_NEW_TOKENS":128,"MMVU_MAX_NUM_BATCHED_TOKENS":65536,"MMVU_MIN_P":0.0,"MMVU_PRESENCE_PENALTY":0.0,"MMVU_PROMPT_MODE":"default","MMVU_TEMPERATURE":0.0,"MMVU_TOP_K":-1,"MMVU_TOP_P":1.0,"MMVU_VIDEO_MAX_PIXELS":262144,"MMVU_VIDEO_MIN_PIXELS":4096,"MMVU_VIDEO_TOTAL_PIXELS":0},"license":"See the upstream benchmark terms","media_paths":["media/video_qa/mmvu/videos","media/video_qa/mmvu/videos/test"],"preprocessing":{"batch_size":1,"enable_thinking":false,"fps":2,"gpu_memory_utilization":0.9,"max_frames":384,"max_model_len":65536,"max_new_tokens":128,"max_num_batched_tokens":65536,"min_p":0.0,"presence_penalty":0.0,"prompt_mode":"default","temperature":0.0,"top_k":-1,"top_p":1.0,"video_max_pixels":262144,"video_min_pixels":4096,"video_total_pixels":0},"redistribution_authorized":true,"schema_version":1,"source_url":"https://github.com/yale-nlp/MMVU","split":"test","task":"mmvu"} +{"annotation_path":"annotations/video_qa/mvbench/test.jsonl","artifact_paths":[],"benchmark":"mvbench","evaluation":{"metric_profile":"accuracy","parser_profile":"multiple_choice","prompt_profile":"mvbench_default"},"expected_count":3586,"family":"video_qa","legacy_environment":{"MVBENCH_BATCH_SIZE":1,"MVBENCH_ENABLE_THINKING":false,"MVBENCH_FPS":2,"MVBENCH_GPU_MEMORY_UTILIZATION":0.9,"MVBENCH_MAX_FRAMES":384,"MVBENCH_MAX_MODEL_LEN":65536,"MVBENCH_MAX_NEW_TOKENS":128,"MVBENCH_MAX_NUM_BATCHED_TOKENS":65536,"MVBENCH_MIN_P":0.0,"MVBENCH_PRESENCE_PENALTY":0.0,"MVBENCH_PROMPT_MODE":"default","MVBENCH_TEMPERATURE":0.0,"MVBENCH_TOP_K":-1,"MVBENCH_TOP_P":1.0,"MVBENCH_VIDEO_MAX_PIXELS":262144,"MVBENCH_VIDEO_MIN_PIXELS":4096,"MVBENCH_VIDEO_TOTAL_PIXELS":0},"license":"See the upstream benchmark terms","media_paths":["media/video_qa/mvbench/videos","media/video_qa/mvbench/videos/test"],"preprocessing":{"batch_size":1,"enable_thinking":false,"fps":2,"gpu_memory_utilization":0.9,"max_frames":384,"max_model_len":65536,"max_new_tokens":128,"max_num_batched_tokens":65536,"min_p":0.0,"presence_penalty":0.0,"prompt_mode":"default","temperature":0.0,"top_k":-1,"top_p":1.0,"video_max_pixels":262144,"video_min_pixels":4096,"video_total_pixels":0},"redistribution_authorized":true,"schema_version":1,"source_url":"https://github.com/OpenGVLab/Ask-Anything","split":"test","task":"mvbench"} +{"annotation_path":"annotations/spatial_intelligence/revsi/test.jsonl","artifact_paths":[],"benchmark":"revsi","evaluation":{"aggregation":"macro_by_question_type","frame_protocol":"native_all_frame","metric_profile":"revsi_macro_average","parser_profile":"revsi_answer","prompt_profile":"revsi_official"},"expected_count":6808,"family":"spatial_intelligence","legacy_environment":{"REVSI_BATCH_SIZE":16,"REVSI_ENABLE_THINKING":false,"REVSI_EXACT_NFRAMES":true,"REVSI_EXPECTED_SAMPLES":6808,"REVSI_FPS":2,"REVSI_FRAME_BUDGET":"all","REVSI_GPU_MEMORY_UTILIZATION":0.9,"REVSI_MAX_FRAMES":128,"REVSI_MAX_MODEL_LEN":32768,"REVSI_MAX_NEW_TOKENS":64,"REVSI_SETTING":"native-all-f128-exacttrue-fps2-min65536-maxnone-total16777216","REVSI_VIDEO_MIN_PIXELS":65536,"REVSI_VIDEO_TOTAL_PIXELS":16777216},"license":"See the upstream benchmark terms","media_paths":["media/spatial_intelligence/revsi/videos","media/spatial_intelligence/revsi/videos/test"],"preprocessing":{"exact_nframes":true,"fps":2,"max_frames":128,"video_min_pixels":65536,"video_total_pixels":16777216},"redistribution_authorized":true,"schema_version":1,"source_url":"https://arxiv.org/abs/2605.25979","split":"test","task":"revsi"} +{"annotation_path":"annotations/segmentation/mevis.jsonl","artifact_paths":[],"benchmark":"segmentation","evaluation":{"metric_profile":"segmentation_official","oracle_location":"task_payload.segmentation_output","oracle_profile":"segmentation_rle","parser_profile":"sam2_prompt","prompt_profile":"train_seg"},"expected_count":424,"family":"segmentation","legacy_environment":{"SEGMENTATION_BATCH_SIZE":16,"SEGMENTATION_DATASETS":"refcoco,refcocop,refcocog,mevis,reasonvos","SEGMENTATION_DATA_TYPE":"all","SEGMENTATION_ENABLE_THINKING":false,"SEGMENTATION_FPS":2,"SEGMENTATION_GPU_MEM_UTIL":0.85,"SEGMENTATION_MAX_FRAMES":128,"SEGMENTATION_MAX_MODEL_LEN":32768,"SEGMENTATION_MAX_NEW_TOKENS":1024,"SEGMENTATION_PROMPT_MODE":"train_seg","SEGMENTATION_RUN_SAM2":false,"SEGMENTATION_SEED":42,"SEGMENTATION_SETTING":"segmentation-refcoco_refcocop_refcocog_mevis_reasonvos-train_seg-f128-fps2-min4096-max262144-total16777216-readerdecord-new1024","SEGMENTATION_VIDEO_MAX_PIXELS":262144,"SEGMENTATION_VIDEO_MIN_PIXELS":4096,"SEGMENTATION_VIDEO_READER":"decord","SEGMENTATION_VIDEO_TOTAL_PIXELS":16777216},"license":"See the upstream benchmark terms","media_paths":["media/segmentation","media/segmentation/videos/mevis"],"preprocessing":{"fps":2,"max_frames":128,"video_max_pixels":262144,"video_min_pixels":4096,"video_reader":"decord","video_total_pixels":16777216},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/OneThink/OneThinker-eval","split":"mevis","task":"segmentation"} +{"annotation_path":"annotations/segmentation/reasonvos.jsonl","artifact_paths":[],"benchmark":"segmentation","evaluation":{"metric_profile":"segmentation_official","oracle_location":"task_payload.segmentation_output","oracle_profile":"segmentation_rle","parser_profile":"sam2_prompt","prompt_profile":"train_seg"},"expected_count":458,"family":"segmentation","legacy_environment":{"SEGMENTATION_BATCH_SIZE":16,"SEGMENTATION_DATASETS":"refcoco,refcocop,refcocog,mevis,reasonvos","SEGMENTATION_DATA_TYPE":"all","SEGMENTATION_ENABLE_THINKING":false,"SEGMENTATION_FPS":2,"SEGMENTATION_GPU_MEM_UTIL":0.85,"SEGMENTATION_MAX_FRAMES":128,"SEGMENTATION_MAX_MODEL_LEN":32768,"SEGMENTATION_MAX_NEW_TOKENS":1024,"SEGMENTATION_PROMPT_MODE":"train_seg","SEGMENTATION_RUN_SAM2":false,"SEGMENTATION_SEED":42,"SEGMENTATION_SETTING":"segmentation-refcoco_refcocop_refcocog_mevis_reasonvos-train_seg-f128-fps2-min4096-max262144-total16777216-readerdecord-new1024","SEGMENTATION_VIDEO_MAX_PIXELS":262144,"SEGMENTATION_VIDEO_MIN_PIXELS":4096,"SEGMENTATION_VIDEO_READER":"decord","SEGMENTATION_VIDEO_TOTAL_PIXELS":16777216},"license":"See the upstream benchmark terms","media_paths":["media/segmentation","media/segmentation/videos/reasonvos"],"preprocessing":{"fps":2,"max_frames":128,"video_max_pixels":262144,"video_min_pixels":4096,"video_reader":"decord","video_total_pixels":16777216},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/OneThink/OneThinker-eval","split":"reasonvos","task":"segmentation"} +{"annotation_path":"annotations/segmentation/refcoco.jsonl","artifact_paths":[],"benchmark":"segmentation","evaluation":{"metric_profile":"segmentation_official","oracle_location":"task_payload.segmentation_output","oracle_profile":"segmentation_rle","parser_profile":"sam2_prompt","prompt_profile":"train_seg"},"expected_count":3811,"family":"segmentation","legacy_environment":{"SEGMENTATION_BATCH_SIZE":16,"SEGMENTATION_DATASETS":"refcoco,refcocop,refcocog,mevis,reasonvos","SEGMENTATION_DATA_TYPE":"all","SEGMENTATION_ENABLE_THINKING":false,"SEGMENTATION_FPS":2,"SEGMENTATION_GPU_MEM_UTIL":0.85,"SEGMENTATION_MAX_FRAMES":128,"SEGMENTATION_MAX_MODEL_LEN":32768,"SEGMENTATION_MAX_NEW_TOKENS":1024,"SEGMENTATION_PROMPT_MODE":"train_seg","SEGMENTATION_RUN_SAM2":false,"SEGMENTATION_SEED":42,"SEGMENTATION_SETTING":"segmentation-refcoco_refcocop_refcocog_mevis_reasonvos-train_seg-f128-fps2-min4096-max262144-total16777216-readerdecord-new1024","SEGMENTATION_VIDEO_MAX_PIXELS":262144,"SEGMENTATION_VIDEO_MIN_PIXELS":4096,"SEGMENTATION_VIDEO_READER":"decord","SEGMENTATION_VIDEO_TOTAL_PIXELS":16777216},"license":"See the upstream benchmark terms","media_paths":["media/segmentation","media/segmentation/images/refcoco"],"preprocessing":{"fps":2,"max_frames":128,"video_max_pixels":262144,"video_min_pixels":4096,"video_reader":"decord","video_total_pixels":16777216},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/OneThink/OneThinker-eval","split":"refcoco","task":"segmentation"} +{"annotation_path":"annotations/segmentation/refcocog.jsonl","artifact_paths":[],"benchmark":"segmentation","evaluation":{"metric_profile":"segmentation_official","oracle_location":"task_payload.segmentation_output","oracle_profile":"segmentation_rle","parser_profile":"sam2_prompt","prompt_profile":"train_seg"},"expected_count":2537,"family":"segmentation","legacy_environment":{"SEGMENTATION_BATCH_SIZE":16,"SEGMENTATION_DATASETS":"refcoco,refcocop,refcocog,mevis,reasonvos","SEGMENTATION_DATA_TYPE":"all","SEGMENTATION_ENABLE_THINKING":false,"SEGMENTATION_FPS":2,"SEGMENTATION_GPU_MEM_UTIL":0.85,"SEGMENTATION_MAX_FRAMES":128,"SEGMENTATION_MAX_MODEL_LEN":32768,"SEGMENTATION_MAX_NEW_TOKENS":1024,"SEGMENTATION_PROMPT_MODE":"train_seg","SEGMENTATION_RUN_SAM2":false,"SEGMENTATION_SEED":42,"SEGMENTATION_SETTING":"segmentation-refcoco_refcocop_refcocog_mevis_reasonvos-train_seg-f128-fps2-min4096-max262144-total16777216-readerdecord-new1024","SEGMENTATION_VIDEO_MAX_PIXELS":262144,"SEGMENTATION_VIDEO_MIN_PIXELS":4096,"SEGMENTATION_VIDEO_READER":"decord","SEGMENTATION_VIDEO_TOTAL_PIXELS":16777216},"license":"See the upstream benchmark terms","media_paths":["media/segmentation","media/segmentation/images/refcocog"],"preprocessing":{"fps":2,"max_frames":128,"video_max_pixels":262144,"video_min_pixels":4096,"video_reader":"decord","video_total_pixels":16777216},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/OneThink/OneThinker-eval","split":"refcocog","task":"segmentation"} +{"annotation_path":"annotations/segmentation/refcocop.jsonl","artifact_paths":[],"benchmark":"segmentation","evaluation":{"metric_profile":"segmentation_official","oracle_location":"task_payload.segmentation_output","oracle_profile":"segmentation_rle","parser_profile":"sam2_prompt","prompt_profile":"train_seg"},"expected_count":3805,"family":"segmentation","legacy_environment":{"SEGMENTATION_BATCH_SIZE":16,"SEGMENTATION_DATASETS":"refcoco,refcocop,refcocog,mevis,reasonvos","SEGMENTATION_DATA_TYPE":"all","SEGMENTATION_ENABLE_THINKING":false,"SEGMENTATION_FPS":2,"SEGMENTATION_GPU_MEM_UTIL":0.85,"SEGMENTATION_MAX_FRAMES":128,"SEGMENTATION_MAX_MODEL_LEN":32768,"SEGMENTATION_MAX_NEW_TOKENS":1024,"SEGMENTATION_PROMPT_MODE":"train_seg","SEGMENTATION_RUN_SAM2":false,"SEGMENTATION_SEED":42,"SEGMENTATION_SETTING":"segmentation-refcoco_refcocop_refcocog_mevis_reasonvos-train_seg-f128-fps2-min4096-max262144-total16777216-readerdecord-new1024","SEGMENTATION_VIDEO_MAX_PIXELS":262144,"SEGMENTATION_VIDEO_MIN_PIXELS":4096,"SEGMENTATION_VIDEO_READER":"decord","SEGMENTATION_VIDEO_TOTAL_PIXELS":16777216},"license":"See the upstream benchmark terms","media_paths":["media/segmentation","media/segmentation/images/refcocop"],"preprocessing":{"fps":2,"max_frames":128,"video_max_pixels":262144,"video_min_pixels":4096,"video_reader":"decord","video_total_pixels":16777216},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/OneThink/OneThinker-eval","split":"refcocop","task":"segmentation"} +{"annotation_path":"annotations/spatial_grounding/refcoco_test_a.jsonl","artifact_paths":[],"benchmark":"spatial_grounding","evaluation":{"metric_profile":"refcoco_iou","parser_profile":"norm1000_bbox","prompt_profile":"qwen_native"},"expected_count":5657,"family":"spatial_grounding","legacy_environment":{"SPATIAL_GROUNDING_BBOX_SELECT":"first","SPATIAL_GROUNDING_COORD_SYSTEM":"norm1000","SPATIAL_GROUNDING_DATASETS":"refcoco-val,refcoco-testA,refcoco-testB,refcoco+-val,refcoco+-testA,refcoco+-testB,refcocog-val,refcocog-test","SPATIAL_GROUNDING_MAX_NEW_TOKENS":1024,"SPATIAL_GROUNDING_MIN_TOKENS":64,"SPATIAL_GROUNDING_PROMPT_STYLE":"qwen_native","SPATIAL_GROUNDING_TOTAL_TOKENS":1024},"license":"See the upstream benchmark terms","media_paths":["media/spatial_grounding/images","media/spatial_grounding/images/refcoco_test_a"],"preprocessing":{"coordinate_system":"norm1000"},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/OneThink/OneThinker-eval","split":"refcoco_test_a","task":"spatial_grounding"} +{"annotation_path":"annotations/spatial_grounding/refcoco_test_b.jsonl","artifact_paths":[],"benchmark":"spatial_grounding","evaluation":{"metric_profile":"refcoco_iou","parser_profile":"norm1000_bbox","prompt_profile":"qwen_native"},"expected_count":5095,"family":"spatial_grounding","legacy_environment":{"SPATIAL_GROUNDING_BBOX_SELECT":"first","SPATIAL_GROUNDING_COORD_SYSTEM":"norm1000","SPATIAL_GROUNDING_DATASETS":"refcoco-val,refcoco-testA,refcoco-testB,refcoco+-val,refcoco+-testA,refcoco+-testB,refcocog-val,refcocog-test","SPATIAL_GROUNDING_MAX_NEW_TOKENS":1024,"SPATIAL_GROUNDING_MIN_TOKENS":64,"SPATIAL_GROUNDING_PROMPT_STYLE":"qwen_native","SPATIAL_GROUNDING_TOTAL_TOKENS":1024},"license":"See the upstream benchmark terms","media_paths":["media/spatial_grounding/images","media/spatial_grounding/images/refcoco_test_b"],"preprocessing":{"coordinate_system":"norm1000"},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/OneThink/OneThinker-eval","split":"refcoco_test_b","task":"spatial_grounding"} +{"annotation_path":"annotations/spatial_grounding/refcoco_val.jsonl","artifact_paths":[],"benchmark":"spatial_grounding","evaluation":{"metric_profile":"refcoco_iou","parser_profile":"norm1000_bbox","prompt_profile":"qwen_native"},"expected_count":10834,"family":"spatial_grounding","legacy_environment":{"SPATIAL_GROUNDING_BBOX_SELECT":"first","SPATIAL_GROUNDING_COORD_SYSTEM":"norm1000","SPATIAL_GROUNDING_DATASETS":"refcoco-val,refcoco-testA,refcoco-testB,refcoco+-val,refcoco+-testA,refcoco+-testB,refcocog-val,refcocog-test","SPATIAL_GROUNDING_MAX_NEW_TOKENS":1024,"SPATIAL_GROUNDING_MIN_TOKENS":64,"SPATIAL_GROUNDING_PROMPT_STYLE":"qwen_native","SPATIAL_GROUNDING_TOTAL_TOKENS":1024},"license":"See the upstream benchmark terms","media_paths":["media/spatial_grounding/images","media/spatial_grounding/images/refcoco_val"],"preprocessing":{"coordinate_system":"norm1000"},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/OneThink/OneThinker-eval","split":"refcoco_val","task":"spatial_grounding"} +{"annotation_path":"annotations/spatial_grounding/refcocog_test.jsonl","artifact_paths":[],"benchmark":"spatial_grounding","evaluation":{"metric_profile":"refcoco_iou","parser_profile":"norm1000_bbox","prompt_profile":"qwen_native"},"expected_count":9602,"family":"spatial_grounding","legacy_environment":{"SPATIAL_GROUNDING_BBOX_SELECT":"first","SPATIAL_GROUNDING_COORD_SYSTEM":"norm1000","SPATIAL_GROUNDING_DATASETS":"refcoco-val,refcoco-testA,refcoco-testB,refcoco+-val,refcoco+-testA,refcoco+-testB,refcocog-val,refcocog-test","SPATIAL_GROUNDING_MAX_NEW_TOKENS":1024,"SPATIAL_GROUNDING_MIN_TOKENS":64,"SPATIAL_GROUNDING_PROMPT_STYLE":"qwen_native","SPATIAL_GROUNDING_TOTAL_TOKENS":1024},"license":"See the upstream benchmark terms","media_paths":["media/spatial_grounding/images","media/spatial_grounding/images/refcocog_test"],"preprocessing":{"coordinate_system":"norm1000"},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/OneThink/OneThinker-eval","split":"refcocog_test","task":"spatial_grounding"} +{"annotation_path":"annotations/spatial_grounding/refcocog_val.jsonl","artifact_paths":[],"benchmark":"spatial_grounding","evaluation":{"metric_profile":"refcoco_iou","parser_profile":"norm1000_bbox","prompt_profile":"qwen_native"},"expected_count":4896,"family":"spatial_grounding","legacy_environment":{"SPATIAL_GROUNDING_BBOX_SELECT":"first","SPATIAL_GROUNDING_COORD_SYSTEM":"norm1000","SPATIAL_GROUNDING_DATASETS":"refcoco-val,refcoco-testA,refcoco-testB,refcoco+-val,refcoco+-testA,refcoco+-testB,refcocog-val,refcocog-test","SPATIAL_GROUNDING_MAX_NEW_TOKENS":1024,"SPATIAL_GROUNDING_MIN_TOKENS":64,"SPATIAL_GROUNDING_PROMPT_STYLE":"qwen_native","SPATIAL_GROUNDING_TOTAL_TOKENS":1024},"license":"See the upstream benchmark terms","media_paths":["media/spatial_grounding/images","media/spatial_grounding/images/refcocog_val"],"preprocessing":{"coordinate_system":"norm1000"},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/OneThink/OneThinker-eval","split":"refcocog_val","task":"spatial_grounding"} +{"annotation_path":"annotations/spatial_grounding/refcocop_test_a.jsonl","artifact_paths":[],"benchmark":"spatial_grounding","evaluation":{"metric_profile":"refcoco_iou","parser_profile":"norm1000_bbox","prompt_profile":"qwen_native"},"expected_count":5726,"family":"spatial_grounding","legacy_environment":{"SPATIAL_GROUNDING_BBOX_SELECT":"first","SPATIAL_GROUNDING_COORD_SYSTEM":"norm1000","SPATIAL_GROUNDING_DATASETS":"refcoco-val,refcoco-testA,refcoco-testB,refcoco+-val,refcoco+-testA,refcoco+-testB,refcocog-val,refcocog-test","SPATIAL_GROUNDING_MAX_NEW_TOKENS":1024,"SPATIAL_GROUNDING_MIN_TOKENS":64,"SPATIAL_GROUNDING_PROMPT_STYLE":"qwen_native","SPATIAL_GROUNDING_TOTAL_TOKENS":1024},"license":"See the upstream benchmark terms","media_paths":["media/spatial_grounding/images","media/spatial_grounding/images/refcocop_test_a"],"preprocessing":{"coordinate_system":"norm1000"},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/OneThink/OneThinker-eval","split":"refcocop_test_a","task":"spatial_grounding"} +{"annotation_path":"annotations/spatial_grounding/refcocop_test_b.jsonl","artifact_paths":[],"benchmark":"spatial_grounding","evaluation":{"metric_profile":"refcoco_iou","parser_profile":"norm1000_bbox","prompt_profile":"qwen_native"},"expected_count":4889,"family":"spatial_grounding","legacy_environment":{"SPATIAL_GROUNDING_BBOX_SELECT":"first","SPATIAL_GROUNDING_COORD_SYSTEM":"norm1000","SPATIAL_GROUNDING_DATASETS":"refcoco-val,refcoco-testA,refcoco-testB,refcoco+-val,refcoco+-testA,refcoco+-testB,refcocog-val,refcocog-test","SPATIAL_GROUNDING_MAX_NEW_TOKENS":1024,"SPATIAL_GROUNDING_MIN_TOKENS":64,"SPATIAL_GROUNDING_PROMPT_STYLE":"qwen_native","SPATIAL_GROUNDING_TOTAL_TOKENS":1024},"license":"See the upstream benchmark terms","media_paths":["media/spatial_grounding/images","media/spatial_grounding/images/refcocop_test_b"],"preprocessing":{"coordinate_system":"norm1000"},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/OneThink/OneThinker-eval","split":"refcocop_test_b","task":"spatial_grounding"} +{"annotation_path":"annotations/spatial_grounding/refcocop_val.jsonl","artifact_paths":[],"benchmark":"spatial_grounding","evaluation":{"metric_profile":"refcoco_iou","parser_profile":"norm1000_bbox","prompt_profile":"qwen_native"},"expected_count":10758,"family":"spatial_grounding","legacy_environment":{"SPATIAL_GROUNDING_BBOX_SELECT":"first","SPATIAL_GROUNDING_COORD_SYSTEM":"norm1000","SPATIAL_GROUNDING_DATASETS":"refcoco-val,refcoco-testA,refcoco-testB,refcoco+-val,refcoco+-testA,refcoco+-testB,refcocog-val,refcocog-test","SPATIAL_GROUNDING_MAX_NEW_TOKENS":1024,"SPATIAL_GROUNDING_MIN_TOKENS":64,"SPATIAL_GROUNDING_PROMPT_STYLE":"qwen_native","SPATIAL_GROUNDING_TOTAL_TOKENS":1024},"license":"See the upstream benchmark terms","media_paths":["media/spatial_grounding/images","media/spatial_grounding/images/refcocop_val"],"preprocessing":{"coordinate_system":"norm1000"},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/OneThink/OneThinker-eval","split":"refcocop_val","task":"spatial_grounding"} +{"annotation_path":"annotations/stvg/stvg.jsonl","artifact_paths":[],"benchmark":"stvg","evaluation":{"metric_profile":"stvg_official","parser_profile":"temporal_spatial_boxes","prompt_profile":"train_stvg"},"expected_count":1851,"family":"spatial_temporal_grounding","legacy_environment":{"STVG_DATASETS":"eval_stvg","STVG_FPS":2,"STVG_MAX_FRAMES":128,"STVG_MAX_NEW_TOKENS":2048,"STVG_PROMPT_MODE":"train_stvg","STVG_VIDEO_MAX_PIXELS":393216,"STVG_VIDEO_MIN_PIXELS":65536,"STVG_VIDEO_TOTAL_PIXELS":10485760},"license":"See the upstream benchmark terms","media_paths":["media/stvg","media/stvg/videos/stvg"],"preprocessing":{"stvg_fps":2,"stvg_max_frames":128,"stvg_video_max_pixels":393216,"stvg_video_min_pixels":65536,"stvg_video_total_pixels":10485760},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/OneThink/OneThinker-eval","split":"stvg","task":"stvg"} +{"annotation_path":"annotations/temporal_grounding/activitynet_timelens.jsonl","artifact_paths":[],"benchmark":"temporal_grounding","checksums":{"annotations/temporal_grounding/activitynet_timelens.jsonl":"3091f03a20b2bf4548d4079d95bb8dc564da53d4e6b2c32470862b15622c7a77"},"evaluation":{"metric_profile":"temporal_iou","parser_profile":"temporal_spans","prompt_profile":"timelens_same"},"expected_count":4500,"family":"temporal_grounding","legacy_environment":{"TIMELENS_DATASETS":"charades_timelens,activitynet_timelens,qvhighlights_timelens","TIMELENS_ENABLE_THINKING":false,"TIMELENS_FPS":4,"TIMELENS_MAX_FRAMES":2048,"TIMELENS_MAX_NEW_TOKENS":128,"TIMELENS_MAX_PIXELS":409600,"TIMELENS_MIN_TOKENS":1,"TIMELENS_NUM_WORKERS":2,"TIMELENS_PROMPT_MODE":"same","TIMELENS_STOP_AFTER_ANSWER":true,"TIMELENS_TOTAL_TOKENS":128000},"license":"See the upstream benchmark terms","media_paths":["media/temporal_grounding/videos"],"preprocessing":{"fps":4,"max_frames":2048,"max_pixels":409600,"min_tokens":1,"total_tokens":128000},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/TencentARC/TimeLens-Bench","split":"activitynet_timelens","task":"temporal_grounding"} +{"annotation_path":"annotations/temporal_grounding/charades_timelens.jsonl","artifact_paths":[],"benchmark":"temporal_grounding","checksums":{"annotations/temporal_grounding/charades_timelens.jsonl":"af289d9be3e8c56f98b12c39bda37758a34205c1aa75e2c03e3174dbbe6f2794"},"evaluation":{"metric_profile":"temporal_iou","parser_profile":"temporal_spans","prompt_profile":"timelens_same"},"expected_count":3363,"family":"temporal_grounding","legacy_environment":{"TIMELENS_DATASETS":"charades_timelens,activitynet_timelens,qvhighlights_timelens","TIMELENS_ENABLE_THINKING":false,"TIMELENS_FPS":4,"TIMELENS_MAX_FRAMES":2048,"TIMELENS_MAX_NEW_TOKENS":128,"TIMELENS_MAX_PIXELS":409600,"TIMELENS_MIN_TOKENS":1,"TIMELENS_NUM_WORKERS":2,"TIMELENS_PROMPT_MODE":"same","TIMELENS_STOP_AFTER_ANSWER":true,"TIMELENS_TOTAL_TOKENS":128000},"license":"See the upstream benchmark terms","media_paths":["media/temporal_grounding/videos"],"preprocessing":{"fps":4,"max_frames":2048,"max_pixels":409600,"min_tokens":1,"total_tokens":128000},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/TencentARC/TimeLens-Bench","split":"charades_timelens","task":"temporal_grounding"} +{"annotation_path":"annotations/temporal_grounding/qvhighlights_timelens.jsonl","artifact_paths":[],"benchmark":"temporal_grounding","checksums":{"annotations/temporal_grounding/qvhighlights_timelens.jsonl":"922a736a73ff9fb3d3996e0b75ee1343283bee23d7e60b368d6840f8612e619b"},"evaluation":{"metric_profile":"temporal_iou","parser_profile":"temporal_spans","prompt_profile":"timelens_same"},"expected_count":1541,"family":"temporal_grounding","legacy_environment":{"TIMELENS_DATASETS":"charades_timelens,activitynet_timelens,qvhighlights_timelens","TIMELENS_ENABLE_THINKING":false,"TIMELENS_FPS":4,"TIMELENS_MAX_FRAMES":2048,"TIMELENS_MAX_NEW_TOKENS":128,"TIMELENS_MAX_PIXELS":409600,"TIMELENS_MIN_TOKENS":1,"TIMELENS_NUM_WORKERS":2,"TIMELENS_PROMPT_MODE":"same","TIMELENS_STOP_AFTER_ANSWER":true,"TIMELENS_TOTAL_TOKENS":128000},"license":"See the upstream benchmark terms","media_paths":["media/temporal_grounding/videos"],"preprocessing":{"fps":4,"max_frames":2048,"max_pixels":409600,"min_tokens":1,"total_tokens":128000},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/TencentARC/TimeLens-Bench","split":"qvhighlights_timelens","task":"temporal_grounding"} +{"annotation_path":"annotations/tracking/got10k.jsonl","artifact_paths":[],"benchmark":"tracking","checksums":{"annotations/tracking/got10k.jsonl":"1206da3c1fd75360ccea074a9e211965fe2d8cfd514903564b037641a00fb12b"},"evaluation":{"metric_profile":"got10k_ao","parser_profile":"tracking_boxes","prompt_profile":"tracking_default"},"expected_count":180,"family":"tracking","legacy_environment":{"TRACKING_DATASETS":"eval_got10k","TRACKING_FPS":1,"TRACKING_MAX_FRAMES":32,"TRACKING_MAX_NEW_TOKENS":8192,"TRACKING_PROMPT_MODE":"default","TRACKING_VIDEO_MAX_PIXELS":786432,"TRACKING_VIDEO_MIN_PIXELS":4096,"TRACKING_VIDEO_TOTAL_PIXELS":8388608},"license":"See the upstream benchmark terms","media_paths":["media/tracking","media/tracking/videos"],"preprocessing":{"tracking_fps":1,"tracking_max_frames":32,"tracking_video_max_pixels":786432,"tracking_video_min_pixels":4096,"tracking_video_total_pixels":8388608},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/OneThink/OneThinker-eval","split":"got10k","task":"tracking"} +{"annotation_path":"annotations/video_qa/videoholmes/test.jsonl","artifact_paths":[],"benchmark":"videoholmes","evaluation":{"metric_profile":"accuracy","parser_profile":"multiple_choice","prompt_profile":"videoholmes_default"},"expected_count":1837,"family":"video_qa","legacy_environment":{"VIDEOHOLMES_BATCH_SIZE":1,"VIDEOHOLMES_ENABLE_THINKING":false,"VIDEOHOLMES_FPS":2,"VIDEOHOLMES_GPU_MEMORY_UTILIZATION":0.9,"VIDEOHOLMES_MAX_FRAMES":384,"VIDEOHOLMES_MAX_MODEL_LEN":65536,"VIDEOHOLMES_MAX_NEW_TOKENS":128,"VIDEOHOLMES_MAX_NUM_BATCHED_TOKENS":65536,"VIDEOHOLMES_MIN_P":0.0,"VIDEOHOLMES_PRESENCE_PENALTY":0.0,"VIDEOHOLMES_PROMPT_MODE":"default","VIDEOHOLMES_TEMPERATURE":0.0,"VIDEOHOLMES_TOP_K":-1,"VIDEOHOLMES_TOP_P":1.0,"VIDEOHOLMES_VIDEO_MAX_PIXELS":262144,"VIDEOHOLMES_VIDEO_MIN_PIXELS":4096,"VIDEOHOLMES_VIDEO_TOTAL_PIXELS":0},"license":"See the upstream benchmark terms","media_paths":["media/video_qa/videoholmes/videos","media/video_qa/videoholmes/videos/test"],"preprocessing":{"batch_size":1,"enable_thinking":false,"fps":2,"gpu_memory_utilization":0.9,"max_frames":384,"max_model_len":65536,"max_new_tokens":128,"max_num_batched_tokens":65536,"min_p":0.0,"presence_penalty":0.0,"prompt_mode":"default","temperature":0.0,"top_k":-1,"top_p":1.0,"video_max_pixels":262144,"video_min_pixels":4096,"video_total_pixels":0},"redistribution_authorized":true,"schema_version":1,"source_url":"https://github.com/TencentARC/Video-Holmes","split":"test","task":"videoholmes"} +{"annotation_path":"annotations/video_qa/videomme/test.jsonl","artifact_paths":["artifacts/video_qa/videomme"],"benchmark":"videomme","evaluation":{"metric_profile":"accuracy","parser_profile":"multiple_choice","prompt_profile":"videomme_default"},"expected_count":2700,"family":"video_qa","legacy_environment":{"VIDEOMME_BATCH_SIZE":1,"VIDEOMME_ENABLE_THINKING":false,"VIDEOMME_FPS":2,"VIDEOMME_GPU_MEMORY_UTILIZATION":0.9,"VIDEOMME_MAX_FRAMES":384,"VIDEOMME_MAX_MODEL_LEN":65536,"VIDEOMME_MAX_NEW_TOKENS":128,"VIDEOMME_MAX_NUM_BATCHED_TOKENS":65536,"VIDEOMME_MIN_P":0.0,"VIDEOMME_PRESENCE_PENALTY":0.0,"VIDEOMME_PROMPT_MODE":"default","VIDEOMME_SETTING":"all-qwen3_vl-sub0-f384-fps2-min4096-max262144-total0-videomme_preprocessed_384f_262k_total0","VIDEOMME_TEMPERATURE":0.0,"VIDEOMME_TOP_K":-1,"VIDEOMME_TOP_P":1.0,"VIDEOMME_VIDEO_MAX_PIXELS":262144,"VIDEOMME_VIDEO_MIN_PIXELS":4096,"VIDEOMME_VIDEO_TOTAL_PIXELS":0},"license":"See the upstream benchmark terms","media_paths":["media/video_qa/videomme/videos","media/video_qa/videomme/videos/test"],"preprocessing":{"batch_size":1,"enable_thinking":false,"fps":2,"gpu_memory_utilization":0.9,"max_frames":384,"max_model_len":65536,"max_new_tokens":128,"max_num_batched_tokens":65536,"min_p":0.0,"presence_penalty":0.0,"prompt_mode":"default","temperature":0.0,"top_k":-1,"top_p":1.0,"video_max_pixels":262144,"video_min_pixels":4096,"video_total_pixels":0},"redistribution_authorized":true,"schema_version":1,"source_url":"https://github.com/BradyFU/Video-MME","split":"test","task":"videomme"} +{"annotation_path":"annotations/video_qa/videommev2/test.jsonl","artifact_paths":["artifacts/video_qa/videommev2"],"benchmark":"videommev2","evaluation":{"metric_profile":"accuracy","parser_profile":"multiple_choice","prompt_profile":"videommev2_default"},"expected_count":3200,"family":"video_qa","legacy_environment":{"VIDEOMMEV2_ANSWER_FILTER":"","VIDEOMMEV2_BATCH_SIZE":1,"VIDEOMMEV2_ENABLE_THINKING":false,"VIDEOMMEV2_FPS":2,"VIDEOMMEV2_GPU_MEMORY_UTILIZATION":0.9,"VIDEOMMEV2_MAX_FRAMES":384,"VIDEOMMEV2_MAX_MODEL_LEN":65536,"VIDEOMMEV2_MAX_NEW_TOKENS":128,"VIDEOMMEV2_MAX_NUM_BATCHED_TOKENS":65536,"VIDEOMMEV2_MIN_P":0.0,"VIDEOMMEV2_PRESENCE_PENALTY":0.0,"VIDEOMMEV2_PROMPT_MODE":"default","VIDEOMMEV2_TEMPERATURE":0.0,"VIDEOMMEV2_TOP_K":-1,"VIDEOMMEV2_TOP_P":1.0,"VIDEOMMEV2_VIDEO_MAX_PIXELS":262144,"VIDEOMMEV2_VIDEO_MIN_PIXELS":4096,"VIDEOMMEV2_VIDEO_TOTAL_PIXELS":0},"license":"See the upstream benchmark terms","media_paths":["media/video_qa/videommev2/videos","media/video_qa/videommev2/videos/test"],"preprocessing":{"batch_size":1,"enable_thinking":false,"fps":2,"gpu_memory_utilization":0.9,"max_frames":384,"max_model_len":65536,"max_new_tokens":128,"max_num_batched_tokens":65536,"min_p":0.0,"presence_penalty":0.0,"prompt_mode":"default","temperature":0.0,"top_k":-1,"top_p":1.0,"video_max_pixels":262144,"video_min_pixels":4096,"video_total_pixels":0},"redistribution_authorized":true,"schema_version":1,"source_url":"https://github.com/BradyFU/Video-MME","split":"test","task":"videommev2"} +{"annotation_path":"annotations/spatial_intelligence/vsi/test.jsonl","artifact_paths":["artifacts/spatial_intelligence/vsi"],"benchmark":"vsi","evaluation":{"metric_profile":"vsi_official","parser_profile":"vsi_answer","prompt_profile":"vsi_default"},"expected_count":5130,"family":"spatial_intelligence","legacy_environment":{"VSI_BATCH_SIZE":16,"VSI_MAX_MODEL_LEN":32768,"VSI_MAX_NEW_TOKENS":1024,"VSI_SETTING":"video128-16M-video-f128-fps2-min65536-maxnone-total16777216-vsibench_preprocessed_128f_16M"},"license":"See the upstream benchmark terms","media_paths":["media/spatial_intelligence/vsi/videos","media/spatial_intelligence/vsi/videos/test"],"preprocessing":{"fps":2,"max_frames":128,"video_min_pixels":65536,"video_total_pixels":16777216},"redistribution_authorized":true,"schema_version":1,"source_url":"https://huggingface.co/datasets/nyu-visionx/VSI-Bench","split":"test","task":"vsi"} diff --git a/code/docs/environment.md b/code/docs/environment.md new file mode 100644 index 0000000000000000000000000000000000000000..2c96e61336151cc7de684c32812a037b7198c2b5 --- /dev/null +++ b/code/docs/environment.md @@ -0,0 +1,60 @@ +

简体中文

+ +# Reproduce the OraRL Environment + +OraRL uses one reproducible software stack for policy optimization and +task-native evaluation. The released configuration was validated on NVIDIA +H20/Hopper GPUs with CUDA 12.9 and includes both runtimes in this repository. + +## Requirements + +- Linux with Conda available in `PATH` +- NVIDIA GPU with a recent driver +- CUDA 12.9-compatible hardware for the paper-matched environment + +CUDA 12.9 GA officially requires NVIDIA Linux driver 575.51.03 or newer. +Other NVIDIA GPUs may work, but the published setup was validated on H20. + +## Install the pinned stack + +```bash +bash scripts/create_conda_env.sh +conda activate orarl + +python scripts/check_environment.py \ + --require-gpu \ + --model /path/to/local/model +``` + +The installer pins Python 3.11, PyTorch 2.10.0+cu129, Transformers 5.5.4, +vLLM 0.19.1, FlashAttention 2.8.3, and the remaining packages in +`requirements-cu129.txt`. The final editable install exposes the `orarl-*` +commands and the bundled `verl` trainer package from this checkout; do not +install an unrelated `verl` release from PyPI over it. + +## Validate an evaluation-only node + +If a node will only run evaluation, skip the training-side checks: + +```bash +bash scripts/create_conda_env.sh --evaluation-only +conda activate orarl + +python scripts/check_environment.py --evaluation-only +``` + +## Verify the release checkout + +Run the release checks before allocating a long job: + +```bash +python scripts/check_release.py +python -m pytest -q +ruff check . +``` + +On a paper-matched H20 node, add `--require-gpu --require-h20` to +`scripts/check_environment.py`. Use the same source revision and environment on +every node of a distributed run. + +Continue with [Training](training.md) or [Evaluation](evaluation.md). diff --git a/code/docs/environment_zh.md b/code/docs/environment_zh.md new file mode 100644 index 0000000000000000000000000000000000000000..8f2ffa45fc1b407d729787d6156e5e38d493727c --- /dev/null +++ b/code/docs/environment_zh.md @@ -0,0 +1,57 @@ +

English

+ +# 复现 OraRL 环境 + +OraRL 使用同一套可复现软件栈完成策略优化与任务原生评测。发布配置在 NVIDIA +H20/Hopper 与 CUDA 12.9 上完成验证，本仓库同时包含两部分运行时。 + +## 环境要求 + +- `PATH` 中可用 Conda 的 Linux 系统 +- 驱动版本较新的 NVIDIA GPU +- 复现论文环境时使用兼容 CUDA 12.9 的硬件 + +CUDA 12.9 GA 官方要求 NVIDIA Linux 驱动不低于 575.51.03。其他 NVIDIA GPU +也可能运行，但发布配置仅在 H20 上完成验证。 + +## 安装固定版本软件栈 + +```bash +bash scripts/create_conda_env.sh +conda activate orarl + +python scripts/check_environment.py \ + --require-gpu \ + --model /path/to/local/model +``` + +脚本会固定 Python 3.11、PyTorch 2.10.0+cu129、Transformers 5.5.4、 +vLLM 0.19.1、FlashAttention 2.8.3，以及 `requirements-cu129.txt` 中的其余依赖。 +最后一步的可编辑安装会同时提供 `orarl-*` 命令和本仓库自带的 `verl` 训练包； +请勿再用 PyPI 上无关的 `verl` 版本覆盖它。 + +## 验证仅用于评测的节点 + +若某个节点只跑评测，可跳过训练侧检查： + +```bash +bash scripts/create_conda_env.sh --evaluation-only +conda activate orarl + +python scripts/check_environment.py --evaluation-only +``` + +## 验证发布源码 + +分配长任务前先执行： + +```bash +python scripts/check_release.py +python -m pytest -q +ruff check . +``` + +在与论文一致的 H20 节点上，可为 `scripts/check_environment.py` 增加 +`--require-gpu --require-h20`。多节点任务应在所有节点使用相同的源码版本和环境。 + +随后参阅[训练](training_zh.md)或[评测](evaluation_zh.md)。 diff --git a/code/docs/evaluation.md b/code/docs/evaluation.md new file mode 100644 index 0000000000000000000000000000000000000000..ee4d3f0efe618f9dbbb2cb6a5273eb78209cf144 --- /dev/null +++ b/code/docs/evaluation.md @@ -0,0 +1,121 @@ +

简体中文

+ +# Evaluate Video-ORA + +OraRL evaluates direct, task-native answers with canonical profiles for the +seven released task families. The corresponding runtime lives under +`eval/task/`; first +[validate the evaluation node](environment.md#validate-an-evaluation-only-node). + +## 1. Materialize model and data + +Install the Hugging Face CLI and download the released assets: + +```bash +python -m pip install -U huggingface_hub + +hf download OraRL/Video-ORA-9B \ + --local-dir "$PWD/models/Video-ORA-9B" + +hf download OraRL/OraRL-Data \ + --repo-type dataset \ + --include "OraRL-eval-data/**" \ + --local-dir "$PWD/OraRL-Data" +``` + +The complete evaluation release is large. Downloads are resumable, and +`OraRL-eval-data/assets.jsonl` is the authoritative file inventory. + +The local dataset layout is: + +```text +OraRL-Data/OraRL-eval-data/ +├── datasets.jsonl # benchmark, prompt, parser, metric, and sampling profiles +├── assets.jsonl # released-file inventory +├── annotations/ # canonical JSONL rows +└── media/ # raw images, videos, and subtitles +``` + +Derived preprocessing caches are intentionally excluded. Evaluators decode the +declared raw media when no compatible cache is present. + +## 2. Run the canonical paper profile + +Preview the resolved command first: + +```bash +orarl-eval \ + --model "$PWD/models/Video-ORA-9B" \ + --tasks paper \ + --dataset "$PWD/OraRL-Data/OraRL-eval-data" \ + --summary "$PWD/outputs/Video-ORA-9B/evaluation.json" +``` + +`orarl-eval` is a dry run by default. After checking the model, dataset, +evaluator, task profiles, and output paths, add `--run`. + +To validate the pipeline with a bounded smoke test: + +```bash +orarl-eval \ + --model "$PWD/models/Video-ORA-9B" \ + --tasks videomme \ + --dataset "$PWD/OraRL-Data/OraRL-eval-data" \ + --max-samples 8 \ + --summary "$PWD/outputs/Video-ORA-9B/videomme-smoke.json" \ + --run +``` + +Smoke scores only validate execution and must not be reported as benchmark +results. + +## 3. Compose a task suite + +`--tasks paper` selects all released tasks. `--tasks video_qa` selects the +seven Video QA benchmarks. Individual task names may be comma-separated. + +| Family | Task names | +| --- | --- | +| Video QA | `videomme`, `videommev2`, `mvbench`, `mmvu`, `videoholmes`, `longvideobench`, `mlvu` | +| Spatial intelligence | `vsi`, `mmsi`, `mindcube`, `revsi` | +| Temporal grounding | `temporal_grounding` | +| Spatial grounding | `spatial_grounding` | +| Tracking | `tracking` | +| Spatial-temporal grounding | `stvg` | +| Segmentation | `segmentation` | + +The canonical profiles in `datasets.jsonl` pin frame sampling, resolution, +prompts, parsers, and metrics. Changing them defines a different evaluation +setting. ReVSI is reported separately from the three-benchmark +spatial-intelligence average. + +## 4. Add segmentation post-processing + +Segmentation inference runs without SAM2 post-processing by default. Enabling +`--segmentation-run-sam2` additionally requires: + +- SAM2 weights (`SEGMENTATION_SAM2_CKPT`) +- the matching Hydra config (`SEGMENTATION_SAM2_CFG`) +- the official OneThinker `seg_post_sam2.py` + (`SEGMENTATION_POSTPROCESSOR_PATH`) +- the `sam2` Python package + +OraRL validates all three paths before launch. + +## 5. Preserve reportable outputs + +Each task evaluator writes its native summary, then `orarl-eval` creates the +requested aggregate JSON with requested, completed, and missing tasks, the +return code, and official metrics. Outputs are grouped by paper task family +under `outputs//`. + +For a reportable run, retain: + +- the OraRL source revision +- the model and data revisions +- the exact command and aggregate summary +- software versions and accelerator type/count +- every non-default profile or CLI override + +See [`../eval/README.md`](../eval/README.md) for the evaluator map, video +decoding backend, and checkpoint-format notes. diff --git a/code/docs/evaluation_zh.md b/code/docs/evaluation_zh.md new file mode 100644 index 0000000000000000000000000000000000000000..145efdb90d5287196cfba0f44904212674c6d1bd --- /dev/null +++ b/code/docs/evaluation_zh.md @@ -0,0 +1,116 @@ +

English

+ +# 评测 Video-ORA + +OraRL 使用七个发布任务族的规范配置评测任务原生答案，对应运行时位于 +`eval/task/`。运行前请先 +[验证评测节点](environment_zh.md#验证仅用于评测的节点)。 + +## 1. 准备模型和数据 + +安装 Hugging Face CLI 并下载发布文件： + +```bash +python -m pip install -U huggingface_hub + +hf download OraRL/Video-ORA-9B \ + --local-dir "$PWD/models/Video-ORA-9B" + +hf download OraRL/OraRL-Data \ + --repo-type dataset \ + --include "OraRL-eval-data/**" \ + --local-dir "$PWD/OraRL-Data" +``` + +完整评测数据体积较大，下载支持断点续传； +`OraRL-eval-data/assets.jsonl` 是权威文件清单。 + +本地数据结构为： + +```text +OraRL-Data/OraRL-eval-data/ +├── datasets.jsonl # benchmark、prompt、parser、metric 与采样配置 +├── assets.jsonl # 发布文件清单 +├── annotations/ # 规范化 JSONL 标注 +└── media/ # 原始图像、视频和字幕 +``` + +发布数据有意排除衍生预处理缓存；不存在兼容缓存时，评测器会直接解码清单中声明的 +原始媒体。 + +## 2. 运行论文规范配置 + +先预览解析后的命令： + +```bash +orarl-eval \ + --model "$PWD/models/Video-ORA-9B" \ + --tasks paper \ + --dataset "$PWD/OraRL-Data/OraRL-eval-data" \ + --summary "$PWD/outputs/Video-ORA-9B/evaluation.json" +``` + +`orarl-eval` 默认为 dry run。检查模型、数据、评测器、任务配置和输出路径后， +增加 `--run`。 + +可先运行限制样本数的 smoke test： + +```bash +orarl-eval \ + --model "$PWD/models/Video-ORA-9B" \ + --tasks videomme \ + --dataset "$PWD/OraRL-Data/OraRL-eval-data" \ + --max-samples 8 \ + --summary "$PWD/outputs/Video-ORA-9B/videomme-smoke.json" \ + --run +``` + +Smoke test 分数仅用于验证流程，不应作为 benchmark 结果报告。 + +## 3. 组合任务集 + +`--tasks paper` 选择所有发布任务，`--tasks video_qa` 选择 7 个视频问答 +benchmark；多个单项任务可用逗号分隔。 + +| 任务族 | 任务名 | +| --- | --- | +| 视频问答 | `videomme`, `videommev2`, `mvbench`, `mmvu`, `videoholmes`, `longvideobench`, `mlvu` | +| 空间智能 | `vsi`, `mmsi`, `mindcube`, `revsi` | +| 时序定位 | `temporal_grounding` | +| 空间定位 | `spatial_grounding` | +| 跟踪 | `tracking` | +| 时空定位 | `stvg` | +| 分割 | `segmentation` | + +`datasets.jsonl` 中的规范配置固定了帧采样、分辨率、prompt、parser 和 metric； +修改这些值即代表不同的评测设置。ReVSI 单独报告，不计入三个 benchmark 的空间 +智能平均分。 + +## 4. 添加分割后处理 + +分割推理默认不执行 SAM2 后处理。启用 `--segmentation-run-sam2` 时还需要： + +- SAM2 权重（`SEGMENTATION_SAM2_CKPT`） +- 匹配的 Hydra 配置（`SEGMENTATION_SAM2_CFG`） +- OneThinker 官方 `seg_post_sam2.py` + （`SEGMENTATION_POSTPROCESSOR_PATH`） +- `sam2` Python 包 + +OraRL 会在启动前检查三个路径。 + +## 5. 保存可报告的输出 + +每个任务评测器先写入原生 summary，随后 `orarl-eval` 生成指定的聚合 JSON， +其中包含请求、完成和缺失的任务、返回码以及官方 metric。输出按论文任务族组织在 +`outputs//` 下。 + +正式报告结果时请保留： + +- OraRL 源码版本 +- 模型和数据版本 +- 完整命令与聚合 summary +- 软件版本和加速卡类型/数量 +- 所有非默认配置或命令行覆盖项 + +评测器映射、视频解码后端和 checkpoint 格式说明见 +[`../eval/README.md`](../eval/README.md)。 diff --git a/code/docs/training.md b/code/docs/training.md new file mode 100644 index 0000000000000000000000000000000000000000..c4441baae6a13115dd415e36c580167fa8b5bb24 --- /dev/null +++ b/code/docs/training.md @@ -0,0 +1,162 @@ +

简体中文

+ +# Train with OraRL + +This workflow turns licensed source records into an auditable GRPO or OraRL +run. Complete the +[pinned environment installation](environment.md#install-the-pinned-stack) +first. + +`orarl-train` is the public launch boundary: it validates paths and overrides, +resolves one of the released recipes, and then starts the trainer bundled with +this checkout. No second runtime repository is required. + +## 1. Build the training manifest + +The prepared training-data release will be uploaded separately and is not +included in this Git repository yet. Until it is available, obtain each +annotation and media source under its upstream license, then copy the example +manifest: + +```bash +cp configs/data_sources.example.yaml ./data_sources.local.yaml +``` + +Replace every `../local_data` placeholder with a licensed local path. Relative +paths are resolved from the manifest location. Each source declares its input +annotations, task, family, quota, media root, and optional license/source-page +metadata. + +The public example reproduces the 100,032-row paper mixture: + +| Family | Rows | +| --- | ---: | +| Temporal grounding | 20,096 | +| Tracking | 13,952 | +| Segmentation | 12,032 | +| Spatial grounding | 7,040 | +| Spatial-temporal grounding | 9,536 | +| Video QA | 20,288 | +| Spatial intelligence | 17,088 | + +Build the deterministic train/canary split: + +```bash +orarl-prepare \ + --config ./data_sources.local.yaml \ + --output ./prepared/train.jsonl \ + --require-media +``` + +This writes: + +```text +prepared/ +├── train.jsonl +├── train.canary.jsonl +└── train.manifest.json +``` + +The builder validates local media, normalizes task records, enforces source +quotas and per-media caps, removes duplicate prompt identities, excludes +supplied benchmark identities, and keeps train/canary media disjoint. The audit +manifest records counts, shortfalls, source metadata, and SHA-256 checksums. + +## 2. Choose GRPO or OraRL + +| Recipe | Model scale | Method | +| --- | --- | --- | +| `grpo_4b.yaml` | 4B | GRPO baseline | +| `grpo_9b.yaml` | 9B | GRPO baseline | +| `orarl_4b.yaml` | 4B | OraRL | +| `orarl_9b.yaml` | 9B | OraRL | + +The paper defaults use 64 prompts per rollout/update batch and eight policy +samples per prompt. The 100,032-row mixture therefore runs for 1,563 steps in +one epoch. + +## 3. Preview, then launch + +Set paths to a compatible local base model and the prepared data: + +```bash +MODEL_DIR=/path/to/local/base-model +OUTPUT_DIR="$PWD/runs/orarl-4b" + +orarl-train \ + --config orarl_4b.yaml \ + --model "$MODEL_DIR" \ + --train-data "$PWD/prepared/train.jsonl" \ + --val-data "$PWD/prepared/train.canary.jsonl" \ + --output "$OUTPUT_DIR" \ + --nodes 1 \ + --gpus-per-node 8 +``` + +The command is a dry run by default. Inspect the resolved invocation, then add +`--run` to start training. Use `--set KEY=VALUE` for an explicit config +override; retain all overrides with the run artifacts. + +For a one-update smoke test: + +```bash +orarl-train \ + --config orarl_4b.yaml \ + --model "$MODEL_DIR" \ + --train-data "$PWD/prepared/train.jsonl" \ + --val-data "$PWD/prepared/train.canary.jsonl" \ + --output "$OUTPUT_DIR" \ + --nodes 1 \ + --gpus-per-node 8 \ + --set trainer.max_steps=1 \ + --run +``` + +Repeat with `grpo_4b.yaml` and a different output directory to validate the +baseline. Use the matching model and recipe for 9B runs. + +## 4. Scale across nodes + +All nodes must see the same source, model, data, and output paths: + +```bash +HOSTS=node-a,node-b \ + bash scripts/launch_multinode.sh \ + --gpus-per-node 8 \ + -- \ + --config "$PWD/configs/orarl_4b.yaml" \ + --model "$MODEL_DIR" \ + --train-data "$PWD/prepared/train.jsonl" \ + --val-data "$PWD/prepared/train.canary.jsonl" \ + --output "$OUTPUT_DIR" +``` + +The launcher is also a dry run unless its own `--run` is supplied before the +`--` separator. SSH host-key checking is strict by default. + +## 5. Accept a run + +`scripts/smoke_training.sh` runs one GRPO update and one OraRL update with small +batches and saves a checkpoint for each: + +```bash +bash scripts/smoke_training.sh \ + --model "$MODEL_DIR" \ + --train-data "$PWD/prepared/train.jsonl" \ + --val-data "$PWD/prepared/train.canary.jsonl" \ + --size 4b \ + --gpus-per-node 8 +``` + +Add `--dry-run` to inspect the resolved commands without allocating GPUs. + +Before a full experiment, verify that: + +- GRPO and OraRL each complete one update with finite rewards, losses, gradient + norms, and selection metrics. +- A checkpoint can be saved, reloaded, and used for another update. +- Multi-node runs form the expected Ray cluster and complete one update. +- The source revision, config, command, data-manifest checksum, environment + versions, accelerator type, and all overrides are retained. + +Use [Evaluation](evaluation.md) to evaluate an exported checkpoint. diff --git a/code/docs/training_zh.md b/code/docs/training_zh.md new file mode 100644 index 0000000000000000000000000000000000000000..103bf2bb94f4e56bb395cb2faeaece4d7b9cc5ff --- /dev/null +++ b/code/docs/training_zh.md @@ -0,0 +1,151 @@ +

English

+ +# 使用 OraRL 训练 + +本流程将已获许可的源数据整理为可审计的 GRPO 或 OraRL 训练任务。请先 +[安装固定版本软件栈](environment_zh.md#安装固定版本软件栈)。 + +`orarl-train` 是公开的训练入口：它会检查路径与覆盖项、解析发布配置，再启动 +本仓库自带的训练器，无需准备第二份运行时源码。 + +## 1. 构建训练清单 + +整理后的训练数据将稍后单独上传，目前尚未包含在本 Git 仓库中。在发布完成前， +请按上游许可证自行获取标注和媒体文件，然后复制示例清单： + +```bash +cp configs/data_sources.example.yaml ./data_sources.local.yaml +``` + +将所有 `../local_data` 占位符替换为已获许可的本地路径。相对路径以清单文件所在 +目录为基准。每个数据源需要声明标注文件、任务、任务族、配额和媒体根目录，并可 +记录许可证与来源页面。 + +公开示例对应论文使用的 100,032 条训练混合： + +| 任务族 | 条数 | +| --- | ---: | +| 时序定位 | 20,096 | +| 跟踪 | 13,952 | +| 分割 | 12,032 | +| 空间定位 | 7,040 | +| 时空定位 | 9,536 | +| 视频问答 | 20,288 | +| 空间智能 | 17,088 | + +构建确定性的训练集和 canary 集： + +```bash +orarl-prepare \ + --config ./data_sources.local.yaml \ + --output ./prepared/train.jsonl \ + --require-media +``` + +输出结构为： + +```text +prepared/ +├── train.jsonl +├── train.canary.jsonl +└── train.manifest.json +``` + +构建器会检查本地媒体、规范化任务记录、执行数据源配额和单媒体上限、去除重复 +prompt、排除给定评测集中的样本，并保证训练集与 canary 集媒体互斥。审计清单会 +记录计数、缺额、来源信息和 SHA-256 校验值。 + +## 2. 选择 GRPO 或 OraRL + +| 配置 | 模型规模 | 方法 | +| --- | --- | --- | +| `grpo_4b.yaml` | 4B | GRPO baseline | +| `grpo_9b.yaml` | 9B | GRPO baseline | +| `orarl_4b.yaml` | 4B | OraRL | +| `orarl_9b.yaml` | 9B | OraRL | + +论文默认每个 rollout/update batch 使用 64 个 prompt，每个 prompt 采样 8 个策略 +回复，因此 100,032 条混合数据在一个 epoch 中对应 1,563 步。 + +## 3. 预览并启动 + +设置兼容的本地基础模型和已准备的数据路径： + +```bash +MODEL_DIR=/path/to/local/base-model +OUTPUT_DIR="$PWD/runs/orarl-4b" + +orarl-train \ + --config orarl_4b.yaml \ + --model "$MODEL_DIR" \ + --train-data "$PWD/prepared/train.jsonl" \ + --val-data "$PWD/prepared/train.canary.jsonl" \ + --output "$OUTPUT_DIR" \ + --nodes 1 \ + --gpus-per-node 8 +``` + +命令默认只执行 dry run。检查解析后的调用后，增加 `--run` 开始训练。可通过 +`--set KEY=VALUE` 显式覆盖配置，并应随运行产物保留全部覆盖项。 + +单步 smoke test： + +```bash +orarl-train \ + --config orarl_4b.yaml \ + --model "$MODEL_DIR" \ + --train-data "$PWD/prepared/train.jsonl" \ + --val-data "$PWD/prepared/train.canary.jsonl" \ + --output "$OUTPUT_DIR" \ + --nodes 1 \ + --gpus-per-node 8 \ + --set trainer.max_steps=1 \ + --run +``` + +再使用 `grpo_4b.yaml` 和独立输出目录验证 baseline。9B 任务需使用匹配的模型与 +配置。 + +## 4. 扩展到多节点 + +所有节点必须看到相同的源码、模型、数据和输出路径： + +```bash +HOSTS=node-a,node-b \ + bash scripts/launch_multinode.sh \ + --gpus-per-node 8 \ + -- \ + --config "$PWD/configs/orarl_4b.yaml" \ + --model "$MODEL_DIR" \ + --train-data "$PWD/prepared/train.jsonl" \ + --val-data "$PWD/prepared/train.canary.jsonl" \ + --output "$OUTPUT_DIR" +``` + +启动器同样默认为 dry run；需要在 `--` 分隔符前加入其自身的 `--run`。SSH +主机密钥检查默认使用严格模式。 + +## 5. 验收一次运行 + +`scripts/smoke_training.sh` 会以较小的 batch 分别跑一步 GRPO 和一步 OraRL， +并各保存一个 checkpoint： + +```bash +bash scripts/smoke_training.sh \ + --model "$MODEL_DIR" \ + --train-data "$PWD/prepared/train.jsonl" \ + --val-data "$PWD/prepared/train.canary.jsonl" \ + --size 4b \ + --gpus-per-node 8 +``` + +加上 `--dry-run` 可在不占用 GPU 的情况下检查解析后的命令。 + +正式实验前确认： + +- GRPO 和 OraRL 均可完成一步更新，reward、loss、梯度范数和选择指标均为有限值。 +- checkpoint 可保存、重新加载并继续完成一步更新。 +- 多节点任务能组成预期的 Ray 集群并完成一步更新。 +- 保存源码版本、配置、命令、数据清单校验值、环境版本、加速卡类型和所有覆盖项。 + +使用[评测文档](evaluation_zh.md)评估导出的 checkpoint。 diff --git a/code/environment.yml b/code/environment.yml new file mode 100644 index 0000000000000000000000000000000000000000..15207b477701ce82aa1cdc5e49b048f1c8dd6c0c --- /dev/null +++ b/code/environment.yml @@ -0,0 +1,18 @@ +# Base Conda layer. Use scripts/create_conda_env.sh so CUDA wheels and editable +# packages are installed in the required order. +name: orarl + +channels: + - nvidia + - conda-forge + - defaults + +dependencies: + - python=3.11 + - pip + - setuptools>=77 + - wheel + - packaging + - ninja + - ffmpeg=7 + - cuda-toolkit=12.9 diff --git a/code/eval/README.md b/code/eval/README.md new file mode 100644 index 0000000000000000000000000000000000000000..65a36c4dab87a4d205b6cc6dcb3190592c985d80 --- /dev/null +++ b/code/eval/README.md @@ -0,0 +1,72 @@ +# Evaluation runtime + +`eval/task/` holds the evaluators that produce every number in the paper. They +ship with this repository, so a single clone can reproduce the full suite. +`orarl-eval` is the supported entry point: it resolves the canonical dataset +layout into per-task environment variables and then runs `eval/task/eval.sh`. + +```bash +orarl-eval \ + --model /path/to/exported-model \ + --tasks paper \ + --dataset OraRL/OraRL-Data \ + --summary ./evaluation.json \ + --run +``` + +Without `--run` the command prints the resolved plan and exits, which is the +fastest way to confirm data and model paths before allocating GPUs. +`eval/task/eval.sh` can also be called directly when you want to bypass the +canonical dataset resolution and set the per-task variables yourself. + +## Task map + +| Family | Benchmarks | Entry point | +| --- | --- | --- | +| Video QA | VideoMME, VideoMME-v2, MVBench, MMVU, Video-Holmes, LongVideoBench, MLVU | `eval_vllm.py` | +| Spatial intelligence | VSI-Bench, MindCube, MMSI-Bench, ReVSI | `eval_vllm.py`, `mmsi/eval_mmsi_transformers.py`, `revsi/eval_revsi_vllm.py` | +| Temporal grounding | TimeLens (Charades, ActivityNet, QVHighlights) | `temporal_grounding/eval_timelens_hf.py` | +| Spatial grounding | RefCOCO, RefCOCO+, RefCOCOg | `spatial_grounding/eval_refcoco_vllm.py` | +| Tracking | GOT-10k | `tracking/eval_tracking_vllm.py` | +| Spatio-temporal grounding | STVG | `spatial_temporal_grounding/eval_stvg_vllm.py` | +| Segmentation | RefCOCO series, MeViS, ReasonVOS | `segmentation/eval_seg_vllm.py` plus `segmentation/post_sam2.py` | + +`eval_prompt.py` is the single source of truth for prompts, and +`canonical_data.py` adapts the canonical `ORARL_EVAL_*` layout for every +evaluator. MMSI-Bench runs through Transformers rather than vLLM; the other +families run through vLLM. + +## Assets you must obtain separately + +Annotations and media come from the `OraRL/OraRL-Data` dataset repository, and +model weights come from the released checkpoints. Neither is vendored here. + +Segmentation additionally needs three inputs, all supplied by you under their +upstream licenses: + +- SAM2 weights and the matching Hydra config, passed as + `SEGMENTATION_SAM2_CKPT` and `SEGMENTATION_SAM2_CFG`. +- The official OneThinker `seg_post_sam2.py`, passed as + `SEGMENTATION_POSTPROCESSOR_PATH`. `segmentation/post_sam2.py` is a thin + wrapper that injects paths into it so mask metrics stay identical to the + upstream implementation. +- The `sam2` Python package. + +`orarl-eval` refuses to start segmentation with `--segmentation-run-sam2` +unless all three paths exist. + +## Video decoding + +Video evaluators pin the decord backend through `FORCE_QWENVL_VIDEO_READER`. +`eval/task/qwenvl_decord_patch.py` replaces `qwen_vl_utils.fetch_video` so the +backend is honoured across `qwen_vl_utils` releases that otherwise hard-code +torchvision. The backend is not cosmetic: switching MeViS and ReasonVOS from +torchcodec to decord moved MeViS J&F from 56.7 to 60.6. Set +`SEGMENTATION_VIDEO_READER` to `torchcodec` or `torchvision` only when +deliberately measuring that difference. + +## Checkpoint format + +Exported Hugging Face checkpoints run as-is. Pass `--force-merge` when the model +path is a sharded FSDP actor directory; `eval/task/eval.sh` then merges it with +`scripts/model_merger.py` before inference. diff --git a/code/eval/task/canonical_data.py b/code/eval/task/canonical_data.py new file mode 100644 index 0000000000000000000000000000000000000000..63f66ac4f25bbc9e864d05d05f991cf04571336a --- /dev/null +++ b/code/eval/task/canonical_data.py @@ -0,0 +1,450 @@ +"""Dependency-light compatibility adapter for canonical OraRL evaluation rows.""" + +from __future__ import annotations + +import copy +import json +import os +import re +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + + +_CANONICAL_FIELDS = { + "schema_version", + "eval_task", + "sample_id", + "benchmark", + "split", + "problem", + "answer", + "images", + "videos", + "problem_type", + "source", +} +_MEDIA_PATH_KEYS = ( + "path", + "video", + "image", + "video_path", + "image_path", + "file_name", +) +_PATH_KEY_RE = re.compile( + r"(?:^|_)(?:artifact|artifacts|cfg|checkpoint|ckpt|dir|directory|file|" + r"files|image|images|media|path|paths|prefix|root|subtitle|" + r"subtitles|tensor|tensors|video|videos)$" +) +_GROUND_TRUTH_KEYS = ( + "bbox", + "bboxes", + "boxes", + "coordinates", + "duration", + "end", + "frame", + "frame_id", + "frame_ids", + "gt_bbox", + "mask", + "masks", + "negative_points", + "normalized_solution", + "object", + "positive_points", + "query", + "segment", + "segmentation", + "span", + "spans", + "start", + "target", + "time", + "timestamp", + "timestamps", + "trajectory", +) +_ASSET_DIRECTORIES = frozenset({"artifacts", "media"}) + + +class CanonicalDataError(ValueError): + """Raised when a canonical row cannot be safely adapted.""" + + +def is_canonical_row(record: Any) -> bool: + """Return whether a mapping uses the canonical evaluation-row contract.""" + + return ( + isinstance(record, Mapping) + and record.get("schema_version") == 1 + and _CANONICAL_FIELDS.issubset(record) + ) + + +def _configured_root() -> Path | None: + value = os.environ.get("ORARL_EVAL_DATA_ROOT", "").strip() + if value: + return Path(value).expanduser().resolve() + manifest = os.environ.get("ORARL_EVAL_DATASETS_JSONL", "").strip() + if manifest: + return Path(manifest).expanduser().resolve().parent + return None + + +def _configured_asset_root() -> Path | None: + value = os.environ.get("ORARL_EVAL_ASSET_ROOT", "").strip() + return Path(value).expanduser().resolve() if value else None + + +def _infer_root(input_path: Path) -> Path | None: + configured = _configured_root() + if configured is not None: + return configured + for candidate in (input_path.parent, *input_path.parents): + if (candidate / "datasets.jsonl").is_file(): + return candidate.resolve() + return None + + +def canonical_dataset_record( + eval_task: str, + split: str, + root: str | os.PathLike[str] | None = None, +) -> dict[str, Any] | None: + """Return one canonical ``datasets.jsonl`` record for a task split.""" + + repository_root = ( + Path(root).expanduser().resolve() if root is not None else _configured_root() + ) + if repository_root is None: + return None + manifest = repository_root / "datasets.jsonl" + if not manifest.is_file(): + return None + + normalized_split = str(split).strip().casefold().replace("-", "_") + matches: list[dict[str, Any]] = [] + try: + with manifest.open("r", encoding="utf-8") as handle: + for line_number, line in enumerate(handle, start=1): + if not line.strip(): + continue + try: + record = json.loads(line) + except json.JSONDecodeError as error: + raise CanonicalDataError( + f"{manifest}:{line_number}: invalid JSON: {error}" + ) from error + if not isinstance(record, Mapping): + raise CanonicalDataError( + f"{manifest}:{line_number}: dataset record must be an object" + ) + record_task = str(record.get("task") or "").casefold() + record_split = str(record.get("split") or "").casefold() + if record_task == str(eval_task).casefold() and record_split == normalized_split: + matches.append(copy.deepcopy(dict(record))) + except OSError as error: + raise CanonicalDataError(f"cannot read canonical dataset manifest: {error}") from error + if len(matches) > 1: + raise CanonicalDataError( + f"duplicate canonical dataset profile for {eval_task}/{normalized_split}" + ) + return matches[0] if matches else None + + +def resolve_repository_path( + value: str, + root: str | os.PathLike[str], + asset_root: str | os.PathLike[str] | None = None, +) -> str: + """Resolve one repository-relative canonical path without allowing escape.""" + + raw = str(value).strip() + if not raw: + raise CanonicalDataError("canonical repository path must be nonempty") + repository_root = Path(root).expanduser().resolve() + configured_asset_root = ( + Path(asset_root).expanduser().resolve() + if asset_root is not None + else _configured_asset_root() + ) + candidate = Path(raw).expanduser() + selected_root = repository_root + if ( + not candidate.is_absolute() + and candidate.parts + and candidate.parts[0].casefold() in _ASSET_DIRECTORIES + and configured_asset_root is not None + ): + selected_root = configured_asset_root + resolved = ( + candidate.resolve() + if candidate.is_absolute() + else (selected_root / candidate).resolve() + ) + allowed_roots = [repository_root] + if configured_asset_root is not None: + allowed_roots.append(configured_asset_root) + try: + next(root for root in allowed_roots if resolved.is_relative_to(root)) + except StopIteration as error: + raise CanonicalDataError( + f"canonical path resolves outside configured evaluation roots: {raw}" + ) from error + return os.path.normpath(str(resolved)) + + +def repository_relative_output_path( + value: str, + root: str | os.PathLike[str], +) -> str: + """Return a ``./``-prefixed path when an asset is contained by ``root``.""" + + raw = str(value).strip() + if not raw or not str(root).strip(): + return raw + candidate = Path(raw).expanduser() + if not candidate.is_absolute(): + return raw + repository_root = Path(root).expanduser().resolve() + resolved = candidate.resolve() + try: + relative = resolved.relative_to(repository_root) + except ValueError: + return raw + return f"./{relative.as_posix()}" + + +def _is_path_key(value: object) -> bool: + return isinstance(value, str) and bool(_PATH_KEY_RE.search(value.casefold())) + + +def _resolve_media_entry(value: Any, root: Path, asset_root: Path | None) -> Any: + if isinstance(value, str): + return resolve_repository_path(value, root, asset_root) + if not isinstance(value, Mapping): + return copy.deepcopy(value) + resolved = copy.deepcopy(dict(value)) + for key in _MEDIA_PATH_KEYS: + path = resolved.get(key) + if isinstance(path, str) and path.strip(): + resolved[key] = resolve_repository_path(path, root, asset_root) + return resolved + + +def _resolve_declared_paths( + value: Any, + root: Path, + asset_root: Path | None, + path_context: bool = False, +) -> Any: + if isinstance(value, str): + return ( + resolve_repository_path(value, root, asset_root) + if path_context + else value + ) + if isinstance(value, Mapping): + return { + key: _resolve_declared_paths( + item, + root, + asset_root, + path_context=path_context or _is_path_key(key), + ) + for key, item in value.items() + } + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + return [ + _resolve_declared_paths( + item, + root, + asset_root, + path_context=path_context, + ) + for item in value + ] + return copy.deepcopy(value) + + +def _media_path(value: Any) -> str: + if isinstance(value, str): + return value + if isinstance(value, Mapping): + for key in _MEDIA_PATH_KEYS: + candidate = value.get(key) + if isinstance(candidate, str) and candidate: + return candidate + return "" + + +def _declared_path_values(value: Any, path_context: bool = False) -> list[str]: + if isinstance(value, str): + return [value] if path_context else [] + if isinstance(value, Mapping): + paths: list[str] = [] + for key, item in value.items(): + paths.extend( + _declared_path_values( + item, + path_context=path_context or _is_path_key(key), + ) + ) + return paths + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + paths = [] + for item in value: + paths.extend(_declared_path_values(item, path_context=path_context)) + return paths + return [] + + +def adapt_canonical_row( + record: Mapping[str, Any], + root: str | os.PathLike[str] | None = None, + asset_root: str | os.PathLike[str] | None = None, +) -> dict[str, Any]: + """Resolve canonical paths and add non-destructive legacy aliases.""" + + if not is_canonical_row(record): + return copy.deepcopy(dict(record)) + repository_root = ( + Path(root).expanduser().resolve() if root is not None else _configured_root() + ) + if repository_root is None: + raise CanonicalDataError( + "canonical rows require ORARL_EVAL_DATA_ROOT or an explicit root" + ) + resolved_asset_root = ( + Path(asset_root).expanduser().resolve() + if asset_root is not None + else _configured_asset_root() + ) + + adapted = copy.deepcopy(dict(record)) + for field in ("images", "videos", "subtitles"): + values = adapted.get(field) + if isinstance(values, list): + adapted[field] = [ + _resolve_media_entry(value, repository_root, resolved_asset_root) + for value in values + ] + for field in ("preprocessed", "task_payload", "metadata", "evaluation"): + value = adapted.get(field) + if isinstance(value, Mapping): + adapted[field] = _resolve_declared_paths( + value, + repository_root, + resolved_asset_root, + ) + + image_paths = [ + path for value in adapted.get("images", []) if (path := _media_path(value)) + ] + video_paths = [ + path for value in adapted.get("videos", []) if (path := _media_path(value)) + ] + subtitle_paths = [ + path + for value in adapted.get("subtitles", []) + if (path := _media_path(value)) + ] + + adapted.setdefault("id", adapted["sample_id"]) + adapted.setdefault("problem_id", adapted["sample_id"]) + adapted.setdefault("question_id", adapted["sample_id"]) + adapted.setdefault("question", adapted["problem"]) + adapted.setdefault("prompt", adapted["problem"]) + adapted.setdefault("ground_truth", adapted["answer"]) + adapted.setdefault("solution", adapted["answer"]) + adapted.setdefault("dataset", adapted["benchmark"]) + adapted.setdefault("options", copy.deepcopy(adapted.get("choices", []))) + adapted.setdefault("image_list", image_paths) + if image_paths: + adapted.setdefault("image", image_paths[0]) + adapted.setdefault("image_path", image_paths[0]) + if video_paths: + adapted.setdefault("video", video_paths[0]) + adapted.setdefault("video_path", video_paths[0]) + if video_paths or image_paths: + adapted.setdefault("path", (video_paths or image_paths)[0]) + if subtitle_paths: + adapted.setdefault("subtitle", subtitle_paths[0]) + adapted.setdefault("subtitle_path", subtitle_paths[0]) + + metadata = adapted.get("metadata") + if isinstance(metadata, Mapping): + for key, value in metadata.items(): + adapted.setdefault(str(key), copy.deepcopy(value)) + payload = adapted.get("task_payload") + if isinstance(payload, Mapping): + for key, value in payload.items(): + adapted.setdefault(str(key), copy.deepcopy(value)) + if isinstance(adapted.get("answer"), Mapping): + for key in _GROUND_TRUTH_KEYS: + if key in adapted["answer"]: + adapted.setdefault(key, copy.deepcopy(adapted["answer"][key])) + + preprocessed = adapted.get("preprocessed") + if isinstance(preprocessed, Mapping): + for key, value in preprocessed.items(): + adapted.setdefault(str(key), copy.deepcopy(value)) + paths = _declared_path_values(preprocessed) + if paths: + adapted.setdefault("preprocessed_video", paths[0]) + adapted.setdefault("preprocessed_video_path", paths[0]) + + if video_paths: + adapted.setdefault("data_type", "video") + elif image_paths: + adapted.setdefault("data_type", "image") + adapted.setdefault("question_type", adapted["problem_type"]) + adapted.setdefault("task_type", adapted["problem_type"]) + return adapted + + +def load_json_records( + path: str | os.PathLike[str], + *, + root: str | os.PathLike[str] | None = None, + asset_root: str | os.PathLike[str] | None = None, +) -> list[dict[str, Any]]: + """Read JSON/JSONL and adapt only rows that use the canonical contract.""" + + input_path = Path(path) + text = input_path.read_text(encoding="utf-8") + if not text.strip(): + records: list[Any] = [] + else: + try: + payload = json.loads(text) + except json.JSONDecodeError: + records = [json.loads(line) for line in text.splitlines() if line.strip()] + else: + records = payload if isinstance(payload, list) else [payload] + if not all(isinstance(record, Mapping) for record in records): + raise CanonicalDataError(f"{input_path}: every record must be a JSON object") + + repository_root = ( + Path(root).expanduser().resolve() + if root is not None + else _infer_root(input_path) + ) + return [ + adapt_canonical_row(record, repository_root, asset_root) + if is_canonical_row(record) + else copy.deepcopy(dict(record)) + for record in records + ] + + +__all__ = [ + "CanonicalDataError", + "adapt_canonical_row", + "canonical_dataset_record", + "is_canonical_row", + "load_json_records", + "resolve_repository_path", +] diff --git a/code/eval/task/eval.sh b/code/eval/task/eval.sh new file mode 100644 index 0000000000000000000000000000000000000000..fa388c1e1c6f48ff534126fa65bbcce5b3d23e18 --- /dev/null +++ b/code/eval/task/eval.sh @@ -0,0 +1,1018 @@ +#!/bin/bash +# Thin shell entry for evaluation. +# +# Supported TASKS: videomme,videommev2,videommmu,mmvu,mvbench,videoholmes,longvideobench,lvbench,mlvu,vsi,mmsi,mindcube,revsi,spatial_grounding,tracking,stvg,temporal_grounding,segmentation,all +# Common env overrides: +# MODEL_PATH=/path/to/model_or_verl_actor +# TASKS=videomme,videommev2,vsi +# GPUS=0,1,2,3,4,5,6,7 +# TP_SIZE=1 +# Prefer explicit CLI args for new runs, which avoids stale env variables: +# bash eval/task/eval.sh --model /path/to/model --tasks videomme,videommev2,mvbench --gpus 0,1,2,3,4,5,6,7 --force-merge +# +# MODEL must be provided explicitly with --model. + +# # —— Video QA / MC —— +# bash eval/task/eval.sh --model $MODEL --tasks mvbench --gpus $GPUS --force-merge +# bash eval/task/eval.sh --model $MODEL --tasks mmvu --gpus $GPUS --force-merge +# bash eval/task/eval.sh --model $MODEL --tasks videomme --gpus $GPUS --force-merge +# bash eval/task/eval.sh --model $MODEL --tasks videoholmes --gpus $GPUS --force-merge +# bash eval/task/eval.sh --model $MODEL --tasks longvideobench --gpus $GPUS --force-merge +# bash eval/task/eval.sh --model $MODEL --tasks mlvu --gpus $GPUS --force-merge + +# # —— Spatial intelligence (MC) —— +# bash eval/task/eval.sh --model $MODEL --tasks vsi --gpus $GPUS --force-merge +# bash eval/task/eval.sh --model $MODEL --tasks mmsi --gpus $GPUS --force-merge +# bash eval/task/eval.sh --model $MODEL --tasks mindcube --gpus $GPUS --force-merge +# bash eval/task/eval.sh --model $MODEL --tasks revsi --gpus $GPUS --force-merge + +# # —— Grounding / tracking / stvg / temporal —— +# bash eval/task/eval.sh --model $MODEL --tasks spatial_grounding --gpus $GPUS --force-merge +# bash eval/task/eval.sh --model $MODEL --tasks tracking --gpus $GPUS --force-merge +# bash eval/task/eval.sh --model $MODEL --tasks stvg --gpus $GPUS --force-merge +# charades（默认） +# TIMELENS_DATASETS=charades-timelens TIMELENS_FPS=4 bash eval/task/eval.sh --model $MODEL --tasks temporal_grounding --gpus $GPUS --force-merge +# TIMELENS_DATASETS=activitynet-timelens TIMELENS_FPS=2 bash eval/task/eval.sh --model $MODEL --tasks temporal_grounding --gpus $GPUS --force-merge +# TIMELENS_DATASETS=qvhighlights-timelens TIMELENS_FPS=2 bash eval/task/eval.sh --model $MODEL --tasks temporal_grounding --gpus $GPUS --force-merge + + + +# # —— Segmentation（需要 SAM2 + OraRL 评测环境）—— +# EVAL_CONDA_ENV=orarl SEGMENTATION_RUN_SAM2=true \ +# bash eval/task/eval.sh --model $MODEL --tasks segmentation --gpus $GPUS --force-merge + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)" +DEFAULT_GPUS="0,1,2,3,4,5,6,7" + +usage() { + cat <<'EOF' +Usage: + bash eval/task/eval.sh [options] + +Options: + --model PATH Model or checkpoint path. + --tasks LIST Comma-separated task list, e.g. videomme,videommev2,mvbench,mlvu. + --gpus LIST Comma-separated GPU ids, e.g. 0,1,2,3,4,5,6,7. + --tp-size N Tensor parallel size. + --shard-mode MODE Sharding mode passed to vLLM task runner. + --video-cache-size N Video cache size. + --videomme-max-new-tokens N + Max new tokens for VideoMME generation. + --force-merge Force merging FSDP actor checkpoints. + --skip-merge Reuse existing merged HF model for actor checkpoints. + --merged-model PATH Explicit merged HF output/input path for actor checkpoints. + --base-model PATH Base model path used when merging actor checkpoints. + --env NAME Conda env to activate. Default: keep current env. + --segmentation-run-sam2 + Enable SAM2 post-processing for segmentation. + --help Show this help. + +Core launch settings intentionally ignore inherited environment variables to +avoid stale MODEL_PATH/TASKS/GPUS/etc. Use explicit CLI options instead. +EOF +} + +# Avoid stale shell environment contaminating evaluation identity. Task-specific +# tuning envs below are still supported, but core launch identity is CLI/defaults. +unset MODEL_PATH TASKS GPUS TP_SIZE SHARD_MODE VIDEO_CACHE_SIZE +unset FORCE_MERGE SKIP_MERGE MERGED_MODEL_PATH BASE_MODEL_PATH EVAL_CONDA_ENV + +while (($# > 0)); do + case "$1" in + --model|--model-path) + [[ $# -ge 2 ]] || { echo "ERROR: $1 requires a value" >&2; exit 1; } + MODEL_PATH="$2" + shift 2 + ;; + --tasks) + [[ $# -ge 2 ]] || { echo "ERROR: $1 requires a value" >&2; exit 1; } + TASKS="$2" + shift 2 + ;; + --gpus) + if [[ $# -ge 2 && "$2" != --* ]]; then + GPUS="$2" + shift 2 + else + GPUS="$DEFAULT_GPUS" + shift + fi + ;; + --tp-size) + [[ $# -ge 2 ]] || { echo "ERROR: $1 requires a value" >&2; exit 1; } + TP_SIZE="$2" + shift 2 + ;; + --shard-mode) + [[ $# -ge 2 ]] || { echo "ERROR: $1 requires a value" >&2; exit 1; } + SHARD_MODE="$2" + shift 2 + ;; + --video-cache-size) + [[ $# -ge 2 ]] || { echo "ERROR: $1 requires a value" >&2; exit 1; } + VIDEO_CACHE_SIZE="$2" + shift 2 + ;; + --videomme-max-new-tokens) + [[ $# -ge 2 ]] || { echo "ERROR: $1 requires a value" >&2; exit 1; } + VIDEOMME_MAX_NEW_TOKENS="$2" + shift 2 + ;; + --force-merge) + FORCE_MERGE=1 + shift + ;; + --skip-merge) + SKIP_MERGE=1 + shift + ;; + --merged-model|--merged-model-path) + [[ $# -ge 2 ]] || { echo "ERROR: $1 requires a value" >&2; exit 1; } + MERGED_MODEL_PATH="$2" + shift 2 + ;; + --base-model|--base-model-path) + [[ $# -ge 2 ]] || { echo "ERROR: $1 requires a value" >&2; exit 1; } + BASE_MODEL_PATH="$2" + shift 2 + ;; + --env|--conda-env) + [[ $# -ge 2 ]] || { echo "ERROR: $1 requires a value" >&2; exit 1; } + EVAL_CONDA_ENV="$2" + shift 2 + ;; + --segmentation-run-sam2) + SEGMENTATION_RUN_SAM2=true + shift + ;; + --help|-h) + usage + exit 0 + ;; + --) + shift + break + ;; + *) + echo "ERROR: unknown option '$1'" >&2 + usage >&2 + exit 1 + ;; + esac +done + +# Require an explicit model so stale machine-local defaults cannot contaminate +# benchmark results. +if [[ -z "${MODEL_PATH:-}" ]]; then + echo "ERROR: --model PATH is required." >&2 + usage >&2 + exit 2 +fi + +MODEL_PATH="$(printf '%s' "$MODEL_PATH" | xargs)" + +# Common launch defaults. +: "${TASKS:=all}" +: "${GPUS:=$DEFAULT_GPUS}" +: "${TP_SIZE:=1}" +: "${SHARD_MODE:=contiguous}" +: "${VIDEO_CACHE_SIZE:=2}" +: "${PREFETCH_BATCHES:=1}" +: "${FORCE_MERGE:=0}" +: "${SKIP_MERGE:=0}" +: "${MERGE_LOG_DIR:=${PROJECT_DIR}/logs/qwen3.5/eval/merge}" + +if [[ ! "$GPUS" =~ ^[0-9]+(,[0-9]+)*$ ]]; then + echo "ERROR: --gpus must be a comma-separated list of GPU ids, got '${GPUS}'" >&2 + exit 1 +fi + +# VideoMME defaults. +: "${VIDEOMME_VIDEO_BASE:=${PROJECT_DIR}/data/eval/videomme}" +: "${VIDEOMME_VIDEO_DIR:=${VIDEOMME_VIDEO_BASE}/videos}" +: "${VIDEOMME_VIDEO_MIN_PIXELS:=4096}" +: "${VIDEOMME_VIDEO_MAX_PIXELS:=262144}" +: "${VIDEOMME_VIDEO_TOTAL_PIXELS:=0}" +: "${VIDEOMME_MAX_FRAMES:=384}" +: "${VIDEOMME_FPS:=2}" +: "${VIDEOMME_SETTING:=all-qwen3_vl-sub0-f${VIDEOMME_MAX_FRAMES}-fps${VIDEOMME_FPS}-min${VIDEOMME_VIDEO_MIN_PIXELS}-max${VIDEOMME_VIDEO_MAX_PIXELS}-total${VIDEOMME_VIDEO_TOTAL_PIXELS}-videomme_preprocessed_384f_262k_total0}" +: "${VIDEOMME_DATA_FILE:=${PROJECT_DIR}/eval/data/valid_data/videomme_preprocessed_384f_262k_total0.jsonl}" +: "${VIDEOMME_PREPROCESSED_VIDEO_DIR:=${VIDEOMME_VIDEO_BASE}/preprocessed_videos_384f_262k_total0}" +: "${VIDEOMME_BATCH_SIZE:=1}" +: "${VIDEOMME_MAX_MODEL_LEN:=65536}" +: "${VIDEOMME_MAX_NEW_TOKENS:=128}" +: "${VIDEOMME_MAX_NUM_BATCHED_TOKENS:=65536}" +: "${VIDEOMME_GPU_MEMORY_UTILIZATION:=0.90}" +: "${VIDEOMME_MAX_SAMPLES:=0}" + +# VideoMME-V2 defaults. Uses the same local schema and evaluator as VideoMME. +: "${VIDEOMMEV2_VIDEO_BASE:=${PROJECT_DIR}/data/eval/videommev2}" +: "${VIDEOMMEV2_VIDEO_DIR:=${VIDEOMMEV2_VIDEO_BASE}/videos}" +: "${VIDEOMMEV2_VIDEO_MIN_PIXELS:=4096}" +: "${VIDEOMMEV2_VIDEO_MAX_PIXELS:=262144}" +: "${VIDEOMMEV2_VIDEO_TOTAL_PIXELS:=0}" +: "${VIDEOMMEV2_MAX_FRAMES:=384}" +: "${VIDEOMMEV2_FPS:=2}" +: "${VIDEOMMEV2_SETTING:=all-qwen3_vl-sub0-f${VIDEOMMEV2_MAX_FRAMES}-fps${VIDEOMMEV2_FPS}-min${VIDEOMMEV2_VIDEO_MIN_PIXELS}-max${VIDEOMMEV2_VIDEO_MAX_PIXELS}-total${VIDEOMMEV2_VIDEO_TOTAL_PIXELS}-videommev2_preprocessed_384f_262k_total0}" +: "${VIDEOMMEV2_DATA_FILE:=${PROJECT_DIR}/eval/data/valid_data/videommev2_preprocessed_384f_262k_total0.jsonl}" +: "${VIDEOMMEV2_PREPROCESSED_VIDEO_DIR:=${VIDEOMMEV2_VIDEO_BASE}/preprocessed_videos_384f_262k_total0}" +: "${VIDEOMMEV2_BATCH_SIZE:=1}" +: "${VIDEOMMEV2_MAX_MODEL_LEN:=65536}" +: "${VIDEOMMEV2_MAX_NEW_TOKENS:=128}" +: "${VIDEOMMEV2_MAX_NUM_BATCHED_TOKENS:=65536}" +: "${VIDEOMMEV2_GPU_MEMORY_UTILIZATION:=0.90}" +: "${VIDEOMMEV2_PROMPT_MODE:=default}" +: "${VIDEOMMEV2_ANSWER_FILTER:=}" +: "${VIDEOMMEV2_MAX_SAMPLES:=0}" + +# VideoMMMU defaults. +: "${VIDEOMMMU_SETTING:=videommmu-f128-fps2-min4096-max262144-total0-with-image}" +: "${VIDEOMMMU_DATA_FILE:=${PROJECT_DIR}/eval/data/valid_data/videommmu_with_image.json}" +: "${VIDEOMMMU_VIDEO_ROOT:=${PROJECT_DIR}/data/eval/videommmu}" +: "${VIDEOMMMU_BATCH_SIZE:=4}" +: "${VIDEOMMMU_MAX_MODEL_LEN:=65536}" +: "${VIDEOMMMU_MAX_NEW_TOKENS:=128}" +: "${VIDEOMMMU_MAX_NUM_BATCHED_TOKENS:=65536}" +: "${VIDEOMMMU_GPU_MEMORY_UTILIZATION:=0.90}" +: "${VIDEOMMMU_VIDEO_MIN_PIXELS:=4096}" +: "${VIDEOMMMU_VIDEO_MAX_PIXELS:=262144}" +: "${VIDEOMMMU_VIDEO_TOTAL_PIXELS:=0}" +: "${VIDEOMMMU_MAX_FRAMES:=128}" +: "${VIDEOMMMU_FPS:=2}" +# OneThinker-style Adaptation uses subject videos + PNGs extracted from +# Adaptation/test-00000-of-00001.parquet. Run once before eval: +# python3 eval/data/extract_videommmu_images.py +# prompt_mode=onethink reproduces OneThinker's CoT prompt; pair it with a large +# VIDEOMMMU_MAX_NEW_TOKENS (OneThinker uses 8192). +: "${VIDEOMMMU_PROMPT_MODE:=default}" +# Separate Adaptation image resolution cap (OneThinker uses 1024*32*32=1048576). +: "${VIDEOMMMU_IMAGE_MAX_PIXELS:=1048576}" +# Qwen3.5 native benchmarking: thinking on + recommended sampling + long output. +# To reproduce the official VideoMMMU number, run with: +# VIDEOMMMU_PROMPT_MODE=qwen VIDEOMMMU_ENABLE_THINKING=true +# VIDEOMMMU_MAX_NEW_TOKENS=32768 VIDEOMMMU_MAX_MODEL_LEN=131072 +# VIDEOMMMU_TEMPERATURE=1.0 VIDEOMMMU_TOP_P=0.95 VIDEOMMMU_TOP_K=20 VIDEOMMMU_PRESENCE_PENALTY=1.5 +: "${VIDEOMMMU_ENABLE_THINKING:=false}" +: "${VIDEOMMMU_TEMPERATURE:=0.0}" +: "${VIDEOMMMU_TOP_P:=1.0}" +: "${VIDEOMMMU_TOP_K:=-1}" +: "${VIDEOMMMU_PRESENCE_PENALTY:=0.0}" +: "${VIDEOMMMU_MIN_P:=0.0}" + +# MMVU defaults (multiple-choice subset, raw video). +# Build the MC JSONL once: python scripts/build_mmvu_mc.py +: "${MMVU_DATA_FILE:=${PROJECT_DIR}/eval/data/valid_data/mmvu_mc.jsonl}" +: "${MMVU_VIDEO_ROOT:=${PROJECT_DIR}/data/eval/mmvu}" +: "${MMVU_BATCH_SIZE:=1}" +: "${MMVU_MAX_MODEL_LEN:=65536}" +: "${MMVU_MAX_NEW_TOKENS:=128}" +: "${MMVU_MAX_NUM_BATCHED_TOKENS:=65536}" +: "${MMVU_GPU_MEMORY_UTILIZATION:=0.90}" +: "${MMVU_VIDEO_MIN_PIXELS:=4096}" +: "${MMVU_VIDEO_MAX_PIXELS:=262144}" +: "${MMVU_VIDEO_TOTAL_PIXELS:=0}" +: "${MMVU_MAX_FRAMES:=384}" +: "${MMVU_FPS:=2}" +: "${MMVU_PROMPT_MODE:=default}" +# Qwen3.5 native: MMVU_PROMPT_MODE=qwen MMVU_ENABLE_THINKING=true with large +# MMVU_MAX_NEW_TOKENS / sampling, mirroring VideoMMMU. +: "${MMVU_ENABLE_THINKING:=false}" +: "${MMVU_TEMPERATURE:=0.0}" +: "${MMVU_TOP_P:=1.0}" +: "${MMVU_TOP_K:=-1}" +: "${MMVU_PRESENCE_PENALTY:=0.0}" +: "${MMVU_MIN_P:=0.0}" +: "${MMVU_MAX_SAMPLES:=0}" +: "${MMVU_SETTING:=mmvu-mc-f${MMVU_MAX_FRAMES}-fps${MMVU_FPS}-min${MMVU_VIDEO_MIN_PIXELS}-max${MMVU_VIDEO_MAX_PIXELS}-total${MMVU_VIDEO_TOTAL_PIXELS}-${MMVU_PROMPT_MODE}}" + +# MVBench defaults (multi-task short-video multiple-choice). +: "${MVBENCH_DATA_FILE:=${PROJECT_DIR}/eval/data/valid_data/mvbench.json}" +: "${MVBENCH_VIDEO_ROOT:=${PROJECT_DIR}/data/eval/mvbench}" +: "${MVBENCH_BATCH_SIZE:=1}" +: "${MVBENCH_MAX_MODEL_LEN:=65536}" +: "${MVBENCH_MAX_NEW_TOKENS:=128}" +: "${MVBENCH_MAX_NUM_BATCHED_TOKENS:=65536}" +: "${MVBENCH_GPU_MEMORY_UTILIZATION:=0.90}" +: "${MVBENCH_VIDEO_MIN_PIXELS:=4096}" +: "${MVBENCH_VIDEO_MAX_PIXELS:=262144}" +: "${MVBENCH_VIDEO_TOTAL_PIXELS:=0}" +: "${MVBENCH_MAX_FRAMES:=384}" +: "${MVBENCH_FPS:=2}" +: "${MVBENCH_PROMPT_MODE:=default}" +: "${MVBENCH_ENABLE_THINKING:=false}" +: "${MVBENCH_TEMPERATURE:=0.0}" +: "${MVBENCH_TOP_P:=1.0}" +: "${MVBENCH_TOP_K:=-1}" +: "${MVBENCH_PRESENCE_PENALTY:=0.0}" +: "${MVBENCH_MIN_P:=0.0}" +: "${MVBENCH_MAX_SAMPLES:=0}" +: "${MVBENCH_SETTING:=mvbench-f${MVBENCH_MAX_FRAMES}-fps${MVBENCH_FPS}-min${MVBENCH_VIDEO_MIN_PIXELS}-max${MVBENCH_VIDEO_MAX_PIXELS}-total${MVBENCH_VIDEO_TOTAL_PIXELS}-${MVBENCH_PROMPT_MODE}}" + +# Video-Holmes defaults (all multiple-choice, raw cropped videos). +# Download: huggingface-cli download TencentARC/Video-Holmes --repo-type dataset +# Build: python scripts/build_videoholmes.py +: "${VIDEOHOLMES_DATA_FILE:=${PROJECT_DIR}/eval/data/valid_data/videoholmes.jsonl}" +: "${VIDEOHOLMES_VIDEO_ROOT:=${PROJECT_DIR}/data/eval/videoholmes}" +: "${VIDEOHOLMES_BATCH_SIZE:=1}" +: "${VIDEOHOLMES_MAX_MODEL_LEN:=65536}" +: "${VIDEOHOLMES_MAX_NEW_TOKENS:=128}" +: "${VIDEOHOLMES_MAX_NUM_BATCHED_TOKENS:=65536}" +: "${VIDEOHOLMES_GPU_MEMORY_UTILIZATION:=0.90}" +: "${VIDEOHOLMES_VIDEO_MIN_PIXELS:=4096}" +: "${VIDEOHOLMES_VIDEO_MAX_PIXELS:=262144}" +: "${VIDEOHOLMES_VIDEO_TOTAL_PIXELS:=0}" +: "${VIDEOHOLMES_MAX_FRAMES:=384}" +: "${VIDEOHOLMES_FPS:=2}" +: "${VIDEOHOLMES_PROMPT_MODE:=default}" +# Official benchmark prompt is reasoning-style: +# VIDEOHOLMES_PROMPT_MODE=holmes VIDEOHOLMES_ENABLE_THINKING=true VIDEOHOLMES_MAX_NEW_TOKENS=1024 +: "${VIDEOHOLMES_ENABLE_THINKING:=false}" +: "${VIDEOHOLMES_TEMPERATURE:=0.0}" +: "${VIDEOHOLMES_TOP_P:=1.0}" +: "${VIDEOHOLMES_TOP_K:=-1}" +: "${VIDEOHOLMES_PRESENCE_PENALTY:=0.0}" +: "${VIDEOHOLMES_MIN_P:=0.0}" +: "${VIDEOHOLMES_MAX_SAMPLES:=0}" +: "${VIDEOHOLMES_SETTING:=videoholmes-f${VIDEOHOLMES_MAX_FRAMES}-fps${VIDEOHOLMES_FPS}-min${VIDEOHOLMES_VIDEO_MIN_PIXELS}-max${VIDEOHOLMES_VIDEO_MAX_PIXELS}-total${VIDEOHOLMES_VIDEO_TOTAL_PIXELS}-${VIDEOHOLMES_PROMPT_MODE}}" + +# LongVideoBench defaults. Main videos/subtitles repo is gated; after getting +# access, download/extract to LONGVIDEOBENCH_VIDEO_ROOT with videos/ + subtitles/. +# Build validation JSONL: python scripts/build_longvideobench.py +: "${LONGVIDEOBENCH_DATA_FILE:=${PROJECT_DIR}/eval/data/valid_data/longvideobench_val.jsonl}" +: "${LONGVIDEOBENCH_VIDEO_ROOT:=${PROJECT_DIR}/data/eval/longvideobench}" +: "${LONGVIDEOBENCH_SUBTITLE_ROOT:=$LONGVIDEOBENCH_VIDEO_ROOT}" +: "${LONGVIDEOBENCH_USE_SUBTITLES:=true}" +: "${LONGVIDEOBENCH_BATCH_SIZE:=1}" +: "${LONGVIDEOBENCH_MAX_MODEL_LEN:=65536}" +: "${LONGVIDEOBENCH_MAX_NEW_TOKENS:=128}" +: "${LONGVIDEOBENCH_MAX_NUM_BATCHED_TOKENS:=65536}" +: "${LONGVIDEOBENCH_GPU_MEMORY_UTILIZATION:=0.90}" +: "${LONGVIDEOBENCH_VIDEO_MIN_PIXELS:=4096}" +: "${LONGVIDEOBENCH_VIDEO_MAX_PIXELS:=262144}" +: "${LONGVIDEOBENCH_VIDEO_TOTAL_PIXELS:=0}" +: "${LONGVIDEOBENCH_MAX_FRAMES:=384}" +: "${LONGVIDEOBENCH_FPS:=2}" +: "${LONGVIDEOBENCH_PROMPT_MODE:=default}" +: "${LONGVIDEOBENCH_ENABLE_THINKING:=false}" +: "${LONGVIDEOBENCH_TEMPERATURE:=0.0}" +: "${LONGVIDEOBENCH_TOP_P:=1.0}" +: "${LONGVIDEOBENCH_TOP_K:=-1}" +: "${LONGVIDEOBENCH_PRESENCE_PENALTY:=0.0}" +: "${LONGVIDEOBENCH_MIN_P:=0.0}" +: "${LONGVIDEOBENCH_MAX_SAMPLES:=0}" +: "${LONGVIDEOBENCH_SETTING:=longvideobench-val-f${LONGVIDEOBENCH_MAX_FRAMES}-fps${LONGVIDEOBENCH_FPS}-min${LONGVIDEOBENCH_VIDEO_MIN_PIXELS}-max${LONGVIDEOBENCH_VIDEO_MAX_PIXELS}-total${LONGVIDEOBENCH_VIDEO_TOTAL_PIXELS}-${LONGVIDEOBENCH_PROMPT_MODE}}" + +# LVBench defaults (long-video multiple-choice; independent from LongVideoBench). +: "${LVBENCH_DATA_FILE:=${PROJECT_DIR}/eval/data/valid_data/lvbench.json}" +: "${LVBENCH_VIDEO_ROOT:=${PROJECT_DIR}/data/eval/lvbench}" +: "${LVBENCH_BATCH_SIZE:=4}" +: "${LVBENCH_MAX_MODEL_LEN:=65536}" +: "${LVBENCH_MAX_NEW_TOKENS:=128}" +: "${LVBENCH_MAX_NUM_BATCHED_TOKENS:=65536}" +: "${LVBENCH_GPU_MEMORY_UTILIZATION:=0.90}" +: "${LVBENCH_VIDEO_MIN_PIXELS:=4096}" +: "${LVBENCH_VIDEO_MAX_PIXELS:=262144}" +: "${LVBENCH_VIDEO_TOTAL_PIXELS:=0}" +: "${LVBENCH_MAX_FRAMES:=128}" +: "${LVBENCH_FPS:=2}" +: "${LVBENCH_PROMPT_MODE:=default}" +: "${LVBENCH_ENABLE_THINKING:=false}" +: "${LVBENCH_TEMPERATURE:=0.0}" +: "${LVBENCH_TOP_P:=1.0}" +: "${LVBENCH_TOP_K:=-1}" +: "${LVBENCH_PRESENCE_PENALTY:=0.0}" +: "${LVBENCH_MIN_P:=0.0}" +: "${LVBENCH_SETTING:=lvbench-f${LVBENCH_MAX_FRAMES}-fps${LVBENCH_FPS}-min${LVBENCH_VIDEO_MIN_PIXELS}-max${LVBENCH_VIDEO_MAX_PIXELS}-total${LVBENCH_VIDEO_TOTAL_PIXELS}-${LVBENCH_PROMPT_MODE}}" + +# MLVU defaults (dev multiple-choice subset = 7 tasks, M-Avg). +# Build the MC JSONL once: python scripts/build_mlvu_mc.py +: "${MLVU_DATA_FILE:=${PROJECT_DIR}/eval/data/valid_data/mlvu_mc.jsonl}" +: "${MLVU_VIDEO_ROOT:=${PROJECT_DIR}/data/eval/mlvu}" +: "${MLVU_BATCH_SIZE:=1}" +: "${MLVU_MAX_MODEL_LEN:=65536}" +: "${MLVU_MAX_NEW_TOKENS:=128}" +: "${MLVU_MAX_NUM_BATCHED_TOKENS:=65536}" +: "${MLVU_GPU_MEMORY_UTILIZATION:=0.90}" +: "${MLVU_VIDEO_MIN_PIXELS:=4096}" +: "${MLVU_VIDEO_MAX_PIXELS:=262144}" +: "${MLVU_VIDEO_TOTAL_PIXELS:=0}" +: "${MLVU_MAX_FRAMES:=384}" +: "${MLVU_FPS:=2}" +: "${MLVU_PROMPT_MODE:=default}" +: "${MLVU_ENABLE_THINKING:=false}" +: "${MLVU_TEMPERATURE:=0.0}" +: "${MLVU_TOP_P:=1.0}" +: "${MLVU_TOP_K:=-1}" +: "${MLVU_PRESENCE_PENALTY:=0.0}" +: "${MLVU_MIN_P:=0.0}" +: "${MLVU_MAX_SAMPLES:=0}" +: "${MLVU_SETTING:=mlvu-mc-f${MLVU_MAX_FRAMES}-fps${MLVU_FPS}-min${MLVU_VIDEO_MIN_PIXELS}-max${MLVU_VIDEO_MAX_PIXELS}-total${MLVU_VIDEO_TOTAL_PIXELS}-${MLVU_PROMPT_MODE}}" + +# VSI-Bench defaults. +: "${VSI_SETTING:=video128-16M-video-f128-fps2-min65536-maxnone-total16777216-vsibench_preprocessed_128f_16M}" +: "${VSI_DATA_FILE:=${PROJECT_DIR}/eval/data/valid_data/vsibench_preprocessed_128f_16M.jsonl}" +: "${VSI_PREPROCESSED_VIDEO_DIR:=${PROJECT_DIR}/data/eval/vsi/preprocessed_videos}" +: "${VSI_BATCH_SIZE:=16}" +: "${VSI_MAX_MODEL_LEN:=32768}" +: "${VSI_MAX_NEW_TOKENS:=1024}" +: "${VSI_GPU_MEMORY_UTILIZATION:=0.90}" +: "${VSI_EXPECTED_SAMPLES:=5130}" + +# MMSI-Bench defaults. Transformers is used so every image can be retained; +# set MMSI_BACKEND=vllm and MMSI_MAX_IMAGES=8 for the legacy path. +: "${MMSI_BACKEND:=transformers}" +: "${MMSI_DATA_FILE:=${PROJECT_DIR}/data/eval/mmsi/MMSI_bench.tsv}" +: "${MMSI_IMAGE_MIN_PIXELS:=4096}" +: "${MMSI_IMAGE_MAX_PIXELS:=262144}" +if [[ "${MMSI_BACKEND}" == "transformers" ]]; then + : "${MMSI_MAX_IMAGES:=0}" +else + : "${MMSI_MAX_IMAGES:=8}" +fi +: "${MMSI_IMAGE_COUNT_TAG:=$([[ "${MMSI_MAX_IMAGES}" -le 0 ]] && echo all || echo "${MMSI_MAX_IMAGES}")}" +: "${MMSI_SETTING:=${MMSI_BACKEND}-img-min${MMSI_IMAGE_MIN_PIXELS}-max${MMSI_IMAGE_MAX_PIXELS}-n${MMSI_IMAGE_COUNT_TAG}-all-MMSI_bench}" +: "${MMSI_BATCH_SIZE:=16}" +: "${MMSI_MAX_MODEL_LEN:=32768}" +: "${MMSI_MAX_NEW_TOKENS:=1024}" +: "${MMSI_MAX_NUM_BATCHED_TOKENS:=32768}" +: "${MMSI_GPU_MEMORY_UTILIZATION:=0.90}" +: "${MMSI_MAX_SAMPLES:=0}" +: "${MMSI_CATEGORY:=}" +: "${MMSI_ENABLE_THINKING:=false}" +: "${MMSI_ATTN_IMPLEMENTATION:=flash_attention_2}" +if [[ -n "${MMSI_CATEGORY}" || "${MMSI_MAX_SAMPLES}" != "0" ]]; then + : "${MMSI_EXPECTED_SAMPLES:=0}" +else + : "${MMSI_EXPECTED_SAMPLES:=1000}" +fi + +# MindCube-Tiny defaults. +: "${MINDCUBE_SETTING:=img-min4096-max262144-n4-official-tiny1050}" +: "${MINDCUBE_DATA_FILE:=${PROJECT_DIR}/data/eval/mindcube_official/combined-00000-of-00001.parquet}" +: "${MINDCUBE_MAX_IMAGES:=4}" +: "${MINDCUBE_EXPECTED_SAMPLES:=1050}" +: "${MINDCUBE_HF_CACHE_DIR:=}" +: "${MINDCUBE_BATCH_SIZE:=16}" +: "${MINDCUBE_MAX_MODEL_LEN:=32768}" +: "${MINDCUBE_MAX_NEW_TOKENS:=1024}" +: "${MINDCUBE_MAX_NUM_BATCHED_TOKENS:=32768}" +: "${MINDCUBE_GPU_MEMORY_UTILIZATION:=0.90}" + +# ReVSI defaults. The all-frame/native-frame profile reproduces the 58.2 result +# reported for Video-ORA-9B; ReVSI remains excluded from the three-benchmark +# spatial-intelligence average. +: "${REVSI_DATA_FILE:=${PROJECT_DIR}/data/eval/revsi/all_frame/test-00000-of-00001.parquet}" +: "${REVSI_VIDEO_ROOT:=${PROJECT_DIR}/data/eval/revsi}" +: "${REVSI_FRAME_BUDGET:=all}" +: "${REVSI_MAX_FRAMES:=128}" +: "${REVSI_EXACT_NFRAMES:=true}" +: "${REVSI_FPS:=2}" +: "${REVSI_VIDEO_MIN_PIXELS:=65536}" +: "${REVSI_VIDEO_MAX_PIXELS:=}" +: "${REVSI_VIDEO_TOTAL_PIXELS:=16777216}" +: "${REVSI_BATCH_SIZE:=16}" +: "${REVSI_MAX_MODEL_LEN:=32768}" +: "${REVSI_MAX_NEW_TOKENS:=64}" +: "${REVSI_GPU_MEMORY_UTILIZATION:=0.90}" +: "${REVSI_MAX_SAMPLES:=0}" +: "${REVSI_EXPECTED_SAMPLES:=6808}" +: "${REVSI_TASK_FILTER:=}" +: "${REVSI_ENABLE_THINKING:=false}" +: "${REVSI_SETTING:=native-all-f${REVSI_MAX_FRAMES}-exact${REVSI_EXACT_NFRAMES}-fps${REVSI_FPS}-min${REVSI_VIDEO_MIN_PIXELS}-max${REVSI_VIDEO_MAX_PIXELS:-none}-total${REVSI_VIDEO_TOTAL_PIXELS}}" + +# Spatial grounding defaults, aligned to data/joint/sft_joint_all.jsonl. +: "${SPATIAL_GROUNDING_DATASETS:=refcoco-val,refcoco-testA,refcoco-testB,refcoco+-val,refcoco+-testA,refcoco+-testB,refcocog-val,refcocog-test}" +: "${SPATIAL_GROUNDING_BENCH_DIR:=${PROJECT_DIR}/eval/task/spatial_grounding}" +: "${SPATIAL_GROUNDING_IMAGE_ROOT:=${PROJECT_DIR}/data/eval/spatial_grounding/images}" +: "${SPATIAL_GROUNDING_PROCESSOR_PATH:=$MODEL_PATH}" +: "${SPATIAL_GROUNDING_PROMPT_STYLE:=qwen_native}" +: "${SPATIAL_GROUNDING_COORD_SYSTEM:=norm1000}" +: "${SPATIAL_GROUNDING_BBOX_SELECT:=first}" +: "${SPATIAL_GROUNDING_MIN_TOKENS:=64}" +: "${SPATIAL_GROUNDING_TOTAL_TOKENS:=1024}" +: "${SPATIAL_GROUNDING_BATCH_SIZE:=64}" +: "${SPATIAL_GROUNDING_MAX_MODEL_LEN:=32768}" +: "${SPATIAL_GROUNDING_MAX_NEW_TOKENS:=1024}" +: "${SPATIAL_GROUNDING_MAX_NUM_BATCHED_TOKENS:=32768}" +: "${SPATIAL_GROUNDING_GPU_MEMORY_UTILIZATION:=0.85}" +: "${SPATIAL_GROUNDING_ENABLE_THINKING:=false}" +: "${SPATIAL_GROUNDING_MAX_SAMPLES:=0}" +: "${SPATIAL_GROUNDING_SETTING:=spatial-grounding-${SPATIAL_GROUNDING_DATASETS//,/_}-${SPATIAL_GROUNDING_PROMPT_STYLE}-min${SPATIAL_GROUNDING_MIN_TOKENS}-total${SPATIAL_GROUNDING_TOTAL_TOKENS}-new${SPATIAL_GROUNDING_MAX_NEW_TOKENS}}" + +# Tracking defaults, aligned to data/joint/sft_joint_all.jsonl tracking videos. +: "${TRACKING_DATASETS:=eval_got10k}" +: "${TRACKING_BENCH_DIR:=${PROJECT_DIR}/data/eval/tracking}" +: "${TRACKING_BASE_PREFIX:=$TRACKING_BENCH_DIR}" +: "${TRACKING_PROCESSOR_PATH:=$MODEL_PATH}" +: "${TRACKING_VIDEO_MIN_PIXELS:=4096}" +: "${TRACKING_VIDEO_MAX_PIXELS:=786432}" +: "${TRACKING_VIDEO_TOTAL_PIXELS:=8388608}" +: "${TRACKING_MAX_FRAMES:=32}" +: "${TRACKING_FPS:=1}" +: "${TRACKING_MAX_MODEL_LEN:=32768}" +: "${TRACKING_MAX_NEW_TOKENS:=8192}" +: "${TRACKING_MAX_NUM_BATCHED_TOKENS:=32768}" +: "${TRACKING_BATCH_SIZE:=64}" +: "${TRACKING_GPU_MEMORY_UTILIZATION:=0.95}" +: "${TRACKING_ENABLE_THINKING:=false}" +: "${TRACKING_PROMPT_MODE:=default}" +: "${TRACKING_CHUNKED_REPROMPT:=0}" +: "${TRACKING_REPROMPT_USE_GT_FIRST_BOX:=0}" +: "${TRACKING_MAX_SAMPLES:=0}" +: "${TRACKING_SETTING:=tracking-${TRACKING_DATASETS//,/_}-f${TRACKING_MAX_FRAMES}-fps${TRACKING_FPS}-min${TRACKING_VIDEO_MIN_PIXELS}-max${TRACKING_VIDEO_MAX_PIXELS}-total${TRACKING_VIDEO_TOTAL_PIXELS}-new${TRACKING_MAX_NEW_TOKENS}}" + +# STVG defaults, aligned to data/joint/sft_joint_all.jsonl stvg videos. +: "${STVG_DATASETS:=eval_stvg}" +: "${STVG_BENCH_DIR:=${PROJECT_DIR}/data/eval/stvg}" +: "${STVG_BASE_PREFIX:=$STVG_BENCH_DIR}" +: "${STVG_PROCESSOR_PATH:=$MODEL_PATH}" +: "${STVG_VIDEO_MIN_PIXELS:=65536}" +: "${STVG_VIDEO_MAX_PIXELS:=393216}" +: "${STVG_VIDEO_TOTAL_PIXELS:=10485760}" +: "${STVG_MAX_FRAMES:=128}" +: "${STVG_FPS:=2}" +: "${STVG_MAX_MODEL_LEN:=65536}" +: "${STVG_MAX_NEW_TOKENS:=2048}" +: "${STVG_MAX_NUM_BATCHED_TOKENS:=65536}" +: "${STVG_BATCH_SIZE:=64}" +: "${STVG_GPU_MEMORY_UTILIZATION:=0.85}" +: "${STVG_ENABLE_THINKING:=false}" +: "${STVG_PROMPT_MODE:=train_stvg}" +: "${STVG_MAX_SAMPLES:=0}" +: "${STVG_SETTING:=stvg-${STVG_DATASETS//,/_}-f${STVG_MAX_FRAMES}-fps${STVG_FPS}-min${STVG_VIDEO_MIN_PIXELS}-max${STVG_VIDEO_MAX_PIXELS}-total${STVG_VIDEO_TOTAL_PIXELS}-new${STVG_MAX_NEW_TOKENS}-${STVG_PROMPT_MODE}}" + +# Segmentation defaults. +# This task is dispatched to eval/task/segmentation/run_eval_vllm.sh, +# which performs vLLM inference plus optional SAM2 post-processing. +: "${SEGMENTATION_DATASETS:=eval_seg_refcoco,eval_seg_refcocop,eval_seg_refcocog,eval_seg_mevis,eval_seg_reasonvos}" +: "${SEGMENTATION_BENCH_DIR:=${PROJECT_DIR}/data/eval/segmentation}" +: "${SEGMENTATION_DATA_ROOT:=$SEGMENTATION_BENCH_DIR}" +: "${SEGMENTATION_PROCESSOR_PATH:=$MODEL_PATH}" +: "${SEGMENTATION_DATA_TYPE:=all}" +# train_seg mirrors the prompt used in joint SFT training data +# (data/train/seg_image_nothink_evalaware.jsonl, seg_video_nothink.jsonl). +: "${SEGMENTATION_PROMPT_MODE:=train_seg}" +: "${SEGMENTATION_ENABLE_THINKING:=false}" +: "${SEGMENTATION_BATCH_SIZE:=16}" +: "${SEGMENTATION_MAX_NEW_TOKENS:=1024}" +: "${SEGMENTATION_MAX_MODEL_LEN:=32768}" +: "${SEGMENTATION_MAX_PIXELS_IMAGE:=1048576}" +: "${SEGMENTATION_MIN_PIXELS_IMAGE:=4096}" +: "${SEGMENTATION_VIDEO_MAX_PIXELS:=262144}" +: "${SEGMENTATION_VIDEO_MIN_PIXELS:=4096}" +: "${SEGMENTATION_VIDEO_TOTAL_PIXELS:=16777216}" +: "${SEGMENTATION_MAX_FRAMES:=128}" +: "${SEGMENTATION_FPS:=2}" +: "${SEGMENTATION_VIDEO_READER:=decord}" +: "${SEGMENTATION_GPU_MEM_UTIL:=0.85}" +: "${SEGMENTATION_SEED:=42}" +: "${SEGMENTATION_MAX_SAMPLES:=}" +: "${SEGMENTATION_RUN_SAM2:=false}" +: "${SEGMENTATION_SAM2_CKPT:=${PROJECT_DIR}/models/sam2/sam2.1_hiera_large.pt}" +: "${SEGMENTATION_SAM2_CFG:=configs/sam2.1/sam2.1_hiera_l.yaml}" +: "${SEGMENTATION_POSTPROCESSOR_PATH:=${PROJECT_DIR}/third_party/OneThinker/Evaluation/Eval/seg_post_sam2.py}" +# SAM2 post-processing parallelism. world_size = NUM_GPUS * WORKERS_PER_GPU. +# With a large SAM2_EPOCH_SIZE (single epoch) + maxtasksperchild=1, EACH worker +# loads the SAM2 model exactly ONCE at startup and then streams its whole slice +# (no repeated reloads). So WORKERS_PER_GPU only trades GPU utilization vs total +# process count: too high (32 -> 256 procs) exhausts CPU/RAM/ffmpeg/handles and +# aborts; too low (1) underutilizes the GPU. 4 per GPU is a balance (8 GPUs -> 32 +# persistent workers, 32 one-time loads); bump to 6-8 if GPU is still not full. +# Leave NUM_GPUS empty to auto-use all visible GPUs. +: "${SEGMENTATION_SAM2_NUM_GPUS:=}" +: "${SEGMENTATION_SAM2_WORKERS_PER_GPU:=8}" +: "${SEGMENTATION_SETTING:=segmentation-${SEGMENTATION_DATASETS//,/_}-${SEGMENTATION_PROMPT_MODE}-f${SEGMENTATION_MAX_FRAMES}-fps${SEGMENTATION_FPS}-min${SEGMENTATION_VIDEO_MIN_PIXELS}-max${SEGMENTATION_VIDEO_MAX_PIXELS}-total${SEGMENTATION_VIDEO_TOTAL_PIXELS}-new${SEGMENTATION_MAX_NEW_TOKENS}}" + +# TimeLens-Bench temporal grounding defaults. +# This task uses HuggingFace transformers, not vLLM. +: "${TIMELENS_DATASETS:=charades-timelens}" +: "${TIMELENS_BENCH_DIR:=${PROJECT_DIR}/data/eval/temporal_grounding}" +: "${TIMELENS_ENABLE_THINKING:=false}" +: "${TIMELENS_MIN_TOKENS:=1}" +: "${TIMELENS_TOTAL_TOKENS:=128000}" +: "${TIMELENS_MAX_PIXELS:=409600}" +: "${TIMELENS_MAX_FRAMES:=2048}" +: "${TIMELENS_FPS:=4}" +: "${TIMELENS_MAX_NEW_TOKENS:=128}" +: "${TIMELENS_REPETITION_PENALTY:=1.0}" +: "${TIMELENS_STOP_AFTER_ANSWER:=true}" +: "${TIMELENS_PROMPT_MODE:=same}" +: "${TIMELENS_NUM_WORKERS:=2}" +: "${TIMELENS_MAX_SAMPLES:=0}" + +export MODEL_PATH TASKS GPUS TP_SIZE SHARD_MODE VIDEO_CACHE_SIZE PREFETCH_BATCHES +export FORCE_MERGE SKIP_MERGE MERGE_LOG_DIR +export VIDEOMME_SETTING VIDEOMME_DATA_FILE VIDEOMME_VIDEO_BASE VIDEOMME_VIDEO_DIR +export VIDEOMME_PREPROCESSED_VIDEO_DIR VIDEOMME_VIDEO_MIN_PIXELS VIDEOMME_VIDEO_MAX_PIXELS +export VIDEOMME_VIDEO_TOTAL_PIXELS VIDEOMME_MAX_FRAMES VIDEOMME_FPS +export VIDEOMME_BATCH_SIZE VIDEOMME_MAX_MODEL_LEN VIDEOMME_MAX_NEW_TOKENS +export VIDEOMME_MAX_NUM_BATCHED_TOKENS VIDEOMME_GPU_MEMORY_UTILIZATION +export VIDEOMME_MAX_SAMPLES +export VIDEOMMEV2_SETTING VIDEOMMEV2_DATA_FILE VIDEOMMEV2_VIDEO_BASE VIDEOMMEV2_VIDEO_DIR +export VIDEOMMEV2_PREPROCESSED_VIDEO_DIR VIDEOMMEV2_VIDEO_MIN_PIXELS VIDEOMMEV2_VIDEO_MAX_PIXELS +export VIDEOMMEV2_VIDEO_TOTAL_PIXELS VIDEOMMEV2_MAX_FRAMES VIDEOMMEV2_FPS +export VIDEOMMEV2_BATCH_SIZE VIDEOMMEV2_MAX_MODEL_LEN VIDEOMMEV2_MAX_NEW_TOKENS +export VIDEOMMEV2_MAX_NUM_BATCHED_TOKENS VIDEOMMEV2_GPU_MEMORY_UTILIZATION +export VIDEOMMEV2_PROMPT_MODE VIDEOMMEV2_ANSWER_FILTER VIDEOMMEV2_MAX_SAMPLES +export VIDEOMMMU_SETTING VIDEOMMMU_DATA_FILE VIDEOMMMU_VIDEO_ROOT +export VIDEOMMMU_BATCH_SIZE VIDEOMMMU_MAX_MODEL_LEN VIDEOMMMU_MAX_NEW_TOKENS +export VIDEOMMMU_MAX_NUM_BATCHED_TOKENS VIDEOMMMU_GPU_MEMORY_UTILIZATION +export VIDEOMMMU_VIDEO_MIN_PIXELS VIDEOMMMU_VIDEO_MAX_PIXELS VIDEOMMMU_VIDEO_TOTAL_PIXELS +export VIDEOMMMU_MAX_FRAMES VIDEOMMMU_FPS VIDEOMMMU_PROMPT_MODE VIDEOMMMU_IMAGE_MAX_PIXELS +export VIDEOMMMU_ENABLE_THINKING VIDEOMMMU_TEMPERATURE VIDEOMMMU_TOP_P VIDEOMMMU_TOP_K +export VIDEOMMMU_PRESENCE_PENALTY VIDEOMMMU_MIN_P +export MMVU_SETTING MMVU_DATA_FILE MMVU_VIDEO_ROOT MMVU_BATCH_SIZE +export MMVU_MAX_MODEL_LEN MMVU_MAX_NEW_TOKENS MMVU_MAX_NUM_BATCHED_TOKENS +export MMVU_GPU_MEMORY_UTILIZATION MMVU_VIDEO_MIN_PIXELS MMVU_VIDEO_MAX_PIXELS +export MMVU_VIDEO_TOTAL_PIXELS MMVU_MAX_FRAMES MMVU_FPS MMVU_PROMPT_MODE +export MMVU_ENABLE_THINKING MMVU_TEMPERATURE MMVU_TOP_P MMVU_TOP_K +export MMVU_PRESENCE_PENALTY MMVU_MIN_P MMVU_MAX_SAMPLES +export MVBENCH_SETTING MVBENCH_DATA_FILE MVBENCH_VIDEO_ROOT MVBENCH_BATCH_SIZE +export MVBENCH_MAX_MODEL_LEN MVBENCH_MAX_NEW_TOKENS MVBENCH_MAX_NUM_BATCHED_TOKENS +export MVBENCH_GPU_MEMORY_UTILIZATION MVBENCH_VIDEO_MIN_PIXELS MVBENCH_VIDEO_MAX_PIXELS +export MVBENCH_VIDEO_TOTAL_PIXELS MVBENCH_MAX_FRAMES MVBENCH_FPS MVBENCH_PROMPT_MODE +export MVBENCH_ENABLE_THINKING MVBENCH_TEMPERATURE MVBENCH_TOP_P MVBENCH_TOP_K +export MVBENCH_PRESENCE_PENALTY MVBENCH_MIN_P MVBENCH_MAX_SAMPLES +export VIDEOHOLMES_SETTING VIDEOHOLMES_DATA_FILE VIDEOHOLMES_VIDEO_ROOT VIDEOHOLMES_BATCH_SIZE +export VIDEOHOLMES_MAX_MODEL_LEN VIDEOHOLMES_MAX_NEW_TOKENS VIDEOHOLMES_MAX_NUM_BATCHED_TOKENS +export VIDEOHOLMES_GPU_MEMORY_UTILIZATION VIDEOHOLMES_VIDEO_MIN_PIXELS VIDEOHOLMES_VIDEO_MAX_PIXELS +export VIDEOHOLMES_VIDEO_TOTAL_PIXELS VIDEOHOLMES_MAX_FRAMES VIDEOHOLMES_FPS VIDEOHOLMES_PROMPT_MODE +export VIDEOHOLMES_ENABLE_THINKING VIDEOHOLMES_TEMPERATURE VIDEOHOLMES_TOP_P VIDEOHOLMES_TOP_K +export VIDEOHOLMES_PRESENCE_PENALTY VIDEOHOLMES_MIN_P VIDEOHOLMES_MAX_SAMPLES +export LONGVIDEOBENCH_SETTING LONGVIDEOBENCH_DATA_FILE LONGVIDEOBENCH_VIDEO_ROOT +export LONGVIDEOBENCH_SUBTITLE_ROOT LONGVIDEOBENCH_USE_SUBTITLES LONGVIDEOBENCH_BATCH_SIZE +export LONGVIDEOBENCH_MAX_MODEL_LEN LONGVIDEOBENCH_MAX_NEW_TOKENS LONGVIDEOBENCH_MAX_NUM_BATCHED_TOKENS +export LONGVIDEOBENCH_GPU_MEMORY_UTILIZATION LONGVIDEOBENCH_VIDEO_MIN_PIXELS LONGVIDEOBENCH_VIDEO_MAX_PIXELS +export LONGVIDEOBENCH_VIDEO_TOTAL_PIXELS LONGVIDEOBENCH_MAX_FRAMES LONGVIDEOBENCH_FPS LONGVIDEOBENCH_PROMPT_MODE +export LONGVIDEOBENCH_ENABLE_THINKING LONGVIDEOBENCH_TEMPERATURE LONGVIDEOBENCH_TOP_P LONGVIDEOBENCH_TOP_K +export LONGVIDEOBENCH_PRESENCE_PENALTY LONGVIDEOBENCH_MIN_P LONGVIDEOBENCH_MAX_SAMPLES +export LVBENCH_SETTING LVBENCH_DATA_FILE LVBENCH_VIDEO_ROOT LVBENCH_BATCH_SIZE +export LVBENCH_MAX_MODEL_LEN LVBENCH_MAX_NEW_TOKENS LVBENCH_MAX_NUM_BATCHED_TOKENS +export LVBENCH_GPU_MEMORY_UTILIZATION LVBENCH_VIDEO_MIN_PIXELS LVBENCH_VIDEO_MAX_PIXELS +export LVBENCH_VIDEO_TOTAL_PIXELS LVBENCH_MAX_FRAMES LVBENCH_FPS LVBENCH_PROMPT_MODE +export LVBENCH_ENABLE_THINKING LVBENCH_TEMPERATURE LVBENCH_TOP_P LVBENCH_TOP_K +export LVBENCH_PRESENCE_PENALTY LVBENCH_MIN_P +export MLVU_SETTING MLVU_DATA_FILE MLVU_VIDEO_ROOT MLVU_BATCH_SIZE +export MLVU_MAX_MODEL_LEN MLVU_MAX_NEW_TOKENS MLVU_MAX_NUM_BATCHED_TOKENS +export MLVU_GPU_MEMORY_UTILIZATION MLVU_VIDEO_MIN_PIXELS MLVU_VIDEO_MAX_PIXELS +export MLVU_VIDEO_TOTAL_PIXELS MLVU_MAX_FRAMES MLVU_FPS MLVU_PROMPT_MODE +export MLVU_ENABLE_THINKING MLVU_TEMPERATURE MLVU_TOP_P MLVU_TOP_K +export MLVU_PRESENCE_PENALTY MLVU_MIN_P MLVU_MAX_SAMPLES +export VSI_SETTING VSI_DATA_FILE VSI_PREPROCESSED_VIDEO_DIR +export VSI_BATCH_SIZE VSI_MAX_MODEL_LEN VSI_MAX_NEW_TOKENS VSI_GPU_MEMORY_UTILIZATION +export VSI_EXPECTED_SAMPLES +export MMSI_BACKEND MMSI_SETTING MMSI_DATA_FILE MMSI_MAX_IMAGES MMSI_BATCH_SIZE +export MMSI_IMAGE_MIN_PIXELS MMSI_IMAGE_MAX_PIXELS MMSI_IMAGE_COUNT_TAG +export MMSI_MAX_MODEL_LEN MMSI_MAX_NEW_TOKENS MMSI_MAX_NUM_BATCHED_TOKENS +export MMSI_GPU_MEMORY_UTILIZATION MMSI_MAX_SAMPLES MMSI_CATEGORY +export MMSI_ENABLE_THINKING MMSI_ATTN_IMPLEMENTATION MMSI_EXPECTED_SAMPLES +export MINDCUBE_SETTING MINDCUBE_DATA_FILE MINDCUBE_MAX_IMAGES MINDCUBE_BATCH_SIZE +export MINDCUBE_MAX_MODEL_LEN MINDCUBE_MAX_NEW_TOKENS MINDCUBE_MAX_NUM_BATCHED_TOKENS +export MINDCUBE_GPU_MEMORY_UTILIZATION MINDCUBE_EXPECTED_SAMPLES MINDCUBE_HF_CACHE_DIR +export REVSI_SETTING REVSI_DATA_FILE REVSI_VIDEO_ROOT REVSI_FRAME_BUDGET +export REVSI_MAX_FRAMES REVSI_EXACT_NFRAMES REVSI_FPS +export REVSI_VIDEO_MIN_PIXELS REVSI_VIDEO_MAX_PIXELS REVSI_VIDEO_TOTAL_PIXELS +export REVSI_BATCH_SIZE REVSI_MAX_MODEL_LEN REVSI_MAX_NEW_TOKENS +export REVSI_GPU_MEMORY_UTILIZATION REVSI_MAX_SAMPLES REVSI_EXPECTED_SAMPLES +export REVSI_TASK_FILTER REVSI_ENABLE_THINKING +export SPATIAL_GROUNDING_SETTING SPATIAL_GROUNDING_DATASETS SPATIAL_GROUNDING_BENCH_DIR +export SPATIAL_GROUNDING_IMAGE_ROOT SPATIAL_GROUNDING_PROCESSOR_PATH SPATIAL_GROUNDING_PROMPT_STYLE +export SPATIAL_GROUNDING_COORD_SYSTEM SPATIAL_GROUNDING_BBOX_SELECT +export SPATIAL_GROUNDING_MIN_TOKENS SPATIAL_GROUNDING_TOTAL_TOKENS +export SPATIAL_GROUNDING_BATCH_SIZE SPATIAL_GROUNDING_MAX_MODEL_LEN +export SPATIAL_GROUNDING_MAX_NEW_TOKENS SPATIAL_GROUNDING_MAX_NUM_BATCHED_TOKENS +export SPATIAL_GROUNDING_GPU_MEMORY_UTILIZATION SPATIAL_GROUNDING_ENABLE_THINKING +export SPATIAL_GROUNDING_MAX_SAMPLES +export TRACKING_SETTING TRACKING_DATASETS TRACKING_BENCH_DIR TRACKING_BASE_PREFIX +export TRACKING_PROCESSOR_PATH TRACKING_VIDEO_MIN_PIXELS TRACKING_VIDEO_MAX_PIXELS +export TRACKING_VIDEO_TOTAL_PIXELS TRACKING_MAX_FRAMES TRACKING_FPS +export TRACKING_MAX_MODEL_LEN TRACKING_MAX_NEW_TOKENS TRACKING_MAX_NUM_BATCHED_TOKENS +export TRACKING_BATCH_SIZE TRACKING_GPU_MEMORY_UTILIZATION TRACKING_ENABLE_THINKING +export TRACKING_PROMPT_MODE TRACKING_CHUNKED_REPROMPT TRACKING_REPROMPT_USE_GT_FIRST_BOX +export TRACKING_MAX_SAMPLES +export STVG_SETTING STVG_DATASETS STVG_BENCH_DIR STVG_BASE_PREFIX STVG_PROCESSOR_PATH +export STVG_VIDEO_MIN_PIXELS STVG_VIDEO_MAX_PIXELS STVG_VIDEO_TOTAL_PIXELS +export STVG_MAX_FRAMES STVG_FPS STVG_MAX_MODEL_LEN STVG_MAX_NEW_TOKENS +export STVG_MAX_NUM_BATCHED_TOKENS STVG_BATCH_SIZE STVG_GPU_MEMORY_UTILIZATION +export STVG_ENABLE_THINKING STVG_PROMPT_MODE STVG_MAX_SAMPLES +export SEGMENTATION_SETTING SEGMENTATION_DATASETS SEGMENTATION_BENCH_DIR SEGMENTATION_DATA_ROOT +export SEGMENTATION_PROCESSOR_PATH SEGMENTATION_DATA_TYPE SEGMENTATION_PROMPT_MODE +export SEGMENTATION_ENABLE_THINKING SEGMENTATION_BATCH_SIZE SEGMENTATION_MAX_NEW_TOKENS +export SEGMENTATION_MAX_MODEL_LEN SEGMENTATION_MAX_PIXELS_IMAGE SEGMENTATION_MIN_PIXELS_IMAGE +export SEGMENTATION_VIDEO_MAX_PIXELS SEGMENTATION_VIDEO_MIN_PIXELS SEGMENTATION_VIDEO_TOTAL_PIXELS +export SEGMENTATION_MAX_FRAMES SEGMENTATION_FPS SEGMENTATION_VIDEO_READER +export SEGMENTATION_GPU_MEM_UTIL SEGMENTATION_SEED +export SEGMENTATION_MAX_SAMPLES +export SEGMENTATION_RUN_SAM2 SEGMENTATION_SAM2_CKPT SEGMENTATION_SAM2_CFG +export SEGMENTATION_SAM2_NUM_GPUS SEGMENTATION_SAM2_WORKERS_PER_GPU +export TIMELENS_BENCH_DIR TIMELENS_MAX_SAMPLES + +# Conda env used to run eval. By default, keep the currently active environment. +# Override with EVAL_CONDA_ENV=xxx only when an explicit env switch is desired. +: "${EVAL_CONDA_ENV:=keep}" +if [[ "${EVAL_CONDA_ENV}" != "keep" && "${CONDA_DEFAULT_ENV:-}" != "${EVAL_CONDA_ENV}" ]]; then + if command -v conda >/dev/null 2>&1; then + eval "$(conda shell.bash hook)" + conda activate "${EVAL_CONDA_ENV}" + else + echo "ERROR: conda not found and CONDA_DEFAULT_ENV=${CONDA_DEFAULT_ENV:-}" >&2 + exit 1 + fi +fi + +cd "${PROJECT_DIR}" + +resolve_model_path() { + local input_path="${MODEL_PATH%/}" + + if ls "${input_path}"/model_world_size_*_rank_0.pt >/dev/null 2>&1; then + local actor_dir="$input_path" + local hf_dir="${MERGED_MODEL_PATH:-${actor_dir}/huggingface}" + local need_merge=0 + + if [[ "${SKIP_MERGE}" = "1" ]]; then + need_merge=0 + elif [[ "${FORCE_MERGE}" = "1" ]]; then + need_merge=1 + elif [[ ! -f "${hf_dir}/model.safetensors" ]]; then + need_merge=1 + fi + + if [[ "${need_merge}" = "1" ]]; then + mkdir -p "${MERGE_LOG_DIR}" + local merge_log="${MERGE_LOG_DIR}/merge-$(basename "$(dirname "${actor_dir}")")-$(date +%Y%m%d_%H%M%S).log" + echo "[merge] FSDP actor checkpoint detected: ${actor_dir}" + echo "[merge] Merging shards -> ${hf_dir}" + if [[ -n "${BASE_MODEL_PATH:-}" ]]; then + python -u scripts/model_merger.py \ + --local_dir "${actor_dir}" \ + --base_model_path "${BASE_MODEL_PATH}" \ + 2>&1 | tee "${merge_log}" + else + python -u scripts/model_merger.py \ + --local_dir "${actor_dir}" \ + 2>&1 | tee "${merge_log}" + fi + if [[ ! -f "${hf_dir}/model.safetensors" ]]; then + echo "ERROR: merge finished but ${hf_dir}/model.safetensors was not created. Log: ${merge_log}" >&2 + exit 1 + fi + else + echo "[merge] Using existing merged HF model: ${hf_dir}" + fi + + MODEL_PATH="${hf_dir}" + export MODEL_PATH + + if [[ "${TRACKING_PROCESSOR_PATH%/}" = "${input_path}" ]]; then + TRACKING_PROCESSOR_PATH="${MODEL_PATH}" + export TRACKING_PROCESSOR_PATH + fi + if [[ "${STVG_PROCESSOR_PATH%/}" = "${input_path}" ]]; then + STVG_PROCESSOR_PATH="${MODEL_PATH}" + export STVG_PROCESSOR_PATH + fi + if [[ "${SPATIAL_GROUNDING_PROCESSOR_PATH%/}" = "${input_path}" ]]; then + SPATIAL_GROUNDING_PROCESSOR_PATH="${MODEL_PATH}" + export SPATIAL_GROUNDING_PROCESSOR_PATH + fi + if [[ "${SEGMENTATION_PROCESSOR_PATH%/}" = "${input_path}" ]]; then + SEGMENTATION_PROCESSOR_PATH="${MODEL_PATH}" + export SEGMENTATION_PROCESSOR_PATH + fi + elif [[ -d "${input_path}/huggingface" && -f "${input_path}/huggingface/model.safetensors" && ! -f "${input_path}/config.json" ]]; then + MODEL_PATH="${input_path}/huggingface" + export MODEL_PATH + echo "[merge] MODEL_PATH points to a checkpoint wrapper; using ${MODEL_PATH}" + fi +} + +resolve_model_path + +VLLM_TASKS="" +RUN_TIMELENS=0 +RUN_SEGMENTATION=0 +RUN_MMSI_TRANSFORMERS=0 +RUN_REVSI=0 + +if [[ "${MMSI_BACKEND}" != "transformers" && "${MMSI_BACKEND}" != "vllm" ]]; then + echo "ERROR: MMSI_BACKEND must be 'transformers' or 'vllm', got '${MMSI_BACKEND}'" >&2 + exit 1 +fi + +append_vllm_task() { + local task="$1" + if [[ -z "$VLLM_TASKS" ]]; then + VLLM_TASKS="$task" + else + VLLM_TASKS="${VLLM_TASKS},${task}" + fi +} + +model_family_tag() { + local model_path="${1%/}" + local name + name="$(basename "$model_path")" + local parent + local grandparent + parent="$(basename "$(dirname "$model_path")")" + grandparent="$(basename "$(dirname "$(dirname "$model_path")")")" + if [[ "$name" == "huggingface" && "$parent" == "actor" && "$grandparent" == global_step_* ]]; then + basename "$(dirname "$(dirname "$(dirname "$model_path")")")" + return + fi + if [[ "$name" == checkpoint-* ]]; then + basename "$(dirname "$(dirname "$model_path")")" + else + echo "$name" + fi +} + +checkpoint_tag() { + local model_path="${1%/}" + local name + name="$(basename "$model_path")" + local parent + local grandparent + parent="$(basename "$(dirname "$model_path")")" + grandparent="$(basename "$(dirname "$(dirname "$model_path")")")" + if [[ "$name" == "huggingface" && "$parent" == "actor" && "$grandparent" == global_step_* ]]; then + basename "$(dirname "$(dirname "$model_path")")" + return + fi + if [[ "$name" == checkpoint-* ]]; then + echo "$name" + else + echo "base" + fi +} + +IFS=',' read -ra TASK_LIST <<< "$TASKS" +for RAW_TASK in "${TASK_LIST[@]}"; do + TASK="$(echo "$RAW_TASK" | xargs)" + case "$TASK" in + all) + VLLM_TASKS="videomme,videommev2,videommmu,mmvu,mvbench,videoholmes,longvideobench,lvbench,mlvu,vsi,mindcube,spatial_grounding,tracking,stvg" + if [[ "${MMSI_BACKEND}" == "transformers" ]]; then + RUN_MMSI_TRANSFORMERS=1 + else + append_vllm_task "mmsi" + fi + RUN_REVSI=1 + ;; + videomme|videommev2|videommmu|mmvu|mvbench|videoholmes|longvideobench|lvbench|mlvu|vsi|mindcube|spatial_grounding|tracking|stvg) + append_vllm_task "$TASK" + ;; + mmsi) + if [[ "${MMSI_BACKEND}" == "transformers" ]]; then + RUN_MMSI_TRANSFORMERS=1 + else + append_vllm_task "mmsi" + fi + ;; + revsi) + RUN_REVSI=1 + ;; + spatial|grounding|refcoco|spatial_grounding_eval) + append_vllm_task "spatial_grounding" + ;; + spatial_temporal_grounding) + append_vllm_task "stvg" + ;; + temporal_grounding|timelens|timelens_bench) + RUN_TIMELENS=1 + ;; + segmentation|seg|reasonseg|refseg) + RUN_SEGMENTATION=1 + ;; + "") + ;; + *) + echo "ERROR: unknown TASK '${TASK}'. Supported: videomme,videommev2,videommmu,mmvu,mvbench,videoholmes,longvideobench,lvbench,mlvu,vsi,mmsi,mindcube,revsi,spatial_grounding,tracking,stvg,temporal_grounding,segmentation,all" >&2 + exit 1 + ;; + esac +done + +if [[ -n "$VLLM_TASKS" ]]; then + TASKS="$VLLM_TASKS" python eval/task/eval_vllm.py "$@" +fi + +if [[ "$RUN_MMSI_TRANSFORMERS" = "1" ]]; then + MMSI_FAMILY="$(model_family_tag "$MODEL_PATH")" + MMSI_CKPT="$(checkpoint_tag "$MODEL_PATH")" + MMSI_RUN_ROOT="${PROJECT_DIR}/outputs/${MMSI_FAMILY}/spatial_intelligence/mmsi/${MMSI_CKPT}/${MMSI_SETTING}/$(date +%Y%m%d_%H%M%S)" + CUDA_VISIBLE_DEVICES="$GPUS" \ + PROCESSOR_PATH="$MODEL_PATH" \ + DATA_FILE="$MMSI_DATA_FILE" \ + OUTPUT_DIR="$MMSI_RUN_ROOT" \ + IMAGE_MIN_PIXELS="$MMSI_IMAGE_MIN_PIXELS" \ + IMAGE_MAX_PIXELS="$MMSI_IMAGE_MAX_PIXELS" \ + MAX_IMAGES="$MMSI_MAX_IMAGES" \ + MAX_NEW_TOKENS="$MMSI_MAX_NEW_TOKENS" \ + MAX_SAMPLES="$MMSI_MAX_SAMPLES" \ + CATEGORY="$MMSI_CATEGORY" \ + ENABLE_THINKING="$MMSI_ENABLE_THINKING" \ + ATTN_IMPLEMENTATION="$MMSI_ATTN_IMPLEMENTATION" \ + EXPECTED_SAMPLES="$MMSI_EXPECTED_SAMPLES" \ + bash eval/task/mmsi/run_eval_transformers.sh "$MODEL_PATH" +fi + +if [[ "$RUN_REVSI" = "1" ]]; then + REVSI_FAMILY="$(model_family_tag "$MODEL_PATH")" + REVSI_CKPT="$(checkpoint_tag "$MODEL_PATH")" + REVSI_RUN_EXPECTED_SAMPLES="$REVSI_EXPECTED_SAMPLES" + if [[ "$REVSI_MAX_SAMPLES" != "0" || -n "$REVSI_TASK_FILTER" ]]; then + REVSI_RUN_EXPECTED_SAMPLES=0 + fi + REVSI_RUN_ROOT="${PROJECT_DIR}/outputs/${REVSI_FAMILY}/spatial_intelligence/revsi/${REVSI_CKPT}/${REVSI_SETTING}/$(date +%Y%m%d_%H%M%S)" + CUDA_VISIBLE_DEVICES="$GPUS" \ + REVSI_ROOT="$REVSI_VIDEO_ROOT" \ + QA_FILE="$REVSI_DATA_FILE" \ + VIDEO_ROOT="$REVSI_VIDEO_ROOT" \ + FRAME_BUDGET="$REVSI_FRAME_BUDGET" \ + OUTPUT_DIR="$REVSI_RUN_ROOT" \ + MAX_FRAMES="$REVSI_MAX_FRAMES" \ + EXACT_NFRAMES="$REVSI_EXACT_NFRAMES" \ + FPS="$REVSI_FPS" \ + VIDEO_MIN_PIXELS="$REVSI_VIDEO_MIN_PIXELS" \ + VIDEO_MAX_PIXELS="$REVSI_VIDEO_MAX_PIXELS" \ + VIDEO_TOTAL_PIXELS="$REVSI_VIDEO_TOTAL_PIXELS" \ + BATCH_SIZE="$REVSI_BATCH_SIZE" \ + MAX_MODEL_LEN="$REVSI_MAX_MODEL_LEN" \ + MAX_NEW_TOKENS="$REVSI_MAX_NEW_TOKENS" \ + GPU_MEM_UTIL="$REVSI_GPU_MEMORY_UTILIZATION" \ + MAX_SAMPLES="$REVSI_MAX_SAMPLES" \ + EXPECTED_SAMPLES="$REVSI_RUN_EXPECTED_SAMPLES" \ + TASK_FILTER="$REVSI_TASK_FILTER" \ + ENABLE_THINKING="$REVSI_ENABLE_THINKING" \ + TP_SIZE="$TP_SIZE" \ + bash eval/task/revsi/run_eval_vllm.sh "$MODEL_PATH" +fi + +if [[ "$RUN_TIMELENS" = "1" ]]; then + TIMELENS_FAMILY="$(model_family_tag "$MODEL_PATH")" + TIMELENS_CKPT="$(checkpoint_tag "$MODEL_PATH")" + TIMELENS_SETTING="timelens-fps${TIMELENS_FPS}-min${TIMELENS_MIN_TOKENS}-total${TIMELENS_TOTAL_TOKENS}-new${TIMELENS_MAX_NEW_TOKENS}-${TIMELENS_DATASETS//,/_}" + TIMELENS_RUN_ROOT="${PROJECT_DIR}/outputs/${TIMELENS_FAMILY}/temporal_grounding/${TIMELENS_CKPT}/${TIMELENS_SETTING}/$(date +%Y%m%d_%H%M%S)" + CUDA_VISIBLE_DEVICES="$GPUS" \ + DATASETS="$TIMELENS_DATASETS" \ + SETTING="$TIMELENS_SETTING" \ + OUTPUT_ROOT="$TIMELENS_RUN_ROOT" \ + TIMELENS_USE_OUTPUT_ROOT_DIRECT=1 \ + MIN_TOKENS="$TIMELENS_MIN_TOKENS" \ + TOTAL_TOKENS="$TIMELENS_TOTAL_TOKENS" \ + MAX_PIXELS="$TIMELENS_MAX_PIXELS" \ + MAX_FRAMES="$TIMELENS_MAX_FRAMES" \ + FPS="$TIMELENS_FPS" \ + MAX_NEW_TOKENS="$TIMELENS_MAX_NEW_TOKENS" \ + REPETITION_PENALTY="$TIMELENS_REPETITION_PENALTY" \ + STOP_AFTER_ANSWER="$TIMELENS_STOP_AFTER_ANSWER" \ + PROMPT_MODE="$TIMELENS_PROMPT_MODE" \ + NUM_WORKERS="$TIMELENS_NUM_WORKERS" \ + bash eval/task/temporal_grounding/run_eval.sh \ + "$MODEL_PATH" \ + "$TIMELENS_ENABLE_THINKING" \ + "$TIMELENS_RUN_ROOT" +fi + +if [[ "$RUN_SEGMENTATION" = "1" ]]; then + SEG_FAMILY="$(model_family_tag "$MODEL_PATH")" + SEG_CKPT="$(checkpoint_tag "$MODEL_PATH")" + SEG_RUN_ROOT="${PROJECT_DIR}/outputs/${SEG_FAMILY}/segmentation/${SEG_CKPT}/${SEGMENTATION_SETTING}/$(date +%Y%m%d_%H%M%S)" + CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-$GPUS}" \ + PROCESSOR_PATH="$SEGMENTATION_PROCESSOR_PATH" \ + BENCH_DIR="$SEGMENTATION_BENCH_DIR" \ + DATA_ROOT="$SEGMENTATION_DATA_ROOT" \ + DATASETS="$SEGMENTATION_DATASETS" \ + DATA_TYPE="$SEGMENTATION_DATA_TYPE" \ + PROMPT_MODE="$SEGMENTATION_PROMPT_MODE" \ + ENABLE_THINKING="$SEGMENTATION_ENABLE_THINKING" \ + BATCH_SIZE="$SEGMENTATION_BATCH_SIZE" \ + MAX_NEW_TOKENS="$SEGMENTATION_MAX_NEW_TOKENS" \ + MAX_MODEL_LEN="$SEGMENTATION_MAX_MODEL_LEN" \ + MAX_PIXELS_IMAGE="$SEGMENTATION_MAX_PIXELS_IMAGE" \ + MIN_PIXELS_IMAGE="$SEGMENTATION_MIN_PIXELS_IMAGE" \ + VIDEO_MAX_PIXELS="$SEGMENTATION_VIDEO_MAX_PIXELS" \ + VIDEO_MIN_PIXELS="$SEGMENTATION_VIDEO_MIN_PIXELS" \ + VIDEO_TOTAL_PIXELS="$SEGMENTATION_VIDEO_TOTAL_PIXELS" \ + MAX_FRAMES="$SEGMENTATION_MAX_FRAMES" \ + FPS="$SEGMENTATION_FPS" \ + VIDEO_READER="$SEGMENTATION_VIDEO_READER" \ + TP_SIZE="$TP_SIZE" \ + GPU_MEM_UTIL="$SEGMENTATION_GPU_MEM_UTIL" \ + SEED="$SEGMENTATION_SEED" \ + MAX_SAMPLES="$SEGMENTATION_MAX_SAMPLES" \ + RUN_SAM2="$SEGMENTATION_RUN_SAM2" \ + SAM2_CKPT="$SEGMENTATION_SAM2_CKPT" \ + SAM2_CFG="$SEGMENTATION_SAM2_CFG" \ + ONETHINKER_SEG_POST="$SEGMENTATION_POSTPROCESSOR_PATH" \ + SAM2_NUM_GPUS="$SEGMENTATION_SAM2_NUM_GPUS" \ + SAM2_WORKERS_PER_GPU="$SEGMENTATION_SAM2_WORKERS_PER_GPU" \ + bash eval/task/segmentation/run_eval_vllm.sh \ + "$MODEL_PATH" \ + "$SEG_RUN_ROOT" +fi diff --git a/code/eval/task/eval_image_mc_vllm.py b/code/eval/task/eval_image_mc_vllm.py new file mode 100644 index 0000000000000000000000000000000000000000..8a8e1c78479d994fcd77817f75fde25b599e09a7 --- /dev/null +++ b/code/eval/task/eval_image_mc_vllm.py @@ -0,0 +1,511 @@ +#!/usr/bin/env python3 +"""Unified vLLM evaluator for image-sequence multiple-choice benchmarks. + +Supported benchmarks: + - MMSI-Bench: TSV/parquet with base64 image list in `image`. + - MindCube-Tiny: official 1,050-sample Hugging Face split. +""" + +from __future__ import annotations + +import argparse +import ast +import base64 +import csv +import io +import json +import os +import re +import sys +import time +from collections import defaultdict +from typing import Any, Dict, List + +from canonical_data import load_json_records +from mindcube.data_utils import ( + OFFICIAL_DATA_SOURCE, + decode_mindcube_images, + load_mindcube_records, + mindcube_answer_value, + mindcube_group, + mindcube_prompt, +) +from PIL import Image + +CHOICES = list("ABCDEFGH") +ANSWER_RE = re.compile(r"\s*(.*?)\s*", re.DOTALL | re.IGNORECASE) +TAG_RE = re.compile(r"", re.IGNORECASE) +FORMAT_PRIORITY = { + "start": 10, + "end": 9, + "phrase": 7, + "parentheses": 6, + "period": 5, + "colon": 4, + "right_paren": 3, + "space": 2, + "fallback": 0, +} +ANSWER_PHRASES = [ + "the answer is", + "answer is", + "the correct answer is", + "correct answer is", + "the best answer is", + "best answer is", + "the correct option is", + "correct option is", + "i choose", + "i select", + "my answer is", + "答案是", + "答案为", +] +PROMPT_TAIL = ( + "Choose the best answer from the options. " + "Put exactly one uppercase option letter inside ... " + "Do not explain. Example: A" +) + + +def strip_think_block(text: str) -> str: + text = (text or "").strip() + if not text: + return "" + parts = re.split(r"", text, flags=re.IGNORECASE) + if len(parts) > 1: + text = parts[-1] + else: + text = re.sub(r".*?", "", text, flags=re.DOTALL | re.IGNORECASE) + return TAG_RE.sub("", text).strip() + + +def strip_answer_tags(text: str) -> str: + matches = ANSWER_RE.findall(text or "") + if matches: + return matches[-1].strip() + return (text or "").strip() + + +def extract_mcq_answer(response: str, choices: List[str] | None = None) -> str: + if not response or not response.strip(): + return "" + all_choices = choices or CHOICES + text = strip_answer_tags(strip_think_block(response)).strip() + if not text: + return "" + for char in [",", ".", "!", "?", ";", ":", "'", '"', "。", "："]: + text = text.strip(char) + padded = " " + text + " " + candidates = [] + + for ch in all_choices: + for token, fmt in ( + (f"({ch})", "parentheses"), + (f"{ch}.", "period"), + (f"{ch}:", "colon"), + (f"{ch})", "right_paren"), + (f"{ch} ", "space"), + ): + pos = padded.rfind(token) + if pos != -1: + candidates.append((ch, pos, fmt)) + + lower = padded.lower() + for phrase in ANSWER_PHRASES: + idx = lower.rfind(phrase.lower()) + if idx != -1: + after = idx + len(phrase) + for ch in all_choices: + m = re.search(rf"\b{re.escape(ch)}\b", padded[after:], flags=re.IGNORECASE) + if m: + candidates.append((ch, after + m.start(), "phrase")) + + stripped = padded.strip() + for ch in all_choices: + if stripped.upper() == ch: + candidates.append((ch, 0, "start")) + elif stripped.startswith(ch) and (len(stripped) == 1 or not stripped[1].isalpha()): + candidates.append((ch, 0, "start")) + elif stripped.endswith(ch) and (len(stripped) == 1 or not stripped[-2].isalpha()): + candidates.append((ch, len(padded) - 1, "end")) + + if not candidates: + for ch in all_choices: + m = re.search(rf"\b{re.escape(ch)}\b", padded) + if m: + candidates.append((ch, m.start(), "fallback")) + if not candidates: + return "" + candidates.sort(key=lambda x: (FORMAT_PRIORITY.get(x[2], 0), x[1]), reverse=True) + return candidates[0][0] + + +def load_records(args) -> List[Dict[str, Any]]: + if args.bench == "mindcube": + return load_mindcube_records( + args.data_file, + chunk=args.chunk, + index=args.index, + expected_samples=args.expected_samples, + cache_dir=args.hf_cache_dir or None, + ) + if args.data_file.endswith((".jsonl", ".json")): + return load_json_records(args.data_file) + if args.data_file.endswith(".parquet"): + import pandas as pd + + return pd.read_parquet(args.data_file).to_dict("records") + + csv.field_size_limit(sys.maxsize) + with open(args.data_file, encoding="utf-8", errors="replace", newline="") as f: + return list(csv.DictReader(f, delimiter="\t")) + + +def shard(records: List[Dict[str, Any]], chunk: int, index: int) -> List[Dict[str, Any]]: + if chunk <= 1: + return records + return records[index::chunk] + + +def decode_mmsi_images(value: Any, max_images: int) -> List[Image.Image]: + images_b64 = ast.literal_eval(value) if isinstance(value, str) else value + if not isinstance(images_b64, (list, tuple)): + images_b64 = [images_b64] + images = [] + for item in images_b64[:max_images]: + if not item: + continue + path = "" + if isinstance(item, dict): + path = str( + item.get("path") + or item.get("image") + or item.get("image_path") + or "" + ) + elif isinstance(item, str) and os.path.isfile(item): + path = item + if path: + images.append(Image.open(path).convert("RGB")) + continue + raw = item if isinstance(item, bytes) else base64.b64decode(str(item)) + images.append(Image.open(io.BytesIO(raw)).convert("RGB")) + return images + + +def normalize_choices(value: Any) -> List[str]: + if value is None: + return [] + if hasattr(value, "tolist"): + value = value.tolist() + if isinstance(value, (list, tuple)): + return [str(v) for v in value] + return [str(value)] + + +def answer_choices(choices: List[Any] | None) -> List[str]: + n = len(choices) if choices is not None else 0 + return CHOICES[:n] if n > 0 else list("ABCD") + + +def choices_from_question(question: str) -> List[str]: + found = re.findall(r"(?:^|[\s,])([A-H])\s*[:\).]", question or "") + out = [] + for ch in found: + if ch not in out: + out.append(ch) + return out or list("ABCD") + + +def build_prompt(args, rec: Dict[str, Any]) -> tuple[str, List[str]]: + if args.bench == "mmsi": + question = str(rec.get("question") or "").strip() + labels = choices_from_question(question) + return f"{question}\n{PROMPT_TAIL}", labels + + return mindcube_prompt(rec, PROMPT_TAIL) + + +def get_answer(args, rec: Dict[str, Any]) -> str: + answer = ( + mindcube_answer_value(rec) + if args.bench == "mindcube" + else rec.get("answer") + ) + if answer is None: + return "" + if args.bench == "mindcube" and isinstance(answer, (int, float)) and not isinstance(answer, bool): + idx = int(answer) + labels = answer_choices(normalize_choices(rec.get("choices"))) + if 0 <= idx < len(labels): + return labels[idx] + return extract_mcq_answer(str(answer), CHOICES) or str(answer).strip().upper()[:1] + + +def get_group_key(args, rec: Dict[str, Any]) -> str: + if args.bench == "mmsi": + return str(rec.get("category") or "unknown") + return mindcube_group(rec) + + +def get_filter_key(args) -> str: + return "category" if args.bench == "mmsi" else "task" + + +def render_chat_prompt(messages, processor, enable_thinking: bool = False) -> str: + return processor.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True, + enable_thinking=enable_thinking, + ) + + +def prepare_for_vllm(images: List[Image.Image], prompt: str, processor, args): + from qwen_vl_utils import process_vision_info + + content: List[Dict[str, Any]] = [] + for image in images: + item: Dict[str, Any] = { + "type": "image", + "image": image, + "max_pixels": args.image_max_pixels, + } + if args.image_min_pixels > 0: + item["min_pixels"] = args.image_min_pixels + content.append(item) + content.append({"type": "text", "text": prompt}) + messages = [{"role": "user", "content": content}] + text = render_chat_prompt(messages, processor, args.enable_thinking) + image_inputs, _video_inputs = process_vision_info(messages, image_patch_size=args.patch_size) + return { + "prompt": text, + "multi_modal_data": {"image": image_inputs}, + "mm_processor_kwargs": {"do_resize": False}, + } + + +def pct(correct: int, total: int) -> float: + return round(100.0 * correct / total, 2) if total else 0.0 + + +def aggregate(args, results: List[Dict[str, Any]]) -> Dict[str, Any]: + correct = sum(int(r.get("score", 0)) for r in results) + parsed = sum(1 for r in results if r.get("pred_answer")) + groups = defaultdict(lambda: {"correct": 0, "total": 0}) + for r in results: + groups[str(r.get("group") or "unknown")]["total"] += 1 + groups[str(r.get("group") or "unknown")]["correct"] += int(r.get("score", 0)) + group_name = "by_category" if args.bench == "mmsi" else "by_task" + return { + "num_samples": len(results), + "correct": correct, + "accuracy": pct(correct, len(results)), + "parse_rate": pct(parsed, len(results)), + group_name: { + k: {"accuracy": pct(v["correct"], v["total"]), "correct": v["correct"], "total": v["total"]} + for k, v in sorted(groups.items()) + }, + } + + +def decode_images_for_record(args, rec: Dict[str, Any]) -> List[Image.Image]: + if args.bench == "mmsi": + return decode_mmsi_images( + rec.get("images") or rec.get("image"), + args.max_images, + ) + return decode_mindcube_images(rec, args.max_images, args.min_image_bytes) + + +def evaluate(llm, sampling_params, processor, args) -> Dict[str, Any]: + records = load_records(args) + filter_value = args.category if args.bench == "mmsi" else args.task + if filter_value: + if args.bench == "mindcube": + records = [ + record + for record in records + if mindcube_group(record) == filter_value + ] + else: + key = get_filter_key(args) + records = [ + record + for record in records + if str(record.get(key) or "") == filter_value + ] + if args.max_samples and args.max_samples > 0: + records = records[: args.max_samples] + if args.bench == "mmsi": + records = shard(records, args.chunk, args.index) + print(f"Loaded {len(records)} {args.bench} samples for shard {args.index}/{args.chunk}", flush=True) + + results: List[Dict[str, Any]] = [] + t0 = time.time() + for start in range(0, len(records), args.batch_size): + batch = records[start:start + args.batch_size] + inputs = [] + keep = [] + prompts: Dict[int, str] = {} + labels_by_j: Dict[int, List[str]] = {} + for j, rec in enumerate(batch): + try: + images = decode_images_for_record(args, rec) + if not images: + raise ValueError("no decoded images") + prompt, labels = build_prompt(args, rec) + inputs.append(prepare_for_vllm(images, prompt, processor, args)) + keep.append(j) + prompts[j] = prompt + labels_by_j[j] = labels + except Exception as e: + print(f"[warn] skip sample {start + j}: {type(e).__name__}: {e}", flush=True) + + outputs = llm.generate(inputs, sampling_params=sampling_params) if inputs else [] + out_by_j = {j: out for j, out in zip(keep, outputs)} + + for j, rec in enumerate(batch): + if j not in out_by_j: + continue + raw = out_by_j[j].outputs[0].text + labels = labels_by_j[j] + pred = extract_mcq_answer(raw, labels) + gt = get_answer(args, rec) + score = 1.0 if pred and gt and pred.upper() == gt.upper() else 0.0 + results.append({ + "id": str(rec.get("index") or rec.get("id") or rec.get("sample_id") or start + j), + "bench": args.bench, + "group": get_group_key(args, rec), + "question": rec.get("question"), + "prompt": prompts.get(j, ""), + "answer": gt, + "pred_answer": pred, + "raw_prediction": raw, + "score": score, + }) + + done = min(start + args.batch_size, len(records)) + if done % 200 == 0 or done == len(records): + elapsed = max(time.time() - t0, 1e-6) + summary = aggregate(args, results) + print( + f"[{done}/{len(records)}] {elapsed:.1f}s " + f"acc={summary['accuracy']:.2f}% parse={summary['parse_rate']:.2f}%", + flush=True, + ) + + full_mindcube_eval = ( + args.bench == "mindcube" + and args.chunk == 1 + and not args.task + and not args.max_samples + ) + if ( + full_mindcube_eval + and args.expected_samples > 0 + and len(results) != args.expected_samples + ): + raise RuntimeError( + f"Expected {args.expected_samples} official MindCube-Tiny " + f"results, got {len(results)}." + ) + + out_name = f"results_{args.bench}" + if args.chunk > 1: + out_name += f"_shard{args.index}" + out_path = os.path.join(args.output_dir, out_name + ".json") + with open(out_path, "w", encoding="utf-8") as f: + json.dump(results, f, ensure_ascii=False, indent=2) + + summary = aggregate(args, results) + if args.bench == "mindcube": + summary["data_source"] = args.data_file + if full_mindcube_eval and args.expected_samples > 0: + summary["expected_samples"] = args.expected_samples + summary["coverage"] = round( + 100.0 * len(results) / args.expected_samples, 2 + ) + summary_name = f"summary_shard{args.index}.json" if args.chunk > 1 else "summary.json" + with open(os.path.join(args.output_dir, summary_name), "w", encoding="utf-8") as f: + json.dump(summary, f, ensure_ascii=False, indent=2) + print(json.dumps(summary, ensure_ascii=False, indent=2), flush=True) + return summary + + +def parse_args(): + p = argparse.ArgumentParser(description="Unified image-sequence MC evaluation via vLLM.") + p.add_argument("--bench", required=True, choices=["mmsi", "mindcube"]) + p.add_argument("--model_path", required=True) + p.add_argument("--processor_path", default="") + p.add_argument("--data_file", default="") + p.add_argument("--output_dir", required=True) + p.add_argument("--image_min_pixels", type=int, default=4096) + p.add_argument("--image_max_pixels", type=int, default=262144) + p.add_argument("--max_images", type=int, default=8) + p.add_argument("--min_image_bytes", type=int, default=1024) + p.add_argument("--patch_size", type=int, default=16) + p.add_argument("--tensor_parallel_size", type=int, default=1) + p.add_argument("--gpu_memory_utilization", type=float, default=0.95) + p.add_argument("--max_model_len", type=int, default=65536) + p.add_argument("--max_new_tokens", type=int, default=64) + p.add_argument("--max_num_batched_tokens", type=int, default=65536) + p.add_argument("--batch_size", type=int, default=16) + p.add_argument("--max_samples", type=int, default=0) + p.add_argument("--expected_samples", type=int, default=1050) + p.add_argument("--hf_cache_dir", default="") + p.add_argument("--category", default="") + p.add_argument("--task", default="") + p.add_argument("--temperature", type=float, default=0.0) + p.add_argument("--top_p", type=float, default=1.0) + p.add_argument("--top_k", type=int, default=-1) + p.add_argument("--chunk", type=int, default=1) + p.add_argument("--index", type=int, default=0) + p.add_argument("--enable_thinking", action="store_true") + return p.parse_args() + + +def main(): + args = parse_args() + if args.bench == "mindcube" and not args.data_file: + args.data_file = OFFICIAL_DATA_SOURCE + if not args.data_file: + raise ValueError("--data_file is required for MMSI-Bench") + args.processor_path = args.processor_path or args.model_path + os.makedirs(args.output_dir, exist_ok=True) + + from transformers import AutoProcessor, AutoTokenizer + from vllm import LLM, SamplingParams + + processor = AutoProcessor.from_pretrained( + args.processor_path, + padding_side="left", + trust_remote_code=True, + min_pixels=args.image_min_pixels, + max_pixels=args.image_max_pixels, + ) + tokenizer = AutoTokenizer.from_pretrained(args.processor_path, trust_remote_code=True) + tokenizer.padding_side = "left" + processor.tokenizer = tokenizer + + llm = LLM( + model=args.model_path, + tensor_parallel_size=args.tensor_parallel_size, + gpu_memory_utilization=args.gpu_memory_utilization, + max_model_len=args.max_model_len, + max_num_batched_tokens=args.max_num_batched_tokens, + trust_remote_code=True, + limit_mm_per_prompt={"image": args.max_images}, + ) + sampling = SamplingParams( + max_tokens=args.max_new_tokens, + temperature=args.temperature, + top_p=args.top_p, + top_k=args.top_k, + ) + evaluate(llm, sampling, processor, args) + + +if __name__ == "__main__": + main() diff --git a/code/eval/task/eval_prompt.py b/code/eval/task/eval_prompt.py new file mode 100644 index 0000000000000000000000000000000000000000..942d75f63b14729ccfc468d4ddf6767f41a85d44 --- /dev/null +++ b/code/eval/task/eval_prompt.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python3 +"""Single source of truth for evaluation prompts. + +Exactly ONE prompt per task — the one used by ``eval/task/eval.sh`` — kept +byte-for-byte aligned with the SFT training data in +``data/joint/sft_joint_all.jsonl``. + +Task -> prompt mapping (verbatim from sft_joint_all.jsonl): + + video_qa_mc (videomme, videommmu, mmvu, mvbench, videoholmes, + longvideobench, lvbench, mlvu) + "{question}\\nOptions:\\n{opts}\\n" + VIDEO_QA_MC_TAIL + + spatial_intelligence (vsi) + MC -> "{q}\\nOptions:\\n{opts}\\n" + VSI_MC_TAIL + integer -> "{q} " + VSI_INTEGER_TAIL + meters -> "{q} " + VSI_METERS_TAIL + cm -> "{q} " + VSI_CENTIMETERS_TAIL + room m2 -> "{q} " + VSI_SQUARE_METERS_TAIL + (each prefixed with "These are frames of a video.\\n") + + sensenova_si / image_sequence_mc (mmsi, mindcube) + "{q}\\n" + SENSENOVA_MC_TAIL (alias PROMPT_TAIL) + + spatial grounding (spatial_grounding) + build_spatial_grounding_prompt(expr) == QWEN_NATIVE_PROMPT_SG + + tracking + GROUNDING_QUESTION_TEMPLATE_NO_THINK.format(Question=q) + TRACKING_TAIL + + stvg + GROUNDING_QUESTION_TEMPLATE_NO_THINK.format( + Question=TRAIN_STVG_QUESTION_PREFIX.format(query=q)) + STVG_TAIL + + segmentation + "{question}\\n" + TRAIN_SEG_IMAGE_TAIL / TRAIN_SEG_VIDEO_TAIL + + temporal grounding (temporal_grounding) + TEMPORAL_GROUNDING_PROMPT.format(event) + +Keep this file dependency-free (stdlib only) so it can be imported from every +worker regardless of the runtime environment. +""" + +# =========================================================================== +# Multiple-choice / numeric answer tails +# =========================================================================== + +# video_qa_mc — videomme, videommmu, mmvu, mvbench, videoholmes, +# longvideobench, lvbench, mlvu. +VIDEO_QA_MC_TAIL = ( + "Answer with the option letter only within ... tags. " + "Example: A" +) + +# spatial_intelligence multiple choice — vsi (object_rel_distance, +# object_rel_direction, obj_appearance_order, route_plan, ...). +VSI_MC_TAIL = ( + "Answer with the option letter within ... tags. " + "Example: A" +) + +# spatial_intelligence numeric — vsi. +VSI_INTEGER_TAIL = ( + "Answer with an integer within ... tags. Example: 3" +) +VSI_METERS_TAIL = ( + "Answer with a number in meters within ... tags. Example: 2.3" +) +VSI_CENTIMETERS_TAIL = ( + "Answer with a number in centimeters within ... tags. Example: 120.5" +) +VSI_SQUARE_METERS_TAIL = ( + "Answer with a number in square meters within ... tags. Example: 25.5" +) + +# sensenova_si / image_sequence_mc_answer_only — mmsi, mindcube. +SENSENOVA_MC_TAIL = ( + "Choose the best answer from the options. " + "Put exactly one uppercase option letter inside ... " + "Do not explain. Example: A" +) +# Backwards-compatible alias used throughout eval_vllm.py. +PROMPT_TAIL = SENSENOVA_MC_TAIL + + +# =========================================================================== +# Tracking +# =========================================================================== + +# Question scaffold shared by tracking + stvg (the tail carries the full +# answer-format spec). +GROUNDING_QUESTION_TEMPLATE_NO_THINK = ( + "{Question}\n" + "Please answer this question based on the visual content. " +) + +TRACKING_TAIL = ( + "Please track the target object throughout the video and provide one bounding box per second, " + "ONLY up to 32 seconds, within the ... tags.\n" + "Example:\n" + "{\"boxes\": {\"1\": [405, 230, 654, 463], \"2\": [435, 223, 678, 446], " + "\"32\": [415, 203, 691, 487]}}\n" + "Note: Each key in 'boxes' must correspond to a second (1, 2, 3, ..., 32) " + "and contain a 4-number bounding box [x1, y1, x2, y2]." +) + + +# =========================================================================== +# Spatial-temporal grounding (STVG) +# =========================================================================== + +STVG_TAIL = ( + "Please provide only the time span in seconds and bounding boxes as JSON " + "within the ... tags.\n" + "You MUST output one bounding box for every integer second within the " + "given time span (inclusive).\n" + "Example:\n" + "{\"time\": [8.1, 13.5], \"boxes\": {\"9\": [317, 422, 582, 997], " + "\"10\": [332, 175, 442, 369], \"11\": [340, 180, 450, 370]}}\n" + "Note: Each key in 'boxes' must be an integer second within the span, " + "and its value must be a 4-number bounding box [x1, y1, x2, y2]." +) + +# Rewrites an eval question into the exact wording used during STVG training. +TRAIN_STVG_QUESTION_PREFIX = ( + 'Given the query "{query}", when and where does the described content ' + 'occur in the video? please firstly give the start and end time, spatial ' + 'bounding box corresponding to each integer second.' +) + + +# =========================================================================== +# Segmentation +# =========================================================================== + +# Matches sft_joint_all.jsonl seg_image / seg_video samples. +TRAIN_SEG_IMAGE_TAIL = ( + "Please answer this question based on the visual content. " + "This task prepares inputs for image object segmentation with a specialized model (e.g., SAM2).\n" + "Please provide ONE bounding box, 3 positive points (clearly INSIDE the object), " + "and 3 negative points (clearly OUTSIDE the object) within the ... tags.\n" + "Choose informative points that help distinguish object vs. background. Prefer negatives on clear non-object " + "pixels INSIDE the box when safe; otherwise place them just outside on obvious background. " + "Negatives must NEVER be on the object or on its boundary.\n" + "Example: {\"boxes\": [x1, y1, x2, y2], \"positive_points\": [[x,y],[x,y],[x,y]], " + "\"negative_points\": [[x,y],[x,y],[x,y]]}" +) + +TRAIN_SEG_VIDEO_TAIL = ( + "Please answer this question based on the visual content. " + "This task prepares inputs for video object segmentation with a specialized model (e.g., SAM2).\n" + "Please select ONE representative time (in seconds), and provide ONE bounding box, " + "3 positive points (clearly INSIDE the object), and 3 negative points (clearly OUTSIDE the object) " + "within the ... tags.\n" + "Choose informative points that help distinguish object vs. background. Prefer negatives on clear non-object " + "pixels INSIDE the box when safe; otherwise place them just outside on obvious background. " + "Negatives must NEVER be on the object or on its boundary.\n" + "Example: {\"time\": , \"boxes\": [x1, y1, x2, y2], " + "\"positive_points\": [[x,y],[x,y],[x,y]], \"negative_points\": [[x,y],[x,y],[x,y]]}" +) + + +# =========================================================================== +# Spatial grounding (RefCOCO / spatial_grounding) +# =========================================================================== + +# Matches sft_joint_all.jsonl spatial grounding samples after the token. +QWEN_NATIVE_PROMPT_SG = ( + 'Locate "{}" in the image. Output its bounding box in JSON format ' + 'within ... tags. ' + 'Example: [{{"bbox_2d": [123, 30, 404, 846]}}]' +) + + +# =========================================================================== +# Temporal grounding (TimeLens / temporal_grounding) +# =========================================================================== + +# Matches sft_joint_all.jsonl temporal grounding samples (after