Initial commit

Co-authored-by: TencentOpen <TencentOpen@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,38 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+PE/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+assets/method_framework_h.png filter=lfs diff=lfs merge=lfs -text
+assets/method_framework_h_cn.png filter=lfs diff=lfs merge=lfs -text

DiT/base_ar/diffusion_pytorch_model_fp32.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aca12a37bd1e829a45491b9b7b13ee7ca8c202ebe99954f7a2dee417a7f0132e
+size 34213991280

DiT/distilled/diffusion_pytorch_model_ema_0_995.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73819bcf3b82b090116dd1a1748795de95eef703c304e7443963d00ea8aa4d70
+size 34213991280

DiT/distilled_int8/diffusion_pytorch_model_ema_0_995_int8.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:973fa409f3b009232db3c83feddc89c5326fc3a5672feb791f0989ea8bf83252
+size 9059126120

DiT/lora_weights/cinematic/adapter.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:765426a6c417eed70a57d72f67cfe002076cfc92b6b35fe185091012ba80c7c2
+size 6122591632

DiT/lora_weights/cinematic/adapter_config.json

@@ -0,0 +1,25 @@
+{
+  "format_version": 1,
+  "rank": 256,
+  "alpha": 512.0,
+  "dropout": 0.0,
+  "targets": [
+    "double_blocks.*.img_attn_q",
+    "double_blocks.*.img_attn_k",
+    "double_blocks.*.img_attn_v",
+    "double_blocks.*.img_attn_proj",
+    "double_blocks.*.txt_attn_q",
+    "double_blocks.*.txt_attn_k",
+    "double_blocks.*.txt_attn_v",
+    "double_blocks.*.txt_attn_proj",
+    "double_blocks.*.img_mlp.fc1",
+    "double_blocks.*.img_mlp.fc2",
+    "double_blocks.*.txt_mlp.fc1",
+    "double_blocks.*.txt_mlp.fc2",
+    "double_blocks.*.img_mod.linear",
+    "double_blocks.*.txt_mod.linear",
+    "action_modules.*.img_attn_q",
+    "action_modules.*.keyboard_attn_kv",
+    "action_modules.*.proj_keyboard"
+  ]
+}

DiT/lora_weights/dual_arm_robotics/adapter.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c96a8cefbf93ecd563fe34497f9aaeb3d353cf81773802663c22ddc47ed909d2
+size 12244965688

DiT/lora_weights/dual_arm_robotics/adapter_config.json

@@ -0,0 +1,25 @@
+{
+  "format_version": 1,
+  "rank": 512,
+  "alpha": 512.0,
+  "dropout": 0.0,
+  "targets": [
+    "double_blocks.*.img_attn_q",
+    "double_blocks.*.img_attn_k",
+    "double_blocks.*.img_attn_v",
+    "double_blocks.*.img_attn_proj",
+    "double_blocks.*.txt_attn_q",
+    "double_blocks.*.txt_attn_k",
+    "double_blocks.*.txt_attn_v",
+    "double_blocks.*.txt_attn_proj",
+    "double_blocks.*.img_mlp.fc1",
+    "double_blocks.*.img_mlp.fc2",
+    "double_blocks.*.txt_mlp.fc1",
+    "double_blocks.*.txt_mlp.fc2",
+    "double_blocks.*.img_mod.linear",
+    "double_blocks.*.txt_mod.linear",
+    "action_modules.*.img_attn_q",
+    "action_modules.*.keyboard_attn_kv",
+    "action_modules.*.proj_keyboard"
+  ]
+}

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

PE/chat_template.jinja

@@ -0,0 +1,170 @@
+{%- set image_count = namespace(value=0) %}
+{%- set video_count = namespace(value=0) %}
+{%- macro render_content(content, do_vision_count, is_system_content=false) %}
+    {%- if content is string %}
+        {{- content }}
+    {%- elif content is iterable and content is not mapping %}
+        {%- for item in content %}
+            {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain images.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set image_count.value = image_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Picture ' ~ image_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|image_pad|><|vision_end|>' }}
+            {%- elif 'video' in item or item.type == 'video' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain videos.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set video_count.value = video_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Video ' ~ video_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|video_pad|><|vision_end|>' }}
+            {%- elif 'text' in item %}
+                {{- item.text }}
+            {%- else %}
+                {{- raise_exception('Unexpected item type in content.') }}
+            {%- endif %}
+        {%- endfor %}
+    {%- elif content is none or content is undefined %}
+        {{- '' }}
+    {%- else %}
+        {{- raise_exception('Unexpected content type.') }}
+    {%- endif %}
+{%- endmacro %}
+{%- if not messages %}
+    {{- raise_exception('No messages provided.') }}
+{%- endif %}
+{%- set reasoning_instructions = '' %}
+{%- if enable_thinking is undefined or enable_thinking is true %}
+    {%- set resolved_reasoning_effort = reasoning_effort|default('xhigh') %}
+    {%- if resolved_reasoning_effort not in ('xhigh', 'medium', 'low') %}
+        {{- raise_exception('Unexpected reasoning effort ' ~ reasoning_effort ~ '. Supported types are xhigh (default), medium, and low.') }}
+    {%- endif %}
+    {%- if resolved_reasoning_effort == 'xhigh' %}
+        {%- set reasoning_instructions = 'Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.' %}
+    {%- elif resolved_reasoning_effort == 'low' %}
+        {%- set reasoning_instructions = 'Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the conclusion without unnecessary elaboration.' %}
+    {%- endif %}
+{%- endif %}
+{%- if tools and tools is iterable and tools is not mapping %}
+    {{- '<|im_start|>system\n' }}
+    {%- if reasoning_instructions %}
+        {{- reasoning_instructions + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>" }}
+    {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {%- if content %}
+            {{- '\n\n' + content }}
+        {%- endif %}
+    {%- endif %}
+    {{- '<|im_end|>\n' }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {%- if content %}
+            {{- '<|im_start|>system\n' + (reasoning_instructions + '\n\n' if reasoning_instructions else '')  + content + '<|im_end|>\n' }}
+        {%- elif reasoning_instructions %}
+            {{- '<|im_start|>system\n' + reasoning_instructions + '<|im_end|>\n' }}
+        {%- endif %}
+    {%- elif reasoning_instructions %}
+        {{- '<|im_start|>system\n' + reasoning_instructions + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" %}
+        {%- set content = render_content(message.content, false)|trim %}
+        {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+            {%- set ns.multi_step_tool = false %}
+            {%- set ns.last_query_index = index %}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if ns.multi_step_tool %}
+    {{- raise_exception('No user query found in messages.') }}
+{%- endif %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content, true)|trim %}
+    {%- if message.role == "system" %}
+        {%- if not loop.first %}
+            {{- raise_exception('System message must be at the beginning.') }}
+        {%- endif %}
+    {%- elif message.role == "user" %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- endif %}
+        {%- set reasoning_content = reasoning_content|trim %}
+        {%- if preserve_thinking is undefined or preserve_thinking is true or loop.index0 > ns.last_query_index %}
+            {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if tool_call.function is defined %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {%- if loop.first %}
+                    {%- if content|trim %}
+                        {{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- else %}
+                        {{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- endif %}
+                {%- else %}
+                    {{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                {%- endif %}
+                {%- if tool_call.arguments is defined and tool_call.arguments != '' %}
+                    {%- for args_name, args_value in tool_call.arguments|items %}
+                        {{- '<parameter=' + args_name + '>\n' }}
+                        {%- set args_value = args_value | string if args_value is string else args_value | tojson | safe %}
+                        {{- args_value }}
+                        {{- '\n</parameter>\n' }}
+                    {%- endfor %}
+                {%- endif %}
+                {{- '</function>\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.previtem and loop.previtem.role != "tool" %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if not loop.last and loop.nextitem.role != "tool" %}
+            {{- '<|im_end|>\n' }}
+        {%- elif loop.last %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- else %}
+        {{- raise_exception('Unexpected message role.') }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- else %}
+        {{- '<think>\n' }}
+    {%- endif %}
+{%- endif %}

PE/config.json

@@ -0,0 +1,141 @@
+{
+  "architectures": [
+    "Qwen3_5ForConditionalGeneration"
+  ],
+  "image_token_id": 248056,
+  "language_model_only": false,
+  "model_type": "qwen3_5",
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "attn_output_gate": true,
+    "bos_token_id": 248044,
+    "dtype": "bfloat16",
+    "eos_token_id": 248044,
+    "full_attention_interval": 4,
+    "head_dim": 256,
+    "hidden_act": "silu",
+    "hidden_size": 5120,
+    "initializer_range": 0.02,
+    "intermediate_size": 17408,
+    "layer_types": [
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention"
+    ],
+    "linear_conv_kernel_dim": 4,
+    "linear_key_head_dim": 128,
+    "linear_num_key_heads": 16,
+    "linear_num_value_heads": 48,
+    "linear_value_head_dim": 128,
+    "mamba_ssm_dtype": "float32",
+    "max_position_embeddings": 262144,
+    "model_type": "qwen3_5_text",
+    "mtp_num_hidden_layers": 1,
+    "mtp_use_dedicated_embeddings": false,
+    "num_attention_heads": 24,
+    "num_hidden_layers": 64,
+    "num_key_value_heads": 4,
+    "output_gate_type": "swish",
+    "pad_token_id": null,
+    "partial_rotary_factor": 0.25,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "mrope_interleaved": true,
+      "mrope_section": [
+        11,
+        11,
+        10
+      ],
+      "partial_rotary_factor": 0.25,
+      "rope_theta": 10000000,
+      "rope_type": "default"
+    },
+    "tie_word_embeddings": false,
+    "use_cache": true,
+    "vocab_size": 248320
+  },
+  "tie_word_embeddings": false,
+  "transformers_version": "5.2.0",
+  "video_token_id": 248057,
+  "vision_config": {
+    "deepstack_visual_indexes": [],
+    "depth": 27,
+    "dtype": "bfloat16",
+    "hidden_act": "gelu_pytorch_tanh",
+    "hidden_size": 1152,
+    "in_channels": 3,
+    "initializer_range": 0.02,
+    "intermediate_size": 4304,
+    "model_type": "qwen3_5",
+    "num_heads": 16,
+    "num_position_embeddings": 2304,
+    "out_hidden_size": 5120,
+    "patch_size": 16,
+    "spatial_merge_size": 2,
+    "temporal_patch_size": 2
+  },
+  "vision_end_token_id": 248054,
+  "vision_start_token_id": 248053
+}

PE/generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 248044,
+  "do_sample": true,
+  "eos_token_id": [
+    248046,
+    248044
+  ],
+  "pad_token_id": 248044,
+  "temperature": 1.0,
+  "top_k": 20,
+  "top_p": 0.95,
+  "transformers_version": "5.2.0",
+  "trust_remote_code": true
+}

PE/model-00001-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21378cddb8af1e4762b49ee854f7feb7313af587c11000aabff74c80a83357ef
+size 3966730520

PE/model-00002-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5507c88114438c3db3183000ba1ff89a54a67a5c386a77a9c08e3790834b6c24
+size 3043080296

PE/model-00003-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfb4d95041d0f45bd10c3842a83261417580224dd183585d14c7140d1ddae8a6
+size 2542796928

PE/model-00004-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e5c5b4671ed0ef1ab5258a044053af20f986eb1b840bd1ddf75d04488233f67
+size 3988973120

PE/model-00005-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbf710e0943b267d5ca7a5e50113d148b25780ddf340f08e7bbe2efb30013c47
+size 2099339832

PE/model-00006-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50328e5d8f742e8744c31315122bd70d1f5504881e24b4a243ace8d73860fb46
+size 3979553664

PE/model-00007-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d7b4ad46aeffc72028cb21a6703f69a10cadf7c4900c27f353116113fbab25a
+size 2108759312

PE/model-00008-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:711ac5079086c3bc02ef346413f532f02ae9b94097ff1acd38283a85049249b2
+size 3979553664

PE/model-00009-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fcb1196f840dd1ac1ef2b0996244e0a720878f8b0ff026ed0f80ff9e3a41f12c
+size 2108759312

PE/model-00010-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e29fff61699cc0d4203622692f602e4dd9cd679a5b3f4b8e0d16d9ab2300117
+size 3979553664

PE/model-00011-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f8bb69f6a54060ff54715df425112b12b470cac6cfbd6b6445664755bccee48
+size 2108759312

PE/model-00012-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:168605230ef0ad1fcca51902ac7ab222f33f1980af30adbf37deeceb099fd204
+size 3979553664

PE/model-00013-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:faf02b649a7e64a08c3c7310a819ec85c5a000959840d05cc312826d77f3ee6d
+size 2108759312

PE/model-00014-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f73e2191c0a1849f023823f62c85fec7c0236b1bd8cf49fada3bfffb4376557
+size 3979553664

PE/model-00015-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb1fb4e1e00ee33c896ad427be4dcd42075d2b6982e8c631b119a2ceb107e5ea
+size 2108759312

PE/model-00016-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fe0742ee6faba0fd73ecaf58483840250c894196660a121a3bc9f93a63be549
+size 3979564008

PE/model-00017-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7838bd2919c05a0f8afabba1a7b0509f5a69b2b5115594e1cc2a19aa4fac4e9f
+size 2108759312

PE/model-00018-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36e15aa729908346909ee6e92fc44618f19c8a19512f9b0400da99bfaa9397de
+size 3392197320

PE/preprocessor_config.json

@@ -0,0 +1,27 @@
+{
+  "data_format": "channels_first",
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "Qwen2VLImageProcessorFast",
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "merge_size": 2,
+  "patch_size": 16,
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "longest_edge": 16777216,
+    "shortest_edge": 65536
+  },
+  "temporal_patch_size": 2
+}

PE/processor_config.json

@@ -0,0 +1,63 @@
+{
+  "image_processor": {
+    "data_format": "channels_first",
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "image_mean": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "image_processor_type": "Qwen2VLImageProcessorFast",
+    "image_std": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "merge_size": 2,
+    "patch_size": 16,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "size": {
+      "longest_edge": 16777216,
+      "shortest_edge": 65536
+    },
+    "temporal_patch_size": 2
+  },
+  "processor_class": "Qwen3VLProcessor",
+  "video_processor": {
+    "data_format": "channels_first",
+    "default_to_square": true,
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "do_sample_frames": true,
+    "fps": 2,
+    "image_mean": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "image_std": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "max_frames": 768,
+    "merge_size": 2,
+    "min_frames": 4,
+    "patch_size": 16,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "return_metadata": false,
+    "size": {
+      "longest_edge": 25165824,
+      "shortest_edge": 4096
+    },
+    "temporal_patch_size": 2,
+    "video_processor_type": "Qwen3VLVideoProcessor"
+  }
+}

PE/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87a7830d63fcf43bf241c3c5242e96e62dd3fdc29224ca26fed8ea333db72de4
+size 19989343

PE/tokenizer_config.json

@@ -0,0 +1,32 @@
+{
+  "add_prefix_space": false,
+  "audio_bos_token": "<|audio_start|>",
+  "audio_eos_token": "<|audio_end|>",
+  "audio_token": "<|audio_pad|>",
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "image_token": "<|image_pad|>",
+  "is_local": true,
+  "model_max_length": 262144,
+  "model_specific_special_tokens": {
+    "audio_bos_token": "<|audio_start|>",
+    "audio_eos_token": "<|audio_end|>",
+    "audio_token": "<|audio_pad|>",
+    "image_token": "<|image_pad|>",
+    "video_token": "<|video_pad|>",
+    "vision_bos_token": "<|vision_start|>",
+    "vision_eos_token": "<|vision_end|>"
+  },
+  "pad_token": "<|endoftext|>",
+  "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+  "processor_class": "Qwen3VLProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": null,
+  "video_token": "<|video_pad|>",
+  "vision_bos_token": "<|vision_start|>",
+  "vision_eos_token": "<|vision_end|>"
+}

PE/video_preprocessor_config.json

@@ -0,0 +1,21 @@
+{
+    "size": {
+        "longest_edge": 25165824,
+        "shortest_edge": 4096
+    },
+    "patch_size": 16,
+    "temporal_patch_size": 2,
+    "merge_size": 2,
+    "image_mean": [
+        0.5,
+        0.5,
+        0.5
+    ],
+    "image_std": [
+        0.5,
+        0.5,
+        0.5
+    ],
+    "processor_class": "Qwen3VLProcessor",
+    "video_processor_type": "Qwen3VLVideoProcessor"
+}

README.md

@@ -0,0 +1,288 @@
+---
+license: apache-2.0
+pipeline_tag: text-to-video
+tags:
+  - hy-nomas
+  - world-model
+  - video-generation
+  - interactive-video-generation
+  - camera-control
+  - autoregressive
+  - diffusion-transformer
+  - pytorch
+---
+
+<p align="left">
+  <a href="README_CN.md">中文</a>&nbsp;｜&nbsp;English
+</p>
+
+<p align="center">
+  <img src="assets/logo-en.png" alt="Hy-Nomas" width="400"/>
+</p>
+
+<h1 align="center">Hy-Nomas: Towards Free-Form Interactive World Generation</h1>
+
+<p align="center">
+  <a href="https://github.com/Tencent-Hunyuan/Hy-Nomas"><img src="https://img.shields.io/badge/GitHub-181717?style=flat-square&amp;logo=github&amp;logoColor=white" alt="GitHub"/></a>
+  <a href="https://github.com/Tencent-Hunyuan/Hy-Nomas/blob/main/Hy_Nomas_Tech_Report.pdf"><img src="https://img.shields.io/badge/Technical%20Report%20%28Coming%20Soon%29-B31B1B?style=flat-square&amp;logo=data%3Aimage%2Fsvg%2Bxml%3Bbase64%2CPHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAyNCAyNCIgZmlsbD0ibm9uZSIgc3Ryb2tlPSJ3aGl0ZSIgc3Ryb2tlLXdpZHRoPSIyIiBzdHJva2UtbGluZWpvaW49InJvdW5kIj48cGF0aCBkPSJNMTQgMkg1djIwaDE0Vjd6TTE0IDJ2NWg1TTggMTJoOE04IDE2aDgiLz48L3N2Zz4%3D" alt="Technical Report (Coming Soon)"/></a>
+  <a href="https://huggingface.co/datasets/tencent/Hy-Nomas-Bench"><img src="https://img.shields.io/badge/Benchmark-FFD21E?style=flat-square&amp;logo=huggingface&amp;logoColor=black" alt="Benchmark"/></a>
+  <a href="https://hy-nomas.hy.tencent.com/"><img src="https://img.shields.io/badge/Official%20Website-666666?style=flat-square&amp;logo=data%3Aimage%2Fsvg%2Bxml%3Bbase64%2CPHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAyNCAyNCIgZmlsbD0ibm9uZSIgc3Ryb2tlPSJ3aGl0ZSIgc3Ryb2tlLXdpZHRoPSIyIj48Y2lyY2xlIGN4PSIxMiIgY3k9IjEyIiByPSIxMCIvPjxlbGxpcHNlIGN4PSIxMiIgY3k9IjEyIiByeD0iNCIgcnk9IjEwIi8%2BPHBhdGggZD0iTTIgMTJoMjAiLz48L3N2Zz4%3D" alt="Official Website"/></a>
+</p>
+
+<h3 align="center">
+  🚀 <a href="https://hy-nomas.hy.tencent.com/">Try the Demo →</a>
+</h3>
+
+## Model Introduction
+
+**Hy-Nomas** is a video world generation model that supports free-form, multi-turn interaction through text instructions and camera control. It generates a world one video segment at a time, allowing users to introduce objects, change styles and scenes, describe actions, or control the camera as generation continues.
+
+The model combines a segment-autoregressive diffusion transformer (DiT), a Qwen text encoder, camera-conditioning modules, and a PixelUMM video decoder. A persistent key/value (KV) cache carries context between segments. The distilled inference configuration uses **8 denoising steps**, with **720 × 1280** output at **24 FPS**. A separate configuration supports video generation on a **single NVIDIA GeForce RTX 4090** using INT8 DiT weights, an INT8 KV cache, an FP8 decoder, and CPU offload.
+
+## Key Features
+
+- **Free-form interaction:** Guide successive segments with instructions about subjects, actions, appearance, style, and environment.
+- **Camera control:** Supply camera trajectories alongside text to control viewpoint and movement.
+- **Continuous generation:** Reuse earlier context through a KV cache when extending the same world.
+- **Distilled inference:** Generate each segment with 8 DiT denoising steps and decode its latents with a fine-tuned PixelUMM decoder.
+- **Flexible deployment:** Use the standard sequence-parallel configuration or the single-GPU configuration, and interact through the web demo.
+
+## Hy-Nomas-Bench
+
+We also release **[Hy-Nomas-Bench](https://huggingface.co/datasets/tencent/Hy-Nomas-Bench)**, a suite of four benchmarks with **280 test cases** covering multi-turn free-form text interaction, camera navigation, interleaved text and camera control, and spatial memory. It evaluates interactive controllability and consistency during continuous world generation, providing structured prompts, interaction sequences, keyboard timelines, and reference camera trajectories. See the [Hy-Nomas-Bench dataset repository](https://huggingface.co/datasets/tencent/Hy-Nomas-Bench) for downloads, data formats, and inference instructions.
+
+## Model Architecture
+
+Hy-Nomas performs diffusion in a compressed video latent space and autoregression across video segments. Text, camera motion, and cached history condition the generation of each new segment.
+
+![Hy-Nomas framework: structured prompts, video latents, and camera trajectories feed dual-stream MMDiT blocks with segment-wise causal joint attention, timestep modulation, camera cross-attention, and a persistent KV cache.](assets/method_framework_h.png)
+
+*Hy-Nomas framework. The inference configuration uses 54 dual-stream MMDiT blocks (N = 54). Purple, blue, and green indicate the text, video, and camera paths, respectively.*
+
+The figure's video-encoder path illustrates how video is represented as DCAE-V latents. In the text-to-video inference flow below, each new segment starts from Gaussian noise in that latent space and uses the KV cache for generated history. The MLLM is the Qwen text encoder, and the default video decoder is the fine-tuned PixelUMM described below.
+
+### Text and Camera Conditioning
+
+**Text conditioning.** The text encoder uses the language-model component of **Qwen/Qwen2.5-VL-7B-Instruct**. Its 3,584-dimensional hidden states pass through a two-layer token refiner and are projected into the DiT's 2,048-dimensional hidden space. Segment-specific prompts let users change the requested content as generation proceeds.
+
+**Online prompt enhancement.** In the web demo, **Nomas-PE** first converts user instructions into structured generation prompts. It is a separate service from the text encoder and is required by the demo. Offline inference can read prepared prompt files directly.
+
+**Camera conditioning.** The inference configuration uses `extrinsics_attn`. Normalized 4 × 4 camera extrinsic matrices are grouped over four video frames and flattened into 64-dimensional inputs, aligning camera features with the temporal compression of the video latents. Learned pose embeddings and projections implement the figure's **Pose Encoder → Camera Proj** path. At each double-stream block, video tokens query the projected camera features through cross-attention after the video MLP. With `per_chunk_camera_attn=true` and `camera_chunk_size=1`, camera attention is aligned to individual latent frames.
+
+### Segment-Autoregressive Diffusion Transformer
+
+The generation backbone is a **dense multimodal diffusion transformer (MMDiT)**. Video latents enter through **Patch Embed**, while refined text tokens enter a separate stream. The **Timestep Embed** supplies noise-level conditioning: each block uses it to modulate the two streams with learned shifts, scales, and residual gates.
+
+Within a block, text and video interact through **segment-wise causal joint attention**, with 3D RoPE encoding video positions and the layer's KV cache supplying context from earlier segments. The two streams then pass through separate MLPs. **Camera cross-attention** updates the video branch using video queries and camera keys/values. Text features continue to the next block; after the last block, the output head uses only the video stream to predict latent-space flow velocity. The sampler uses these predictions to denoise the segment before video decoding.
+
+The model architecture is illustrated in the following table:
+
+| Property | Configuration |
+|:---|:---|
+| Model identifier | `Hy-Nomas` |
+| Transformer blocks | 54 double-stream blocks; 0 single-stream blocks |
+| Hidden size | 2,048 |
+| Attention heads | 16 heads; 128 dimensions per head |
+| Feed-forward expansion | 4×; intermediate size 8,192 |
+| Latent input / output channels | 32 / 32 |
+| Latent patch size, time × height × width | 1 × 1 × 1 |
+| Position encoding | 3D RoPE; temporal/height/width dimensions `[16, 56, 56]`; theta 256 |
+| Text conditioning | 3,584-dimensional Qwen features; two-layer token refiner |
+| Camera conditioning | Extrinsic embeddings with cross-attention in the double-stream blocks |
+| Denoising prediction | Flow velocity; Euler ODE sampling in the inference configuration |
+
+At each turn, the DiT denoises the current segment while attending to all past KV cache and the current conditioning. Attention is causal across segments, so future segments are not required to generate the current one. Cached attention keys and values are reused when generation advances to the next segment.
+
+The standard configuration keeps the full history (`max_cache_segments=null`, sliding window disabled). The single-GPU configuration selects a bounded set of historical latent frames for GPU attention and retains an archive on CPU. Its default history budget is **18 latent frames**; that budget applies to historical context, not to the length of the current segment.
+
+### Video Latents and PixelUMM Decoder
+
+Hy-Nomas uses **DCAE-V `f32t4c32` latents**: 32 channels, 32× spatial compression in each dimension, and 4× temporal compression. The DiT produces latents in this representation. The default inference pipeline decodes them with a **fine-tuned PixelUMM decoder**.
+
+The decoder uses a transformer with **36 layers**, a **4,096-dimensional hidden size**, **32 query heads**, and **8 KV heads**. It conditions pixel reconstruction on the generated DCAE-V latents and processes RGB video in 4 × 16 × 16 tubelets. The configuration uses **one PixelUMM decoding step**, with temporal windows of 10 latent frames and an overlap of 5 latent frames. This decoding step is separate from the DiT's 8 denoising steps.
+
+Architecture values are specified in the [DiT model definitions](https://github.com/Tencent-Hunyuan/Hy-Nomas/blob/HEAD/hymm/models/dit/modules/__init__.py), [DiT implementation](https://github.com/Tencent-Hunyuan/Hy-Nomas/blob/HEAD/hymm/models/dit/nomas_cpt.py), [PixelUMM implementation](https://github.com/Tencent-Hunyuan/Hy-Nomas/blob/HEAD/hymm/models/vae/pixelumm_decoder/model.py), and [inference configuration](https://github.com/Tencent-Hunyuan/Hy-Nomas/blob/HEAD/configs/experiment/inference.yaml).
+
+## Model Assets
+
+Select compatible generation and decoding checkpoints for the inference configuration you use.
+
+| Component | Asset | Role | Path in this repository |
+|:---|:---|:---|:---|
+| DiT | **Distilled DiT (FP32)** | Main checkpoint for 8-step inference and the interactive demo; the standard configuration loads the DiT in BF16. | `DiT/distilled/diffusion_pytorch_model_ema_0_995.safetensors` |
+| DiT | **Quantized Distilled DiT (INT8)** | Quantized checkpoint for the single-GPU configuration; weights are stored in INT8 and DiT linear computation remains BF16. | `DiT/distilled_int8/diffusion_pytorch_model_ema_0_995_int8.safetensors` |
+| DiT | **Base autoregressive DiT (FP32)** | Non-distilled model; use the distilled model with the 8-step examples below. | `DiT/base_ar/diffusion_pytorch_model_fp32.safetensors` |
+| Adapter | **Cinematic LoRA** | Optional visual-quality adapter for a compatible floating-point DiT. | `DiT/lora_weights/cinematic/` |
+| Adapter | **Robotics LoRA** | LoRA adapter trained on the [RealSource-World](https://huggingface.co/datasets/RealmanRobot/RealSource-World) real-world robotic-arm dataset, to be merged into the distilled DiT to enhance the robotics simulation capability of generated worlds. | `DiT/lora_weights/dual_arm_robotics/` |
+| Video autoencoder | **DCAE-V** | Defines the 32-channel compressed video representation and provides video encoding and decoding. | `VAE/DCAE-V/dc-ae-v-1.0-f32t4c32-bf16.pt` |
+| Video decoder | **Fine-tuned PixelUMM** | Default decoder for generated DCAE-V latents. | `VAE/PixelUMM_FT/diffusion_pytorch_model.safetensors` |
+| Video decoder | **Quantized PixelUMM (FP8)** | Decoder checkpoint for the low-memory configuration. | `VAE/PixelUMM_FT_FP8/diffusion_pytorch_model_fp8.safetensors` |
+| Prompt enhancement | **Nomas-PE** | Separate online service that converts user instructions into structured prompts for the web demo. | `PE/` |
+
+Standard offline inference requires the **distilled DiT**, a **matching PixelUMM checkpoint**, and the **Qwen text encoder**. The single-GPU configuration requires the compatible **INT8 DiT** and **FP8 PixelUMM** pair. The INT8 loader does not support loading a separate LoRA adapter at inference time.
+
+Download the applicable assets from this repository, preserving the `DiT/`, `VAE/`, and `PE/` directory layout. The inference commands below use `HY_NOMAS_MODEL_DIR` to locate the downloaded model repository. The text encoder is automatically downloaded from `Qwen/Qwen2.5-VL-7B-Instruct` by Huggingface and reused from the local cache. To use an existing local copy, append `model.text_encoder.model_name=/path/to/Qwen2.5-VL-7B-Instruct`.
+
+## Usage
+
+### Environment Setup
+
+Run inference with the [Hy-Nomas code repository](https://github.com/Tencent-Hunyuan/Hy-Nomas):
+
+```bash
+git clone https://github.com/Tencent-Hunyuan/Hy-Nomas.git
+cd Hy-Nomas
+```
+
+Choose one of the following two setup options. **We strongly recommend the pre-packaged Docker image**, which provides the Python environment, PyTorch, and the required CUDA attention kernels together.
+
+#### Option 1: Pre-packaged Docker Image (Recommended)
+
+Prepare a Linux host with a compatible NVIDIA driver, Docker, and NVIDIA Container Toolkit. Run these commands from the code repository root:
+
+```bash
+export HY_NOMAS_IMAGE="hynomasoss/hynomasoss:cu130"
+
+docker pull "$HY_NOMAS_IMAGE"
+docker run --rm -it --gpus all --ipc=host --network=host \
+  -v "$PWD:/workspace/Hy-Nomas" \
+  -w /workspace/Hy-Nomas \
+  "$HY_NOMAS_IMAGE" bash
+```
+
+Run the remaining commands inside the container, using its pre-installed Python environment. Place model assets under the mounted repository or add a volume mount for your model directory; use the paths visible inside the container in the launch commands below.
+
+#### Option 2: Manual Installation
+
+Prepare Linux, Python 3.12+, uv, a compatible NVIDIA driver, the CUDA 13.0 Toolkit (`nvcc`), and a C++ compiler. Run the following from the code repository root:
+
+```bash
+uv sync --extra all
+source .venv/bin/activate
+```
+
+The `all` extra includes PyTorch 2.10.0 (CUDA 13.0), FlashAttention 2, FlashAttention 3, and MagiAttention. `pyproject.toml` configures the PyTorch CUDA index, the official FA3 source, and disables build isolation for the attention kernels so their builds can use the environment's PyTorch. Kernel compilation may take some time.
+
+### Model Paths
+
+In the environment used for inference, set `HY_NOMAS_MODEL_DIR` to the local model repository root containing the weight paths listed above. When using Docker, use a path visible inside the container.
+
+```bash
+export HY_NOMAS_MODEL_DIR="/path/to/Hy-Nomas-models"
+```
+
+Run the commands below from the Hy-Nomas **code repository root**. They use this variable to read the weights and example prompts from the model repository.
+
+### Prepare Prompts
+
+Offline inference accepts a JSON list of video-generation requests. Each request can contain multiple `segments` with start and end timestamps in seconds. The first segment describes the initial world, and later segments use `description_change` to describe how it should evolve.
+
+The inference commands below use the bundled [example prompt file](examples/prompts.json) in the model repository. For custom content, save a JSON file in the following format and set `inference.prompt=/path/to/prompts.json`:
+
+```json
+[
+  {
+    "test_id": "forest_world",
+    "Style": "Cinematic fantasy",
+    "perspective_view": "Third-person view",
+    "subject_list": [
+      {
+        "subject_id": "fox",
+        "display_name": "a small orange fox",
+        "subject_attribute": "Orange fur, white chest, and a fluffy tail."
+      }
+    ],
+    "segments": [
+      {
+        "start_time": 0,
+        "end_time": 2,
+        "visual_description_long": "A small orange fox stands on a stone path in a luminous forest. Blue fireflies drift between the trees.",
+        "contained_actions": [
+          {"start_time": 0, "end_time": 2, "action_description": "The fox walks slowly along the path."}
+        ]
+      },
+      {
+        "start_time": 2,
+        "end_time": 4,
+        "description_change": "Glowing blue flowers open beside the path as the fox continues forward.",
+        "contained_actions": [
+          {"start_time": 2, "end_time": 4, "action_description": "The fox passes the flowers and looks around."}
+        ]
+      }
+    ]
+  }
+]
+```
+
+The configuration derives segment lengths from event timestamps (`inference.video_length=event`). Action timestamps in this example use the same video timeline; the configuration normalizes them to each segment internally.
+
+### Standard Inference
+
+The standard configuration uses **8-way sequence parallelism** (`sp_size=8`) and distributed PixelUMM decoding. Its default launch therefore requires eight GPUs in one inference group. Run from the code repository root:
+
+```bash
+bash scripts/inference/inference_from_config.sh configs/experiment/inference.yaml \
+  dit_weight="$HY_NOMAS_MODEL_DIR/DiT/distilled/diffusion_pytorch_model_ema_0_995.safetensors" \
+  model.vae.checkpoint="$HY_NOMAS_MODEL_DIR/VAE/PixelUMM_FT/diffusion_pytorch_model.safetensors" \
+  inference.prompt="$HY_NOMAS_MODEL_DIR/examples/prompts.json" \
+  inference.save_path=./outputs/distill
+```
+
+Generated videos are written under `inference.save_path`. The configuration also saves generated latents and prompt text.
+
+| Setting | Default | Override |
+|:---|:---|:---|
+| Output resolution, height × width | 720 × 1280 | `'inference.video_size=[480,832]'` |
+| Output video frame rate | 24 FPS | `inference.video_fps=24` |
+| DiT denoising steps | 8 | `inference.infer_steps_video=8` |
+| Solver | Euler ODE | `inference.solver_type=euler` |
+| Text / camera guidance scale | 1.0 / 1.0 | `inference.cfg_scale_video=1.0 inference.camera_cfg_scale=1.0` |
+| Random seed | 42 | `inference.seed=42` |
+| Camera translation / rotation strength | 1.0 / 2.5 | `inference.trajectory_translation_scale=1.0 inference.trajectory_rotation_scale=2.5` |
+
+For camera-controlled generation, append `inference.camera_trajectory=/path/to/camera.npz`. The trajectory file contains camera-to-world matrices `c2ws` with shape `[N, 4, 4]` and camera `intrinsics` with shape `[N, 3, 3]`; align it with the requested video duration and frame rate. The repository includes [camera trajectory utilities](https://github.com/Tencent-Hunyuan/Hy-Nomas/blob/HEAD/hymm/models/camera/geometry/generate_npz_trajectories.py).
+
+### Single-GPU Low-Memory Inference
+
+Use `configs/experiment/single_gpu_low_mem_inference.yaml` for **single NVIDIA GeForce RTX 4090** inference. This configuration inherits the standard architecture, prompt processing, resolution, frame rate, and 8-step DiT schedule, and changes memory management and execution:
+
+- **INT8 DiT and KV storage:** Store DiT weights and cached keys/values in INT8 while retaining BF16 DiT linear and attention computation.
+- **Bounded GPU history:** Select up to 18 historical latent frames using `temporal_context` sampling, with earlier history archived on CPU.
+- **CPU offload:** Move the text encoder, DiT, and decoder to the GPU for their processing stages and return them to CPU afterward.
+- **Single-GPU kernels:** Use FlexAttention for the DiT, CUTLASS FP8 for the decoder, and `sp_size=1`.
+
+Prepare the compatible INT8 DiT and FP8 PixelUMM checkpoints. Run on a host or in a container exposing **one GPU to `nvidia-smi`**, which the launcher uses to count GPUs:
+
+```bash
+bash scripts/inference/inference_from_config.sh configs/experiment/single_gpu_low_mem_inference.yaml \
+  dit_weight="$HY_NOMAS_MODEL_DIR/DiT/distilled_int8/diffusion_pytorch_model_ema_0_995_int8.safetensors" \
+  model.vae.checkpoint="$HY_NOMAS_MODEL_DIR/VAE/PixelUMM_FT_FP8/diffusion_pytorch_model_fp8.safetensors" \
+  inference.prompt="$HY_NOMAS_MODEL_DIR/examples/prompts.json" \
+  inference.save_path=./outputs/single_gpu
+```
+
+System RAM must accommodate the offloaded models and the growing history archive. CPU–GPU transfers add latency. Peak GPU memory also depends on output resolution and the current segment length; the history budget does not cap current-segment memory.
+
+### Interactive Demo
+
+Try the [online interactive demo](https://hy-nomas.hy.tencent.com/) to start a world, continue it with text instructions or camera paths, and download the generated video.
+
+For local deployment, follow the [web demo deployment guide](https://github.com/Tencent-Hunyuan/Hy-Nomas/blob/HEAD/apps/README.md). It covers the frontend, gateway, GPU workers, single-GPU mode, and multiple-host deployment. **Nomas-PE is required for the web demo** and uses separate serving resources; the single-GPU claim above applies to video inference. The demo enables streaming decoding at segment boundaries.
+
+## Runtime Considerations
+
+- Full-history inference retains more context as generation continues and consumes increasing cache memory. The single-GPU configuration bounds the active GPU history while its CPU archive can continue to grow.
+- The 24 FPS setting specifies the output video's playback frame rate. Generation speed depends on hardware, resolution, segment length, history, and the selected configuration.
+- The 8-step examples require the distilled DiT and its compatible decoder. Architecture and sampling defaults are configuration details, rather than benchmark results.
+
+## License
+
+Hy-Nomas is licensed under the [Apache License, Version 2.0](LICENSE). Third-party models and components remain subject to their respective licenses.
+
+## Acknowledgement
+
+We thank the teams behind [Gemma 4](https://deepmind.google/models/gemma/gemma-4/), [Qwen3.8](https://qwen.ai/blog?id=qwen3.8), [Kimi-K3](https://github.com/MoonshotAI/Kimi-K3), [DC-AE](https://github.com/dc-ai-projects/DC-Gen), and [MagiAttention](https://github.com/SandAI-org/MagiAttention) for their valuable contributions to the community and the models, research, and tools that support this project.
+
+## Contact
+
+For questions, non-security bug reports, or suggestions, open an issue in the [Hy-Nomas repository](https://github.com/Tencent-Hunyuan/Hy-Nomas/issues) or contact [hy_nomas@tencent.com](mailto:hy_nomas@tencent.com). Report suspected security vulnerabilities through the repository's [security guidance](https://github.com/Tencent-Hunyuan/Hy-Nomas/blob/HEAD/SECURITY.md).

README_CN.md

@@ -0,0 +1,288 @@
+---
+license: apache-2.0
+pipeline_tag: text-to-video
+tags:
+  - hy-nomas
+  - world-model
+  - video-generation
+  - interactive-video-generation
+  - camera-control
+  - autoregressive
+  - diffusion-transformer
+  - pytorch
+---
+
+<p align="left">
+  <a href="README.md">English</a>&nbsp;｜&nbsp;中文
+</p>
+
+<p align="center">
+  <img src="assets/logo-zh.png" alt="Hy-Nomas" width="400"/>
+</p>
+
+<h1 align="center">Hy-Nomas: Towards Free-Form Interactive World Generation</h1>
+
+<p align="center">
+  <a href="https://github.com/Tencent-Hunyuan/Hy-Nomas"><img src="https://img.shields.io/badge/GitHub-181717?style=flat-square&amp;logo=github&amp;logoColor=white" alt="GitHub"/></a>
+  <a href="https://github.com/Tencent-Hunyuan/Hy-Nomas/blob/main/Hy_Nomas_Tech_Report.pdf"><img src="https://img.shields.io/badge/%E6%8A%80%E6%9C%AF%E6%8A%A5%E5%91%8A%20%28Coming%20Soon%29-B31B1B?style=flat-square&amp;logo=data%3Aimage%2Fsvg%2Bxml%3Bbase64%2CPHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAyNCAyNCIgZmlsbD0ibm9uZSIgc3Ryb2tlPSJ3aGl0ZSIgc3Ryb2tlLXdpZHRoPSIyIiBzdHJva2UtbGluZWpvaW49InJvdW5kIj48cGF0aCBkPSJNMTQgMkg1djIwaDE0Vjd6TTE0IDJ2NWg1TTggMTJoOE04IDE2aDgiLz48L3N2Zz4%3D" alt="技术报告 (Coming Soon)"/></a>
+  <a href="https://huggingface.co/datasets/tencent/Hy-Nomas-Bench"><img src="https://img.shields.io/badge/%E8%AF%84%E6%B5%8B%E5%9F%BA%E5%87%86-FFD21E?style=flat-square&amp;logo=huggingface&amp;logoColor=black" alt="评测基准"/></a>
+  <a href="https://hy-nomas.hy.tencent.com/"><img src="https://img.shields.io/badge/Official%20Website-666666?style=flat-square&amp;logo=data%3Aimage%2Fsvg%2Bxml%3Bbase64%2CPHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAyNCAyNCIgZmlsbD0ibm9uZSIgc3Ryb2tlPSJ3aGl0ZSIgc3Ryb2tlLXdpZHRoPSIyIj48Y2lyY2xlIGN4PSIxMiIgY3k9IjEyIiByPSIxMCIvPjxlbGxpcHNlIGN4PSIxMiIgY3k9IjEyIiByeD0iNCIgcnk9IjEwIi8%2BPHBhdGggZD0iTTIgMTJoMjAiLz48L3N2Zz4%3D" alt="Official Website"/></a>
+</p>
+
+<h3 align="center">
+  🚀 <a href="https://hy-nomas.hy.tencent.com/">体验演示 →</a>
+</h3>
+
+## 模型介绍
+
+**Hy-Nomas** 是一个支持自由形式、多轮交互的视频世界生成模型，可通过文本指令和相机控制信号逐段生成视频世界。用户可以在生成过程中引入物体、改变风格和场景、描述动作，或控制相机运动。
+
+模型由分段自回归扩散 Transformer（DiT）、Qwen 文本编码器、相机条件模块和 PixelUMM 视频解码器组成，通过持久化的键值（KV）缓存在片段之间传递上下文。蒸馏推理预设采用 **8 步去噪**，默认输出分辨率为 **720 × 1280**，视频帧率为 **24 FPS**。另提供 **NVIDIA GeForce RTX 4090 单卡**视频推理配置，结合 INT8 DiT 权重、INT8 KV cache、FP8 解码器和 CPU offload 降低显存占用。
+
+## 核心特性
+
+- **自由形式交互：** 通过多轮指令控制主体、动作、外观、风格和环境。
+- **相机控制：** 将相机轨迹与文本提示词一起作为条件，控制视角与镜头运动。
+- **连续生成：** 通过 KV cache 复用历史上下文，在同一世界中持续生成后续片段。
+- **蒸馏推理：** 每个片段使用 8 步 DiT 去噪，再由微调版 PixelUMM 将潜变量解码为视频。
+- **灵活部署：** 支持标准序列并行配置、单卡配置和 Web 交互式演示。
+
+## Hy-Nomas-Bench
+
+我们同时发布 **[Hy-Nomas-Bench](https://huggingface.co/datasets/tencent/Hy-Nomas-Bench)**，包含 **280 个测试案例**，覆盖多轮自由形式文本交互、相机导航、文本与相机交替控制，以及空间记忆四项基准，用于评估视频世界模型在连续生成过程中的交互可控性与一致性。数据集提供结构化提示词、交互序列、按键时间线和参考相机轨迹。数据下载、格式说明及推理指南请参阅 [Hy-Nomas-Bench 数据集仓库](https://huggingface.co/datasets/tencent/Hy-Nomas-Bench)。
+
+## 模型架构
+
+Hy-Nomas 在压缩的视频潜空间中进行扩散去噪，在视频片段之间进行自回归生成。文本、相机运动和缓存的历史共同作为新片段的生成条件。
+
+![Hy-Nomas 中文框架：结构化提示词、视频潜变量和相机轨迹输入双流 MMDiT，通过分段因果联合注意力、时间步调制、相机交叉注意力和持久化 KV cache 生成视频片段。](assets/method_framework_h_cn.png)
+
+*Hy-Nomas 整体框架。推理预设使用 54 个双流 MMDiT 块（N = 54）；紫色、蓝色和绿色分别表示文本、视频和相机路径。*
+
+图中的视频编码路径展示了视频如何表示为 DCAE-V 潜变量。下方文本生成视频的推理流程从该潜空间中的高斯噪声开始生成每个新片段，并通过 KV cache 获取已生成的历史上下文。图中的“多模态大模型”对应 Qwen 文本编码器，默认视频解码器使用下文介绍的微调版 PixelUMM。
+
+### 文本与相机条件
+
+**文本条件。** 文本编码器使用 **Qwen/Qwen2.5-VL-7B-Instruct** 的语言模型部分，其 3,584 维��藏状态通过两层 token refiner，映射到 DiT 的 2,048 维隐藏空间。各片段使用相应的提示词，用户可随生成过程更新内容要求。
+
+**在线提示词增强。** Web 演示首先通过 **Nomas-PE** 将用户指令转换为结构化生成提示词。它独立于文本编码器，以单独的服务运行，是演示的必要依赖。离线推理可以直接读取已准备好的提示词文件。
+
+**相机条件。** 推理预设采用 `extrinsics_attn`：将归一化的 4 × 4 相机外参矩阵按四个视频帧分组，展平为 64 维输入，与视频潜变量的时间压缩比例对齐。可学习的位姿嵌入和投影对应图中的**位姿编码器 → 相机投影**路径。在每个双流块中，视频 token 经过视频 MLP 后，通过交叉注意力查询投影后的相机特征。`per_chunk_camera_attn=true`、`camera_chunk_size=1` 表示相机注意力按单个潜变量帧对齐。
+
+### 分段自回归扩散 Transformer
+
+生成主干是一个**稠密多模态扩散 Transformer（MMDiT）**。视频潜变量经过**时空块嵌入**进入视频流，细化后的文本 token 进入独立的文本流。**时间步嵌入**提供当前噪声水平的条件，各块据此生成偏移、缩放和残差门控，对两个分支分别进行调制。
+
+在每个块中，文本与视频通过**分段因果联合注意力**交换信息，使用 3D RoPE 编码视频位置，并从该层的 KV cache 读取历史片段上下文。随后，两个分支分别经过各自的 MLP。**相机交叉注意力**以视频 token 为 Query、相机特征为 Key/Value，进一步更新视频分支。文本特征继续传入下一块；经过最后一块后，输出头仅使用视频分支预测潜空间中的 flow velocity，采样器据此完成片段去噪，再进行视频解码。
+
+模型架构如下表所示：
+
+| 属性 | 配置 |
+|:---|:---|
+| 模型标识 | `Hy-Nomas` |
+| Transformer 层数 | 54 个双流块；0 个单流块 |
+| 隐藏维度 | 2,048 |
+| 注意力头 | 16 个头；每头 128 维 |
+| 前馈网络扩展比例 | 4 倍；中间维度 8,192 |
+| 潜变量输入 / 输出通道 | 32 / 32 |
+| 潜变量 patch 大小，时间 × 高 × 宽 | 1 × 1 × 1 |
+| 位置编码 | 3D RoPE；时间 / 高 / 宽维度为 `[16, 56, 56]`；theta 为 256 |
+| 文本条件 | 3,584 维 Qwen 特征；两层 token refiner |
+| 相机条件 | 相机外参嵌入，通过双流块中的交叉注意力注入 |
+| 去噪预测 | 预测 flow velocity；推理预设采用 Euler ODE 采样 |
+
+每轮生成时，DiT 通过注意力访问全部历史 KV cache 和当前条件，对当前片段进行去噪。片段之间采用因果注意力，因此生成当前片段无需访问未来片段。进入下一片段时，复用已缓存的 Key 和 Value。
+
+标准预设保留完整历史（`max_cache_segments=null`，关闭滑动窗口）。单卡预设为 GPU 注意力选择有限数量的历史潜变量帧，并在 CPU 上保留历史存档。默认历史预算为 **18 个潜变量帧**，该预算限制的是历史上下文，不限制当前片段长度。
+
+### 视频潜变量与 PixelUMM 解码器
+
+Hy-Nomas 使用 **DCAE-V `f32t4c32` 潜变量**，包含 32 个通道，空间高、宽各压缩 32 倍，时间压缩 4 倍。DiT 生成该表示下的潜变量，默认推理流程使用**微调版 PixelUMM 解码器**将其还原为视频。
+
+解码器包含 **36 层 Transformer**，隐藏维度为 **4,096**，采用 **32 个 Query 头**和 **8 个 KV 头**。它以生成的 DCAE-V 潜变量为条件重建像素，将 RGB 视频划分为 4 × 16 × 16 的时空块（tubelet）。预设使用 **1 步 PixelUMM 解码**，时间窗口为 10 个潜变量帧，相邻窗口重叠 5 个潜变量帧。该解码步数与 DiT 的 8 步去噪分别配置。
+
+架构参数见 [DiT 模型定义](https://github.com/Tencent-Hunyuan/Hy-Nomas/blob/HEAD/hymm/models/dit/modules/__init__.py)、[DiT 实现](https://github.com/Tencent-Hunyuan/Hy-Nomas/blob/HEAD/hymm/models/dit/nomas_cpt.py)、[PixelUMM 实现](https://github.com/Tencent-Hunyuan/Hy-Nomas/blob/HEAD/hymm/models/vae/pixelumm_decoder/model.py)及[推理配置](https://github.com/Tencent-Hunyuan/Hy-Nomas/blob/HEAD/configs/experiment/inference.yaml)。
+
+## 模型资源
+
+根据所用推理配置，选择匹配的生成模型与解码器权重。
+
+| 组件 | 模型资源 | 用途 | 本仓库中的路径 |
+|:---|:---|:---|:---|
+| DiT | **蒸馏 DiT（FP32）** | 用于 8 步推理和交互式演示的主要检查点；标准配置以 BF16 加载 DiT。 | `DiT/distilled/diffusion_pytorch_model_ema_0_995.safetensors` |
+| DiT | **量化蒸馏 DiT（INT8）** | 用于单卡配置的量化检查点；权重以 INT8 存储，DiT 线性运算仍采用 BF16。 | `DiT/distilled_int8/diffusion_pytorch_model_ema_0_995_int8.safetensors` |
+| DiT | **基础自回归 DiT（FP32）** | 未蒸馏模型；下方的 8 步推理示例应使用蒸馏模型。 | `DiT/base_ar/diffusion_pytorch_model_fp32.safetensors` |
+| 适配器 | **Cinematic LoRA** | 可选的视觉质��适配器，适用于兼容的浮点 DiT。 | `DiT/lora_weights/cinematic/` |
+| 适配器 | **Robotics LoRA** | 在 [RealSource-World](https://huggingface.co/datasets/RealmanRobot/RealSource-World) 真实世界机械臂数据集上训练的 LoRA 适配器，合并到蒸馏 DiT 中，以增强生成世界的机器人仿真能力。 | `DiT/lora_weights/dual_arm_robotics/` |
+| 视频自编码器 | **DCAE-V** | 定义 32 通道的压缩视频表示，并提供视频编码和解码功能。 | `VAE/DCAE-V/dc-ae-v-1.0-f32t4c32-bf16.pt` |
+| 视频解码器 | **微调后的 PixelUMM 解码器** | 默认用于解码生成的 DCAE-V 潜变量。 | `VAE/PixelUMM_FT/diffusion_pytorch_model.safetensors` |
+| 视频解码器 | **量化后的 PixelUMM 解码器（FP8）** | 用于低显存配置的解码器检查点。 | `VAE/PixelUMM_FT_FP8/diffusion_pytorch_model_fp8.safetensors` |
+| 提示词增强 | **Nomas-PE** | 独立的在线服务，将用户指令转换为供 Web 演示使用的结构化提示词。 | `PE/` |
+
+标准离线推理需要**蒸馏 DiT**、**匹配的 PixelUMM 权重**和 **Qwen 文本编码器**。单卡预设需要兼容的 **INT8 DiT** 与 **FP8 PixelUMM** 组合。INT8 加载器不支持在推理时额外加载独立的 LoRA 适配器。
+
+从本仓库下载所需模型资源，并保留 `DiT/`、`VAE/` 和 `PE/` 的目录结构。下方推理命令通过 `HY_NOMAS_MODEL_DIR` 指定下载后的模型仓库根目录。文本编码器由 Hugging Face 自动从 `Qwen/Qwen2.5-VL-7B-Instruct` 下载，并复用本地缓存。如已有本地副本，可在命令后添加 `model.text_encoder.model_name=/path/to/Qwen2.5-VL-7B-Instruct`。
+
+## 使用指南
+
+### 环境配置
+
+使用 [Hy-Nomas 代码仓库](https://github.com/Tencent-Hunyuan/Hy-Nomas) 进行推理：
+
+```bash
+git clone https://github.com/Tencent-Hunyuan/Hy-Nomas.git
+cd Hy-Nomas
+```
+
+可选择以下两种环境配置方式。**强烈推荐使用我们发布的预打包 Docker 镜像**，其中已包含 Python 环境、PyTorch 和所需的 CUDA 注意力内核。
+
+#### 方案一：预打包 Docker 镜像（推荐）
+
+准备一台安装了兼容 NVIDIA 驱动、Docker 和 NVIDIA Container Toolkit 的 Linux 主机。在代码仓库根目录运行：
+
+```bash
+export HY_NOMAS_IMAGE="hynomasoss/hynomasoss:cu130"
+
+docker pull "$HY_NOMAS_IMAGE"
+docker run --rm -it --gpus all --ipc=host --network=host \
+  -v "$PWD:/workspace/Hy-Nomas" \
+  -w /workspace/Hy-Nomas \
+  "$HY_NOMAS_IMAGE" bash
+```
+
+后续命令在容器内执行，使用镜像预装的 Python 环境。将模型资源放在挂载的仓库目录下，或额外挂载模型目录；下方启动命令应使用容器内可见的路径。
+
+#### 方案二：手动安装
+
+准备 Linux、Python 3.12+、uv、兼容的 NVIDIA 驱动、CUDA 13.0 Toolkit（`nvcc`）和 C++ 编译器。在代码仓库根目录运行：
+
+```bash
+uv sync --extra all
+source .venv/bin/activate
+```
+
+`all` 可选依赖组包含 PyTorch 2.10.0（CUDA 13.0）、FlashAttention 2、FlashAttention 3 和 MagiAttention。`pyproject.toml` 已配置 PyTorch CUDA 包源和官方 FA3 源码，并为注意力内核禁用构建隔离，使其构建时能够使用当前环境中的 PyTorch。内核编译可能需要一些时间。
+
+### 模型路径
+
+在运行推理的环境中，将 `HY_NOMAS_MODEL_DIR` 设置为本地模型仓库根目录；该目录应包含上表所列的权重路径。使用 Docker 时，请填写容器内可见的路径。
+
+```bash
+export HY_NOMAS_MODEL_DIR="/path/to/Hy-Nomas-models"
+```
+
+下方命令仍在 Hy-Nomas **代码仓库根目录**运行，通过此变量读取模型仓库中的权重和示例提示词。
+
+### 准备提示词
+
+离线推理支持包含多个视频生成请求的 JSON 列表。每个请求可包含多个 `segments`，用秒为单位的起止时间定义各片段。首个片段描述初始世界，后续片段通过 `description_change` 描述世界如何变化。
+
+下方推理命令使用模型仓库中随附的[提示词示例文件](examples/prompts.json)。如需自定义内容，可按以下格式保存 JSON，并通过 `inference.prompt=/path/to/prompts.json` 指定文件：
+
+```json
+[
+  {
+    "test_id": "forest_world",
+    "Style": "Cinematic fantasy",
+    "perspective_view": "Third-person view",
+    "subject_list": [
+      {
+        "subject_id": "fox",
+        "display_name": "a small orange fox",
+        "subject_attribute": "Orange fur, white chest, and a fluffy tail."
+      }
+    ],
+    "segments": [
+      {
+        "start_time": 0,
+        "end_time": 2,
+        "visual_description_long": "A small orange fox stands on a stone path in a luminous forest. Blue fireflies drift between the trees.",
+        "contained_actions": [
+          {"start_time": 0, "end_time": 2, "action_description": "The fox walks slowly along the path."}
+        ]
+      },
+      {
+        "start_time": 2,
+        "end_time": 4,
+        "description_change": "Glowing blue flowers open beside the path as the fox continues forward.",
+        "contained_actions": [
+          {"start_time": 2, "end_time": 4, "action_description": "The fox passes the flowers and looks around."}
+        ]
+      }
+    ]
+  }
+]
+```
+
+预设通过事件时间戳计算片段长度（`inference.video_length=event`）。示例中的动作时间戳使用同一条视频时间轴，程序会将其转换为各片段的局部时间。
+
+### 标准推理
+
+标准预设采用 **8 路序列并行**（`sp_size=8`）和分布式 PixelUMM 解码，因此默认启动需要一个推理组内的八张 GPU。在代码仓库根目录执行：
+
+```bash
+bash scripts/inference/inference_from_config.sh configs/experiment/inference.yaml \
+  dit_weight="$HY_NOMAS_MODEL_DIR/DiT/distilled/diffusion_pytorch_model_ema_0_995.safetensors" \
+  model.vae.checkpoint="$HY_NOMAS_MODEL_DIR/VAE/PixelUMM_FT/diffusion_pytorch_model.safetensors" \
+  inference.prompt="$HY_NOMAS_MODEL_DIR/examples/prompts.json" \
+  inference.save_path=./outputs/distill
+```
+
+生成的视频保存在 `inference.save_path` 下，预设还会保存生成的潜变量和提示词文本。
+
+| 设置 | 默认值 | 覆盖参数 |
+|:---|:---|:---|
+| 输出分辨率，高 × 宽 | 720 × 1280 | `'inference.video_size=[480,832]'` |
+| 输出视频帧率 | 24 FPS | `inference.video_fps=24` |
+| DiT 去噪步数 | 8 | `inference.infer_steps_video=8` |
+| 求解器 | Euler ODE | `inference.solver_type=euler` |
+| 文本 / 相机引导强度 | 1.0 / 1.0 | `inference.cfg_scale_video=1.0 inference.camera_cfg_scale=1.0` |
+| 随机种子 | 42 | `inference.seed=42` |
+| 相机平移 / 旋转强度 | 1.0 / 2.5 | `inference.trajectory_translation_scale=1.0 inference.trajectory_rotation_scale=2.5` |
+
+进行相机控制生成时，可添加 `inference.camera_trajectory=/path/to/camera.npz`。轨迹文件包含形状为 `[N, 4, 4]` 的 camera-to-world 矩阵 `c2ws`，以及形状为 `[N, 3, 3]` 的相机内参 `intrinsics`；轨迹应与目标视频时长和帧率对齐。仓库提供[相机轨迹工具](https://github.com/Tencent-Hunyuan/Hy-Nomas/blob/HEAD/hymm/models/camera/geometry/generate_npz_trajectories.py)。
+
+### 单卡低显存推理
+
+使用 `configs/experiment/single_gpu_low_mem_inference.yaml` 在 **NVIDIA GeForce RTX 4090 单卡**上推理。该配置继承标准预设的模型架构、提示词处理、分辨率、帧率及 8 步 DiT 去噪设置，并调整内存管理和执行方式：
+
+- **INT8 DiT 与 KV 存储：** 使用 INT8 存储 DiT 权重及缓存的 Key/Value，DiT 线性层和注意力仍采用 BF16 计算。
+- **有限的 GPU 历史上下文：** 使用 `temporal_context` 策略选择最多 18 个历史潜变量帧，将历史存档保存在 CPU 上。
+- **CPU offload：** 文本编码器、DiT 和解码器在各自处理阶段移入 GPU，完成后移回 CPU。
+- **单卡内核：** DiT 使用 FlexAttention，解码器使用 CUTLASS FP8，并设置 `sp_size=1`。
+
+准备兼容的 INT8 DiT 与 FP8 PixelUMM 权重。启动脚本通过 `nvidia-smi` 统计 GPU 数量，请在 **`nvidia-smi` 仅能看到一张 GPU** 的主机或容器中执行：
+
+```bash
+bash scripts/inference/inference_from_config.sh configs/experiment/single_gpu_low_mem_inference.yaml \
+  dit_weight="$HY_NOMAS_MODEL_DIR/DiT/distilled_int8/diffusion_pytorch_model_ema_0_995_int8.safetensors" \
+  model.vae.checkpoint="$HY_NOMAS_MODEL_DIR/VAE/PixelUMM_FT_FP8/diffusion_pytorch_model_fp8.safetensors" \
+  inference.prompt="$HY_NOMAS_MODEL_DIR/examples/prompts.json" \
+  inference.save_path=./outputs/single_gpu
+```
+
+系统内存需要容纳卸载的模型以及随生成增长的历史存档。CPU–GPU 数据传输会增加延迟。峰值显存还取决于输出分辨率和当前片段长度；历史帧预算不限制当前片段的显存占用。
+
+### 交互式演示
+
+访问[在线交互式演示](https://hy-nomas.hy.tencent.com/)，创建世界，通过文本指令或相机路径继续生成，并下载视频。
+
+本地部署请参阅 [Web 演示部署指南](https://github.com/Tencent-Hunyuan/Hy-Nomas/blob/HEAD/apps/README.md)，其中包含前端、网关、GPU 工作进程、单卡模式和多机部署说明。**Web 演示需要 Nomas-PE**，其服务资源单独计算；上文的单卡支持指视频推理部分。演示在片段边界进行流式解码。
+
+## 运行说明
+
+- 完整历史推理会随生成持续增加上下文和缓存内存占用。单卡预设限制 GPU 上活跃的历史上下文，其 CPU 历史存档仍可增长。
+- 24 FPS 指输出视频的播放帧率。实际生成速度取决于硬件、分辨率、片段长度、历史上下文和所选配置。
+- 8 步示例需要蒸馏 DiT 及匹配的解码器。文中的架构和采样参数来自配置，不代表性能评测结果。
+
+## 许可证
+
+Hy-Nomas 采用 [Apache License 2.0](LICENSE) 许可证。第三方模型和组件仍遵循各自的许可证。
+
+## 致谢
+
+感谢 [Gemma 4](https://deepmind.google/models/gemma/gemma-4/)、[Qwen3.8](https://qwen.ai/blog?id=qwen3.8)、[Kimi-K3](https://github.com/MoonshotAI/Kimi-K3)、[DC-AE](https://github.com/dc-ai-projects/DC-Gen) 和 [MagiAttention](https://github.com/SandAI-org/MagiAttention) 团队为社区作出的贡献，以及为本项目提供支持的模型、研究成果和工具。
+
+## 联系我们
+
+如有一般问题、非安全缺陷或建议，欢迎在 [Hy-Nomas 仓库](https://github.com/Tencent-Hunyuan/Hy-Nomas/issues)提交 issue，或通过 [hy_nomas@tencent.com](mailto:hy_nomas@tencent.com) 联系我们。疑似安全漏洞请按照仓库的[安全问题报告指南](https://github.com/Tencent-Hunyuan/Hy-Nomas/blob/HEAD/SECURITY.md)私下报告。

VAE/DCAE-V/dc-ae-v-1.0-f32t4c32-bf16.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebbee483c80890c7795a2babc06e1cb4d42564174cf3bae2b698c0589d95cc3b
+size 4277389995

VAE/PixelUMM_FT/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0bbfe8fd44bd2ebf1059fe65079caba50fd6f52ea8e1b5b35393e3050b9b8378
+size 31553868864

VAE/PixelUMM_FT_FP8/diffusion_pytorch_model_fp8.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ada04d7cedf9fe95bbaf87b4a8c0334944b312af5571568853b37d9b7256d78
+size 9134846208

examples/prompts.json

@@ -0,0 +1,32 @@
+[
+  {
+    "test_id": "forest_world",
+    "Style": "Cinematic fantasy",
+    "perspective_view": "Third-person view",
+    "subject_list": [
+      {
+        "subject_id": "fox",
+        "display_name": "a small orange fox",
+        "subject_attribute": "Orange fur, white chest, and a fluffy tail."
+      }
+    ],
+    "segments": [
+      {
+        "start_time": "00:00.000",
+        "end_time": "00:05.000",
+        "visual_description_long": "A small orange fox stands on a stone path in a luminous forest. Blue fireflies drift between the trees.",
+        "contained_actions": [
+          {"start_time": "00:00.000", "end_time": "00:05.000", "action_description": "The fox walks slowly along the path."}
+        ]
+      },
+      {
+        "start_time": "00:05.000",
+        "end_time": "00:10.000",
+        "description_change": "Glowing blue flowers open beside the path as the fox continues forward.",
+        "contained_actions": [
+          {"start_time": "00:05.000", "end_time": "00:10.000", "action_description": "The fox passes the flowers and looks around."}
+        ]
+      }
+    ]
+  }
+]