Spaces:
Running
on
Zero
Running
on
Zero
Initial commit
Browse files- .gitattributes +38 -35
- .gitignore +219 -0
- .python-version +1 -0
- README.md +17 -12
- app.py +500 -0
- assets/examples/Booster_train_balanced_Bathroom_camera_00_im0_png_00000_0000-0001.jpg +3 -0
- assets/examples/Booster_train_balanced_Bathroom_camera_00_im0_png_00000_0000-0001.mp4 +3 -0
- assets/examples/Booster_train_balanced_Bathroom_camera_00_im0_png_00000_0000-0001.ply +3 -0
- assets/examples/ETH3D_courtyard_00000_0000-0001.jpg +3 -0
- assets/examples/ETH3D_courtyard_00000_0000-0001.mp4 +3 -0
- assets/examples/ETH3D_courtyard_00000_0000-0001.ply +3 -0
- assets/examples/Middlebury_49b2bcfdd9_000_0000-0001.jpg +3 -0
- assets/examples/Middlebury_49b2bcfdd9_000_0000-0001.mp4 +3 -0
- assets/examples/Middlebury_49b2bcfdd9_000_0000-0001.ply +3 -0
- assets/examples/ScanNetPP_09c1414f1b_00000_0000-0001.jpg +3 -0
- assets/examples/ScanNetPP_09c1414f1b_00000_0000-0001.mp4 +3 -0
- assets/examples/ScanNetPP_09c1414f1b_00000_0000-0001.ply +3 -0
- assets/examples/TanksAndTemples_Church_00022_0000-0002.jpg +3 -0
- assets/examples/TanksAndTemples_Church_00022_0000-0002.mp4 +3 -0
- assets/examples/TanksAndTemples_Church_00022_0000-0002.ply +3 -0
- assets/examples/Unsplash_-591oIJnyEQ_0000-0001.jpg +3 -0
- assets/examples/Unsplash_-591oIJnyEQ_0000-0001.mp4 +3 -0
- assets/examples/Unsplash_-591oIJnyEQ_0000-0001.ply +3 -0
- assets/examples/Unsplash_SharpPaperVideo_-B_lu05yfgE_0000-0001.jpg +3 -0
- assets/examples/Unsplash_SharpPaperVideo_-B_lu05yfgE_0000-0001.mp4 +3 -0
- assets/examples/Unsplash_SharpPaperVideo_-B_lu05yfgE_0000-0001.ply +3 -0
- assets/examples/WildRGBD_TV_scene_000_00028_0000-0002.jpg +0 -0
- assets/examples/WildRGBD_TV_scene_000_00028_0000-0002.mp4 +3 -0
- assets/examples/WildRGBD_TV_scene_000_00028_0000-0002.ply +3 -0
- assets/examples/manifest.json +11 -0
- model_utils.py +612 -0
- pyproject.toml +23 -0
- requirements.txt +6 -0
- uv.lock +0 -0
.gitattributes
CHANGED
|
@@ -1,35 +1,38 @@
|
|
| 1 |
-
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
-
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
-
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
-
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
-
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
-
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
-
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
-
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
-
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
-
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
-
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
-
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
-
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
-
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
-
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
-
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
-
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
-
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
-
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
-
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
-
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
-
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
-
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
-
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
-
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
-
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
-
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
-
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
-
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
-
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
-
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
-
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
-
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
-
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
-
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
+
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
+
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
+
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
+
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
+
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
+
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
+
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
+
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
+
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
+
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
+
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
+
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
+
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
+
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
+
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
+
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
+
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
+
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
+
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
+
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
+
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
+
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
+
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
+
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
+
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
+
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
+
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
+
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
+
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
+
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
+
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
*.ply filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
*.jpg filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
*.mp4 filter=lfs diff=lfs merge=lfs -text
|
.gitignore
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Byte-compiled / optimized / DLL files
|
| 2 |
+
__pycache__/
|
| 3 |
+
*.py[codz]
|
| 4 |
+
*$py.class
|
| 5 |
+
|
| 6 |
+
# C extensions
|
| 7 |
+
*.so
|
| 8 |
+
|
| 9 |
+
# Distribution / packaging
|
| 10 |
+
.Python
|
| 11 |
+
build/
|
| 12 |
+
develop-eggs/
|
| 13 |
+
dist/
|
| 14 |
+
downloads/
|
| 15 |
+
eggs/
|
| 16 |
+
.eggs/
|
| 17 |
+
lib/
|
| 18 |
+
lib64/
|
| 19 |
+
parts/
|
| 20 |
+
sdist/
|
| 21 |
+
var/
|
| 22 |
+
wheels/
|
| 23 |
+
share/python-wheels/
|
| 24 |
+
*.egg-info/
|
| 25 |
+
.installed.cfg
|
| 26 |
+
*.egg
|
| 27 |
+
MANIFEST
|
| 28 |
+
|
| 29 |
+
# PyInstaller
|
| 30 |
+
# Usually these files are written by a python script from a template
|
| 31 |
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
| 32 |
+
*.manifest
|
| 33 |
+
*.spec
|
| 34 |
+
|
| 35 |
+
# Installer logs
|
| 36 |
+
pip-log.txt
|
| 37 |
+
pip-delete-this-directory.txt
|
| 38 |
+
|
| 39 |
+
# Unit test / coverage reports
|
| 40 |
+
htmlcov/
|
| 41 |
+
.tox/
|
| 42 |
+
.nox/
|
| 43 |
+
.coverage
|
| 44 |
+
.coverage.*
|
| 45 |
+
.cache
|
| 46 |
+
nosetests.xml
|
| 47 |
+
coverage.xml
|
| 48 |
+
*.cover
|
| 49 |
+
*.py.cover
|
| 50 |
+
.hypothesis/
|
| 51 |
+
.pytest_cache/
|
| 52 |
+
cover/
|
| 53 |
+
|
| 54 |
+
# Translations
|
| 55 |
+
*.mo
|
| 56 |
+
*.pot
|
| 57 |
+
|
| 58 |
+
# Django stuff:
|
| 59 |
+
*.log
|
| 60 |
+
local_settings.py
|
| 61 |
+
db.sqlite3
|
| 62 |
+
db.sqlite3-journal
|
| 63 |
+
|
| 64 |
+
# Flask stuff:
|
| 65 |
+
instance/
|
| 66 |
+
.webassets-cache
|
| 67 |
+
|
| 68 |
+
# Scrapy stuff:
|
| 69 |
+
.scrapy
|
| 70 |
+
|
| 71 |
+
# Sphinx documentation
|
| 72 |
+
docs/_build/
|
| 73 |
+
|
| 74 |
+
# PyBuilder
|
| 75 |
+
.pybuilder/
|
| 76 |
+
target/
|
| 77 |
+
|
| 78 |
+
# Jupyter Notebook
|
| 79 |
+
.ipynb_checkpoints
|
| 80 |
+
|
| 81 |
+
# IPython
|
| 82 |
+
profile_default/
|
| 83 |
+
ipython_config.py
|
| 84 |
+
|
| 85 |
+
# pyenv
|
| 86 |
+
# For a library or package, you might want to ignore these files since the code is
|
| 87 |
+
# intended to run in multiple environments; otherwise, check them in:
|
| 88 |
+
# .python-version
|
| 89 |
+
|
| 90 |
+
# pipenv
|
| 91 |
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
| 92 |
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
| 93 |
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
| 94 |
+
# install all needed dependencies.
|
| 95 |
+
# Pipfile.lock
|
| 96 |
+
|
| 97 |
+
# UV
|
| 98 |
+
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
| 99 |
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
| 100 |
+
# commonly ignored for libraries.
|
| 101 |
+
# uv.lock
|
| 102 |
+
|
| 103 |
+
# poetry
|
| 104 |
+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
| 105 |
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
| 106 |
+
# commonly ignored for libraries.
|
| 107 |
+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
| 108 |
+
# poetry.lock
|
| 109 |
+
# poetry.toml
|
| 110 |
+
|
| 111 |
+
# pdm
|
| 112 |
+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
| 113 |
+
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
|
| 114 |
+
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
|
| 115 |
+
# pdm.lock
|
| 116 |
+
# pdm.toml
|
| 117 |
+
.pdm-python
|
| 118 |
+
.pdm-build/
|
| 119 |
+
|
| 120 |
+
# pixi
|
| 121 |
+
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
|
| 122 |
+
# pixi.lock
|
| 123 |
+
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
|
| 124 |
+
# in the .venv directory. It is recommended not to include this directory in version control.
|
| 125 |
+
.pixi
|
| 126 |
+
|
| 127 |
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
| 128 |
+
__pypackages__/
|
| 129 |
+
|
| 130 |
+
# Celery stuff
|
| 131 |
+
celerybeat-schedule
|
| 132 |
+
celerybeat.pid
|
| 133 |
+
|
| 134 |
+
# Redis
|
| 135 |
+
*.rdb
|
| 136 |
+
*.aof
|
| 137 |
+
*.pid
|
| 138 |
+
|
| 139 |
+
# RabbitMQ
|
| 140 |
+
mnesia/
|
| 141 |
+
rabbitmq/
|
| 142 |
+
rabbitmq-data/
|
| 143 |
+
|
| 144 |
+
# ActiveMQ
|
| 145 |
+
activemq-data/
|
| 146 |
+
|
| 147 |
+
# SageMath parsed files
|
| 148 |
+
*.sage.py
|
| 149 |
+
|
| 150 |
+
# Environments
|
| 151 |
+
.env
|
| 152 |
+
.envrc
|
| 153 |
+
.venv
|
| 154 |
+
env/
|
| 155 |
+
venv/
|
| 156 |
+
ENV/
|
| 157 |
+
env.bak/
|
| 158 |
+
venv.bak/
|
| 159 |
+
|
| 160 |
+
# Spyder project settings
|
| 161 |
+
.spyderproject
|
| 162 |
+
.spyproject
|
| 163 |
+
|
| 164 |
+
# Rope project settings
|
| 165 |
+
.ropeproject
|
| 166 |
+
|
| 167 |
+
# mkdocs documentation
|
| 168 |
+
/site
|
| 169 |
+
|
| 170 |
+
# mypy
|
| 171 |
+
.mypy_cache/
|
| 172 |
+
.dmypy.json
|
| 173 |
+
dmypy.json
|
| 174 |
+
|
| 175 |
+
# Pyre type checker
|
| 176 |
+
.pyre/
|
| 177 |
+
|
| 178 |
+
# pytype static type analyzer
|
| 179 |
+
.pytype/
|
| 180 |
+
|
| 181 |
+
# Cython debug symbols
|
| 182 |
+
cython_debug/
|
| 183 |
+
|
| 184 |
+
# PyCharm
|
| 185 |
+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
| 186 |
+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
| 187 |
+
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
| 188 |
+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
| 189 |
+
# .idea/
|
| 190 |
+
|
| 191 |
+
# Abstra
|
| 192 |
+
# Abstra is an AI-powered process automation framework.
|
| 193 |
+
# Ignore directories containing user credentials, local state, and settings.
|
| 194 |
+
# Learn more at https://abstra.io/docs
|
| 195 |
+
.abstra/
|
| 196 |
+
|
| 197 |
+
# Visual Studio Code
|
| 198 |
+
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
|
| 199 |
+
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
|
| 200 |
+
# and can be added to the global gitignore or merged into this file. However, if you prefer,
|
| 201 |
+
# you could uncomment the following to ignore the entire vscode folder
|
| 202 |
+
# .vscode/
|
| 203 |
+
|
| 204 |
+
# Ruff stuff:
|
| 205 |
+
.ruff_cache/
|
| 206 |
+
|
| 207 |
+
# PyPI configuration file
|
| 208 |
+
.pypirc
|
| 209 |
+
|
| 210 |
+
# Marimo
|
| 211 |
+
marimo/_static/
|
| 212 |
+
marimo/_lsp/
|
| 213 |
+
__marimo__/
|
| 214 |
+
|
| 215 |
+
# Streamlit
|
| 216 |
+
.streamlit/secrets.toml
|
| 217 |
+
|
| 218 |
+
# Kilo Code
|
| 219 |
+
.kilocode/
|
.python-version
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
3.13
|
README.md
CHANGED
|
@@ -1,12 +1,17 @@
|
|
| 1 |
-
---
|
| 2 |
-
title:
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo:
|
| 6 |
-
sdk: gradio
|
| 7 |
-
sdk_version: 6.1.0
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
title: "SHARP - 3D Gaussian Scene Prediction"
|
| 3 |
+
emoji: 🔪
|
| 4 |
+
colorFrom: purple
|
| 5 |
+
colorTo: indigo
|
| 6 |
+
sdk: gradio
|
| 7 |
+
sdk_version: 6.1.0
|
| 8 |
+
python_version: 3.13
|
| 9 |
+
app_file: app.py
|
| 10 |
+
pinned: false
|
| 11 |
+
short_description: "Sharp Monocular View Synthesis in Less Than a Second"
|
| 12 |
+
models:
|
| 13 |
+
- apple/Sharp
|
| 14 |
+
startup_duration_timeout: 1h
|
| 15 |
+
preload_from_hub:
|
| 16 |
+
- apple/Sharp sharp_2572gikvuh.pt
|
| 17 |
+
---
|
app.py
ADDED
|
@@ -0,0 +1,500 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""SHARP Gradio demo (minimal, responsive UI).
|
| 2 |
+
|
| 3 |
+
This Space:
|
| 4 |
+
- Runs Apple's SHARP model to predict a 3D Gaussian scene from a single image.
|
| 5 |
+
- Exports a canonical `.ply` file for download.
|
| 6 |
+
- Optionally renders a camera trajectory `.mp4` (CUDA / ZeroGPU only).
|
| 7 |
+
|
| 8 |
+
Precompiled examples
|
| 9 |
+
Place precompiled examples under `assets/examples/`.
|
| 10 |
+
|
| 11 |
+
Recommended structure (matching stem):
|
| 12 |
+
assets/examples/<name>.jpg|png|webp
|
| 13 |
+
assets/examples/<name>.mp4
|
| 14 |
+
assets/examples/<name>.ply
|
| 15 |
+
|
| 16 |
+
Optional manifest (assets/examples/manifest.json):
|
| 17 |
+
[
|
| 18 |
+
{"label": "Desk", "image": "desk.jpg", "video": "desk.mp4", "ply": "desk.ply"},
|
| 19 |
+
...
|
| 20 |
+
]
|
| 21 |
+
"""
|
| 22 |
+
|
| 23 |
+
from __future__ import annotations
|
| 24 |
+
|
| 25 |
+
import json
|
| 26 |
+
from dataclasses import dataclass
|
| 27 |
+
from pathlib import Path
|
| 28 |
+
from typing import Final
|
| 29 |
+
|
| 30 |
+
import gradio as gr
|
| 31 |
+
|
| 32 |
+
from model_utils import TrajectoryType, predict_and_maybe_render_gpu
|
| 33 |
+
|
| 34 |
+
# -----------------------------------------------------------------------------
|
| 35 |
+
# Paths & constants
|
| 36 |
+
# -----------------------------------------------------------------------------
|
| 37 |
+
|
| 38 |
+
APP_DIR: Final[Path] = Path(__file__).resolve().parent
|
| 39 |
+
OUTPUTS_DIR: Final[Path] = APP_DIR / "outputs"
|
| 40 |
+
ASSETS_DIR: Final[Path] = APP_DIR / "assets"
|
| 41 |
+
EXAMPLES_DIR: Final[Path] = ASSETS_DIR / "examples"
|
| 42 |
+
|
| 43 |
+
IMAGE_EXTS: Final[tuple[str, ...]] = (".png", ".jpg", ".jpeg", ".webp")
|
| 44 |
+
DEFAULT_QUEUE_MAX_SIZE: Final[int] = 32
|
| 45 |
+
|
| 46 |
+
THEME: Final = gr.themes.Soft(
|
| 47 |
+
primary_hue="indigo",
|
| 48 |
+
secondary_hue="blue",
|
| 49 |
+
neutral_hue="slate",
|
| 50 |
+
)
|
| 51 |
+
|
| 52 |
+
CSS: Final[str] = """
|
| 53 |
+
/* Keep layout stable when scrollbars appear/disappear */
|
| 54 |
+
html { scrollbar-gutter: stable; }
|
| 55 |
+
|
| 56 |
+
/* Use normal document flow (no fixed-height viewport shell) */
|
| 57 |
+
html, body { height: auto; }
|
| 58 |
+
body { overflow: auto; }
|
| 59 |
+
|
| 60 |
+
/* Comfortable max width; still fills small screens */
|
| 61 |
+
.gradio-container {
|
| 62 |
+
max-width: 1400px;
|
| 63 |
+
margin: 0 auto;
|
| 64 |
+
padding: 0.75rem 1rem 1rem;
|
| 65 |
+
box-sizing: border-box;
|
| 66 |
+
}
|
| 67 |
+
|
| 68 |
+
/* Make media components responsive without stretching */
|
| 69 |
+
#run-image, #run-video,
|
| 70 |
+
#examples-image, #examples-video {
|
| 71 |
+
width: 100%;
|
| 72 |
+
}
|
| 73 |
+
|
| 74 |
+
/* Keep aspect ratio and prevent runaway vertical growth on tall viewports */
|
| 75 |
+
#run-image img, #examples-image img {
|
| 76 |
+
width: 100%;
|
| 77 |
+
height: auto;
|
| 78 |
+
max-height: 70vh;
|
| 79 |
+
object-fit: contain;
|
| 80 |
+
}
|
| 81 |
+
#run-video video, #examples-video video {
|
| 82 |
+
width: 100%;
|
| 83 |
+
height: auto;
|
| 84 |
+
max-height: 70vh;
|
| 85 |
+
object-fit: contain;
|
| 86 |
+
}
|
| 87 |
+
|
| 88 |
+
/* On very small screens, reduce max media height a bit */
|
| 89 |
+
@media (max-width: 640px) {
|
| 90 |
+
#run-image img, #examples-image img,
|
| 91 |
+
#run-video video, #examples-video video {
|
| 92 |
+
max-height: 55vh;
|
| 93 |
+
}
|
| 94 |
+
}
|
| 95 |
+
|
| 96 |
+
/* Reduce extra whitespace in markdown blocks */
|
| 97 |
+
.gr-markdown > :first-child { margin-top: 0 !important; }
|
| 98 |
+
.gr-markdown > :last-child { margin-bottom: 0 !important; }
|
| 99 |
+
"""
|
| 100 |
+
|
| 101 |
+
# -----------------------------------------------------------------------------
|
| 102 |
+
# Helpers
|
| 103 |
+
# -----------------------------------------------------------------------------
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def _ensure_dir(path: Path) -> Path:
|
| 107 |
+
path.mkdir(parents=True, exist_ok=True)
|
| 108 |
+
return path
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
@dataclass(frozen=True, slots=True)
|
| 112 |
+
class ExampleSpec:
|
| 113 |
+
"""A precompiled example bundle (image + optional mp4 + optional ply)."""
|
| 114 |
+
|
| 115 |
+
label: str
|
| 116 |
+
image: Path
|
| 117 |
+
video: Path | None
|
| 118 |
+
ply: Path | None
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def _normalize_key(path: str) -> str:
|
| 122 |
+
"""Normalize a path-like string for stable dictionary keys."""
|
| 123 |
+
try:
|
| 124 |
+
return str(Path(path).resolve())
|
| 125 |
+
except Exception:
|
| 126 |
+
return path
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def _load_manifest(manifest_path: Path) -> list[dict]:
|
| 130 |
+
"""Load manifest.json if present; return an empty list on errors."""
|
| 131 |
+
try:
|
| 132 |
+
data = json.loads(manifest_path.read_text(encoding="utf-8"))
|
| 133 |
+
if not isinstance(data, list):
|
| 134 |
+
raise ValueError("manifest.json must contain a JSON list.")
|
| 135 |
+
return [x for x in data if isinstance(x, dict)]
|
| 136 |
+
except FileNotFoundError:
|
| 137 |
+
return []
|
| 138 |
+
except Exception as e:
|
| 139 |
+
# Manifest errors should not crash the app.
|
| 140 |
+
print(f"[examples] Failed to parse manifest.json: {type(e).__name__}: {e}")
|
| 141 |
+
return []
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
def discover_examples(examples_dir: Path) -> list[ExampleSpec]:
|
| 145 |
+
"""Discover example bundles under assets/examples/."""
|
| 146 |
+
_ensure_dir(examples_dir)
|
| 147 |
+
|
| 148 |
+
manifest_rows = _load_manifest(examples_dir / "manifest.json")
|
| 149 |
+
if manifest_rows:
|
| 150 |
+
specs: list[ExampleSpec] = []
|
| 151 |
+
for row in manifest_rows:
|
| 152 |
+
label = str(row.get("label") or "Example").strip() or "Example"
|
| 153 |
+
image_rel = row.get("image")
|
| 154 |
+
if not image_rel:
|
| 155 |
+
continue
|
| 156 |
+
|
| 157 |
+
image = (examples_dir / str(image_rel)).resolve()
|
| 158 |
+
if not image.exists():
|
| 159 |
+
continue
|
| 160 |
+
|
| 161 |
+
video = None
|
| 162 |
+
ply = None
|
| 163 |
+
if row.get("video"):
|
| 164 |
+
v = (examples_dir / str(row["video"])).resolve()
|
| 165 |
+
if v.exists():
|
| 166 |
+
video = v
|
| 167 |
+
if row.get("ply"):
|
| 168 |
+
p = (examples_dir / str(row["ply"])).resolve()
|
| 169 |
+
if p.exists():
|
| 170 |
+
ply = p
|
| 171 |
+
|
| 172 |
+
specs.append(ExampleSpec(label=label, image=image, video=video, ply=ply))
|
| 173 |
+
return specs
|
| 174 |
+
|
| 175 |
+
# Fallback: infer bundles by filename stem
|
| 176 |
+
images: list[Path] = []
|
| 177 |
+
for ext in IMAGE_EXTS:
|
| 178 |
+
images.extend(sorted(examples_dir.glob(f"*{ext}")))
|
| 179 |
+
|
| 180 |
+
specs = []
|
| 181 |
+
for img in images:
|
| 182 |
+
stem = img.stem
|
| 183 |
+
video = examples_dir / f"{stem}.mp4"
|
| 184 |
+
ply = examples_dir / f"{stem}.ply"
|
| 185 |
+
specs.append(
|
| 186 |
+
ExampleSpec(
|
| 187 |
+
label=stem.replace("_", " ").strip() or stem,
|
| 188 |
+
image=img.resolve(),
|
| 189 |
+
video=video.resolve() if video.exists() else None,
|
| 190 |
+
ply=ply.resolve() if ply.exists() else None,
|
| 191 |
+
)
|
| 192 |
+
)
|
| 193 |
+
return specs
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
_ensure_dir(OUTPUTS_DIR)
|
| 197 |
+
|
| 198 |
+
EXAMPLE_SPECS: Final[list[ExampleSpec]] = discover_examples(EXAMPLES_DIR)
|
| 199 |
+
EXAMPLE_INDEX_BY_PATH: Final[dict[str, ExampleSpec]] = {
|
| 200 |
+
_normalize_key(str(s.image)): s for s in EXAMPLE_SPECS
|
| 201 |
+
}
|
| 202 |
+
EXAMPLE_INDEX_BY_NAME: Final[dict[str, ExampleSpec]] = {
|
| 203 |
+
s.image.name: s for s in EXAMPLE_SPECS
|
| 204 |
+
}
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
def load_example_assets(
|
| 208 |
+
image_path: str | None,
|
| 209 |
+
) -> tuple[str | None, str | None, str | None, str]:
|
| 210 |
+
"""Return (image, video, ply_path, status) for the selected example image."""
|
| 211 |
+
if not image_path:
|
| 212 |
+
return None, None, None, "No example selected."
|
| 213 |
+
|
| 214 |
+
spec = EXAMPLE_INDEX_BY_PATH.get(_normalize_key(image_path))
|
| 215 |
+
if spec is None:
|
| 216 |
+
spec = EXAMPLE_INDEX_BY_NAME.get(Path(image_path).name)
|
| 217 |
+
|
| 218 |
+
if spec is None:
|
| 219 |
+
return image_path, None, None, "No matching example bundle found."
|
| 220 |
+
|
| 221 |
+
video = str(spec.video) if spec.video is not None else None
|
| 222 |
+
ply_path = str(spec.ply) if spec.ply is not None else None
|
| 223 |
+
|
| 224 |
+
missing: list[str] = []
|
| 225 |
+
if video is None:
|
| 226 |
+
missing.append("MP4")
|
| 227 |
+
if ply_path is None:
|
| 228 |
+
missing.append("PLY")
|
| 229 |
+
|
| 230 |
+
msg = f"Loaded example: **{spec.label}**."
|
| 231 |
+
if missing:
|
| 232 |
+
msg += f" Missing: {', '.join(missing)}."
|
| 233 |
+
|
| 234 |
+
return str(spec.image), video, ply_path, msg
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
def _validate_image(image_path: str | None) -> None:
|
| 238 |
+
if not image_path:
|
| 239 |
+
raise gr.Error("Upload an image first.")
|
| 240 |
+
|
| 241 |
+
|
| 242 |
+
def run_sharp(
|
| 243 |
+
image_path: str | None,
|
| 244 |
+
trajectory_type: TrajectoryType,
|
| 245 |
+
output_long_side: int,
|
| 246 |
+
num_frames: int,
|
| 247 |
+
fps: int,
|
| 248 |
+
render_video: bool,
|
| 249 |
+
) -> tuple[str | None, str | None, str]:
|
| 250 |
+
"""Run SHARP inference and return (video_path, ply_path, status_markdown)."""
|
| 251 |
+
_validate_image(image_path)
|
| 252 |
+
out_long_side: int | None = (
|
| 253 |
+
None if int(output_long_side) <= 0 else int(output_long_side)
|
| 254 |
+
)
|
| 255 |
+
|
| 256 |
+
try:
|
| 257 |
+
video_path, ply_path = predict_and_maybe_render_gpu(
|
| 258 |
+
image_path,
|
| 259 |
+
trajectory_type=trajectory_type,
|
| 260 |
+
num_frames=int(num_frames),
|
| 261 |
+
fps=int(fps),
|
| 262 |
+
output_long_side=out_long_side,
|
| 263 |
+
render_video=bool(render_video),
|
| 264 |
+
)
|
| 265 |
+
|
| 266 |
+
lines: list[str] = [f"**PLY:** `{ply_path.name}` (ready to download)"]
|
| 267 |
+
if render_video:
|
| 268 |
+
if video_path is None:
|
| 269 |
+
lines.append("**Video:** not rendered (CUDA unavailable).")
|
| 270 |
+
else:
|
| 271 |
+
lines.append(f"**Video:** `{video_path.name}`")
|
| 272 |
+
else:
|
| 273 |
+
lines.append("**Video:** disabled.")
|
| 274 |
+
|
| 275 |
+
return (
|
| 276 |
+
str(video_path) if video_path is not None else None,
|
| 277 |
+
str(ply_path),
|
| 278 |
+
"\n".join(lines),
|
| 279 |
+
)
|
| 280 |
+
except gr.Error:
|
| 281 |
+
raise
|
| 282 |
+
except Exception as e:
|
| 283 |
+
raise gr.Error(f"SHARP failed: {type(e).__name__}: {e}") from e
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
# -----------------------------------------------------------------------------
|
| 287 |
+
# UI
|
| 288 |
+
# -----------------------------------------------------------------------------
|
| 289 |
+
|
| 290 |
+
|
| 291 |
+
def build_demo() -> gr.Blocks:
|
| 292 |
+
with gr.Blocks(
|
| 293 |
+
title="SHARP • Single-Image 3D Gaussian Prediction",
|
| 294 |
+
elem_id="sharp-root",
|
| 295 |
+
fill_height=True,
|
| 296 |
+
) as demo:
|
| 297 |
+
gr.Markdown("## SHARP\nSingle-image **3D Gaussian scene** prediction.")
|
| 298 |
+
|
| 299 |
+
# Run tab components are referenced by Examples tab, so keep them in outer scope.
|
| 300 |
+
with gr.Column(elem_id="tabs-shell"):
|
| 301 |
+
with gr.Tabs():
|
| 302 |
+
with gr.Tab("Run", id="run"):
|
| 303 |
+
with gr.Column(elem_id="run-panel"):
|
| 304 |
+
with gr.Row(equal_height=True, elem_id="run-media-row"):
|
| 305 |
+
with gr.Column(
|
| 306 |
+
scale=5, min_width=360, elem_id="run-left-col"
|
| 307 |
+
):
|
| 308 |
+
image_in = gr.Image(
|
| 309 |
+
label="Input image",
|
| 310 |
+
type="filepath",
|
| 311 |
+
sources=["upload"],
|
| 312 |
+
elem_id="run-image",
|
| 313 |
+
)
|
| 314 |
+
|
| 315 |
+
with gr.Row():
|
| 316 |
+
trajectory = gr.Dropdown(
|
| 317 |
+
label="Trajectory",
|
| 318 |
+
choices=[
|
| 319 |
+
"swipe",
|
| 320 |
+
"shake",
|
| 321 |
+
"rotate",
|
| 322 |
+
"rotate_forward",
|
| 323 |
+
],
|
| 324 |
+
value="rotate_forward",
|
| 325 |
+
)
|
| 326 |
+
output_res = gr.Dropdown(
|
| 327 |
+
label="Output long side",
|
| 328 |
+
info="0 = match input",
|
| 329 |
+
choices=[
|
| 330 |
+
("Match input", 0),
|
| 331 |
+
("512", 512),
|
| 332 |
+
("768", 768),
|
| 333 |
+
("1024", 1024),
|
| 334 |
+
("1280", 1280),
|
| 335 |
+
("1536", 1536),
|
| 336 |
+
],
|
| 337 |
+
value=0,
|
| 338 |
+
)
|
| 339 |
+
|
| 340 |
+
with gr.Row():
|
| 341 |
+
frames = gr.Slider(
|
| 342 |
+
label="Frames",
|
| 343 |
+
minimum=24,
|
| 344 |
+
maximum=120,
|
| 345 |
+
step=1,
|
| 346 |
+
value=60,
|
| 347 |
+
)
|
| 348 |
+
fps_in = gr.Slider(
|
| 349 |
+
label="FPS",
|
| 350 |
+
minimum=8,
|
| 351 |
+
maximum=60,
|
| 352 |
+
step=1,
|
| 353 |
+
value=30,
|
| 354 |
+
)
|
| 355 |
+
|
| 356 |
+
render_toggle = gr.Checkbox(
|
| 357 |
+
label="Render MP4 (CUDA / ZeroGPU only)",
|
| 358 |
+
value=True,
|
| 359 |
+
)
|
| 360 |
+
|
| 361 |
+
with gr.Column(
|
| 362 |
+
scale=5, min_width=360, elem_id="run-right-col"
|
| 363 |
+
):
|
| 364 |
+
video_out = gr.Video(
|
| 365 |
+
label="Trajectory video (MP4)",
|
| 366 |
+
elem_id="run-video",
|
| 367 |
+
)
|
| 368 |
+
with gr.Row(elem_id="run-download-row"):
|
| 369 |
+
ply_download = gr.DownloadButton(
|
| 370 |
+
label="Download PLY (.ply)",
|
| 371 |
+
value=None,
|
| 372 |
+
visible=True,
|
| 373 |
+
elem_id="run-ply-download",
|
| 374 |
+
)
|
| 375 |
+
status_md = gr.Markdown("", elem_id="run-status")
|
| 376 |
+
|
| 377 |
+
with gr.Row(elem_id="run-actions-row"):
|
| 378 |
+
run_btn = gr.Button("Generate", variant="primary")
|
| 379 |
+
clear_btn = gr.ClearButton(
|
| 380 |
+
[image_in, video_out, ply_download, status_md],
|
| 381 |
+
value="Clear",
|
| 382 |
+
)
|
| 383 |
+
|
| 384 |
+
# Ensure clearing also clears any previous download target.
|
| 385 |
+
clear_btn.click(
|
| 386 |
+
fn=lambda: None,
|
| 387 |
+
outputs=[ply_download],
|
| 388 |
+
queue=False,
|
| 389 |
+
)
|
| 390 |
+
|
| 391 |
+
run_btn.click(
|
| 392 |
+
fn=run_sharp,
|
| 393 |
+
inputs=[
|
| 394 |
+
image_in,
|
| 395 |
+
trajectory,
|
| 396 |
+
output_res,
|
| 397 |
+
frames,
|
| 398 |
+
fps_in,
|
| 399 |
+
render_toggle,
|
| 400 |
+
],
|
| 401 |
+
outputs=[video_out, ply_download, status_md],
|
| 402 |
+
api_visibility="public",
|
| 403 |
+
)
|
| 404 |
+
|
| 405 |
+
with gr.Tab("Examples", id="examples"):
|
| 406 |
+
with gr.Column(elem_id="examples-panel"):
|
| 407 |
+
if EXAMPLE_SPECS:
|
| 408 |
+
gr.Markdown(
|
| 409 |
+
"Click an example to preview precompiled outputs. "
|
| 410 |
+
"The example image will also be loaded into the Run tab."
|
| 411 |
+
)
|
| 412 |
+
|
| 413 |
+
# Define preview outputs first (unrendered), so we can reference them from gr.Examples.
|
| 414 |
+
ex_img = gr.Image(
|
| 415 |
+
label="Example image",
|
| 416 |
+
type="filepath",
|
| 417 |
+
interactive=False,
|
| 418 |
+
render=False,
|
| 419 |
+
height=360,
|
| 420 |
+
elem_id="examples-image",
|
| 421 |
+
)
|
| 422 |
+
ex_vid = gr.Video(
|
| 423 |
+
label="Pre-rendered MP4",
|
| 424 |
+
render=False,
|
| 425 |
+
height=360,
|
| 426 |
+
elem_id="examples-video",
|
| 427 |
+
)
|
| 428 |
+
ex_ply = gr.DownloadButton(
|
| 429 |
+
label="Download PLY (.ply)",
|
| 430 |
+
value=None,
|
| 431 |
+
visible=True,
|
| 432 |
+
render=False,
|
| 433 |
+
elem_id="examples-ply-download",
|
| 434 |
+
)
|
| 435 |
+
ex_status = gr.Markdown(
|
| 436 |
+
render=False, elem_id="examples-status"
|
| 437 |
+
)
|
| 438 |
+
|
| 439 |
+
with gr.Row(equal_height=True):
|
| 440 |
+
with gr.Column(scale=4, min_width=320):
|
| 441 |
+
gr.Examples(
|
| 442 |
+
examples=[
|
| 443 |
+
[str(s.image)] for s in EXAMPLE_SPECS
|
| 444 |
+
],
|
| 445 |
+
example_labels=[s.label for s in EXAMPLE_SPECS],
|
| 446 |
+
inputs=[image_in],
|
| 447 |
+
outputs=[ex_img, ex_vid, ex_ply, ex_status],
|
| 448 |
+
fn=load_example_assets,
|
| 449 |
+
cache_examples=False,
|
| 450 |
+
run_on_click=True,
|
| 451 |
+
examples_per_page=10,
|
| 452 |
+
label=None,
|
| 453 |
+
)
|
| 454 |
+
|
| 455 |
+
with gr.Column(scale=6, min_width=360):
|
| 456 |
+
ex_img.render()
|
| 457 |
+
ex_vid.render()
|
| 458 |
+
ex_ply.render()
|
| 459 |
+
ex_status.render()
|
| 460 |
+
|
| 461 |
+
gr.Markdown(
|
| 462 |
+
"Add example bundles under `assets/examples/` "
|
| 463 |
+
"(image + mp4 + ply) or provide a `manifest.json`."
|
| 464 |
+
)
|
| 465 |
+
else:
|
| 466 |
+
gr.Markdown(
|
| 467 |
+
"No precompiled examples found.\n\n"
|
| 468 |
+
"Add files under `assets/examples/`:\n"
|
| 469 |
+
"- `example.jpg` (or png/webp)\n"
|
| 470 |
+
"- `example.mp4`\n"
|
| 471 |
+
"- `example.ply`\n\n"
|
| 472 |
+
"Optionally add `assets/examples/manifest.json` to define labels and filenames."
|
| 473 |
+
)
|
| 474 |
+
|
| 475 |
+
with gr.Tab("About", id="about"):
|
| 476 |
+
with gr.Column(elem_id="about-panel"):
|
| 477 |
+
gr.Markdown(
|
| 478 |
+
"""
|
| 479 |
+
*Sharp Monocular View Synthesis in Less Than a Second* (Apple, 2025)
|
| 480 |
+
|
| 481 |
+
```bibtex
|
| 482 |
+
@inproceedings{Sharp2025:arxiv,
|
| 483 |
+
title = {Sharp Monocular View Synthesis in Less Than a Second},
|
| 484 |
+
author = {Lars Mescheder and Wei Dong and Shiwei Li and Xuyang Bai and Marcel Santos and Peiyun Hu and Bruno Lecouat and Mingmin Zhen and Ama\\"{e}l Delaunoyand Tian Fang and Yanghai Tsin and Stephan R. Richter and Vladlen Koltun},
|
| 485 |
+
journal = {arXiv preprint arXiv:2512.10685},
|
| 486 |
+
year = {2025},
|
| 487 |
+
url = {https://arxiv.org/abs/2512.10685},
|
| 488 |
+
}
|
| 489 |
+
```
|
| 490 |
+
""".strip()
|
| 491 |
+
)
|
| 492 |
+
|
| 493 |
+
demo.queue(max_size=DEFAULT_QUEUE_MAX_SIZE, default_concurrency_limit=1)
|
| 494 |
+
return demo
|
| 495 |
+
|
| 496 |
+
|
| 497 |
+
demo = build_demo()
|
| 498 |
+
|
| 499 |
+
if __name__ == "__main__":
|
| 500 |
+
demo.launch(theme=THEME, css=CSS)
|
assets/examples/Booster_train_balanced_Bathroom_camera_00_im0_png_00000_0000-0001.jpg
ADDED
|
Git LFS Details
|
assets/examples/Booster_train_balanced_Bathroom_camera_00_im0_png_00000_0000-0001.mp4
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:23946e8345738bec5052c11ef259490e8fa003a9f0c87c5cae4b0434d6b0b211
|
| 3 |
+
size 506496
|
assets/examples/Booster_train_balanced_Bathroom_camera_00_im0_png_00000_0000-0001.ply
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:54d28194b0ae41fd2a2f09b07de28d2305c5181b0664cd25ce29f9e114ba2ea8
|
| 3 |
+
size 66061086
|
assets/examples/ETH3D_courtyard_00000_0000-0001.jpg
ADDED
|
Git LFS Details
|
assets/examples/ETH3D_courtyard_00000_0000-0001.mp4
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a5bdbcc32493174b684aff6b2ab0701f4c037e40929991948a379c9d7c323792
|
| 3 |
+
size 538810
|
assets/examples/ETH3D_courtyard_00000_0000-0001.ply
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:282fe4719d2822ea66cf3ab02160ec6bc030c7a68bff8849033d679a9d522438
|
| 3 |
+
size 66061086
|
assets/examples/Middlebury_49b2bcfdd9_000_0000-0001.jpg
ADDED
|
Git LFS Details
|
assets/examples/Middlebury_49b2bcfdd9_000_0000-0001.mp4
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1b44afd2cf1fffdb3a3b00840461b17bbdf44036a6fed85df6fd69da6db6d256
|
| 3 |
+
size 573636
|
assets/examples/Middlebury_49b2bcfdd9_000_0000-0001.ply
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:109b98f9262368a69c55bb5771771c887576460efdd1c638924501b3170c35e7
|
| 3 |
+
size 66061086
|
assets/examples/ScanNetPP_09c1414f1b_00000_0000-0001.jpg
ADDED
|
Git LFS Details
|
assets/examples/ScanNetPP_09c1414f1b_00000_0000-0001.mp4
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:05b6cc0f5fb874474fe40ae7ce5af4326608c5f71a2bef0be6b9ab9041dec6e8
|
| 3 |
+
size 449544
|
assets/examples/ScanNetPP_09c1414f1b_00000_0000-0001.ply
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6ac8a893e652a3bce88b6dbface6effaea183d0fe9a8ebf2bc4ce0385738c74a
|
| 3 |
+
size 66061086
|
assets/examples/TanksAndTemples_Church_00022_0000-0002.jpg
ADDED
|
Git LFS Details
|
assets/examples/TanksAndTemples_Church_00022_0000-0002.mp4
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2270c0185e91fa2222190c5a5546c14b07b806071538c915b2c0aa7fbe6d90be
|
| 3 |
+
size 592290
|
assets/examples/TanksAndTemples_Church_00022_0000-0002.ply
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f859a1e39d6fd1d9e69a74718f3872e722744ca61260ad7bac1d5c20ad7401ee
|
| 3 |
+
size 66061086
|
assets/examples/Unsplash_-591oIJnyEQ_0000-0001.jpg
ADDED
|
Git LFS Details
|
assets/examples/Unsplash_-591oIJnyEQ_0000-0001.mp4
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:65f9f31b3a4a8a13e50b5133f16805e034582fad81f9125bed64eea5abf9b894
|
| 3 |
+
size 546457
|
assets/examples/Unsplash_-591oIJnyEQ_0000-0001.ply
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8593a2fc8f6e60f6bcbca2d4d52510ff972d3206e7259fac0d46a49383398296
|
| 3 |
+
size 66061086
|
assets/examples/Unsplash_SharpPaperVideo_-B_lu05yfgE_0000-0001.jpg
ADDED
|
Git LFS Details
|
assets/examples/Unsplash_SharpPaperVideo_-B_lu05yfgE_0000-0001.mp4
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0be43c1aa7d4295c670aef5f4cb99691cd1a66f04dc7cb41ef11f23c72c46b3c
|
| 3 |
+
size 616789
|
assets/examples/Unsplash_SharpPaperVideo_-B_lu05yfgE_0000-0001.ply
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0205454aef8b65c7b48d9598cfd4bf963f68e4032e87f00b8fb2da6707965679
|
| 3 |
+
size 66061086
|
assets/examples/WildRGBD_TV_scene_000_00028_0000-0002.jpg
ADDED
|
assets/examples/WildRGBD_TV_scene_000_00028_0000-0002.mp4
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c7745960cbfb1218a5a41fa58f801de922452c53b330230c075981ee83058404
|
| 3 |
+
size 113720
|
assets/examples/WildRGBD_TV_scene_000_00028_0000-0002.ply
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4bc4951045bdc51ddfdd4f7d874deebc1e99e32f6d29e3caa2c13f805f27aac9
|
| 3 |
+
size 66061086
|
assets/examples/manifest.json
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{"label": "Desk", "image": "desk.jpg", "video": "desk.mp4", "ply": "desk.ply"},
|
| 3 |
+
{"label": "Desk", "image": "desk.jpg", "video": "desk.mp4", "ply": "desk.ply"},
|
| 4 |
+
{"label": "Desk", "image": "desk.jpg", "video": "desk.mp4", "ply": "desk.ply"},
|
| 5 |
+
{"label": "Desk", "image": "desk.jpg", "video": "desk.mp4", "ply": "desk.ply"},
|
| 6 |
+
{"label": "Desk", "image": "desk.jpg", "video": "desk.mp4", "ply": "desk.ply"},
|
| 7 |
+
{"label": "Desk", "image": "desk.jpg", "video": "desk.mp4", "ply": "desk.ply"},
|
| 8 |
+
{"label": "Desk", "image": "desk.jpg", "video": "desk.mp4", "ply": "desk.ply"},
|
| 9 |
+
{"label": "Desk", "image": "desk.jpg", "video": "desk.mp4", "ply": "desk.ply"},
|
| 10 |
+
{"label": "Desk", "image": "desk.jpg", "video": "desk.mp4", "ply": "desk.ply"}
|
| 11 |
+
]
|
model_utils.py
ADDED
|
@@ -0,0 +1,612 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""SHARP inference + optional CUDA video rendering utilities.
|
| 2 |
+
|
| 3 |
+
Design goals:
|
| 4 |
+
- Reuse SHARP's own predict/render pipeline (no subprocess calls).
|
| 5 |
+
- Be robust on Hugging Face Spaces + ZeroGPU.
|
| 6 |
+
- Cache model weights and predictor construction across requests.
|
| 7 |
+
|
| 8 |
+
Public API (used by the Gradio app):
|
| 9 |
+
- TrajectoryType
|
| 10 |
+
- predict_and_maybe_render_gpu(...)
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
import os
|
| 16 |
+
import threading
|
| 17 |
+
import time
|
| 18 |
+
import uuid
|
| 19 |
+
from contextlib import contextmanager
|
| 20 |
+
from dataclasses import dataclass
|
| 21 |
+
from pathlib import Path
|
| 22 |
+
from typing import Final, Literal
|
| 23 |
+
|
| 24 |
+
import torch
|
| 25 |
+
|
| 26 |
+
try:
|
| 27 |
+
import spaces
|
| 28 |
+
except Exception: # pragma: no cover
|
| 29 |
+
spaces = None # type: ignore[assignment]
|
| 30 |
+
|
| 31 |
+
try:
|
| 32 |
+
# Prefer HF cache / Hub downloads (works with Spaces `preload_from_hub`).
|
| 33 |
+
from huggingface_hub import hf_hub_download, try_to_load_from_cache
|
| 34 |
+
except Exception: # pragma: no cover
|
| 35 |
+
hf_hub_download = None # type: ignore[assignment]
|
| 36 |
+
try_to_load_from_cache = None # type: ignore[assignment]
|
| 37 |
+
|
| 38 |
+
from sharp.cli.predict import DEFAULT_MODEL_URL, predict_image
|
| 39 |
+
from sharp.cli.render import render_gaussians as sharp_render_gaussians
|
| 40 |
+
from sharp.models import PredictorParams, create_predictor
|
| 41 |
+
from sharp.utils import camera, io
|
| 42 |
+
from sharp.utils.gaussians import Gaussians3D, SceneMetaData, save_ply
|
| 43 |
+
from sharp.utils.gsplat import GSplatRenderer
|
| 44 |
+
|
| 45 |
+
TrajectoryType = Literal["swipe", "shake", "rotate", "rotate_forward"]
|
| 46 |
+
|
| 47 |
+
# -----------------------------------------------------------------------------
|
| 48 |
+
# Helpers
|
| 49 |
+
# -----------------------------------------------------------------------------
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def _now_ms() -> int:
|
| 53 |
+
return int(time.time() * 1000)
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def _ensure_dir(path: Path) -> Path:
|
| 57 |
+
path.mkdir(parents=True, exist_ok=True)
|
| 58 |
+
return path
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def _make_even(x: int) -> int:
|
| 62 |
+
return x if x % 2 == 0 else x + 1
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def _select_device(preference: str = "auto") -> torch.device:
|
| 66 |
+
"""Select the best available device for inference (CPU/CUDA/MPS)."""
|
| 67 |
+
if preference not in {"auto", "cpu", "cuda", "mps"}:
|
| 68 |
+
raise ValueError("device preference must be one of: auto|cpu|cuda|mps")
|
| 69 |
+
|
| 70 |
+
if preference == "cpu":
|
| 71 |
+
return torch.device("cpu")
|
| 72 |
+
if preference == "cuda":
|
| 73 |
+
return torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 74 |
+
if preference == "mps":
|
| 75 |
+
return torch.device("mps" if torch.backends.mps.is_available() else "cpu")
|
| 76 |
+
|
| 77 |
+
# auto
|
| 78 |
+
if torch.cuda.is_available():
|
| 79 |
+
return torch.device("cuda")
|
| 80 |
+
if torch.backends.mps.is_available():
|
| 81 |
+
return torch.device("mps")
|
| 82 |
+
return torch.device("cpu")
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
# -----------------------------------------------------------------------------
|
| 86 |
+
# Prediction outputs
|
| 87 |
+
# -----------------------------------------------------------------------------
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
@dataclass(frozen=True, slots=True)
|
| 91 |
+
class PredictionOutputs:
|
| 92 |
+
"""Outputs of SHARP inference (plus derived metadata for rendering)."""
|
| 93 |
+
|
| 94 |
+
ply_path: Path
|
| 95 |
+
gaussians: Gaussians3D
|
| 96 |
+
metadata_for_render: SceneMetaData
|
| 97 |
+
input_resolution_hw: tuple[int, int]
|
| 98 |
+
focal_length_px: float
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
# -----------------------------------------------------------------------------
|
| 102 |
+
# Patch SHARP VideoWriter to properly close the optional depth writer
|
| 103 |
+
# -----------------------------------------------------------------------------
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
class _PatchedVideoWriter(io.VideoWriter):
|
| 107 |
+
"""Ensure depth writer is closed so files can be safely cleaned up."""
|
| 108 |
+
|
| 109 |
+
def __init__(
|
| 110 |
+
self, output_path: Path, fps: float = 30.0, render_depth: bool = True
|
| 111 |
+
) -> None:
|
| 112 |
+
super().__init__(output_path, fps=fps, render_depth=render_depth)
|
| 113 |
+
# Ensure attribute exists for downstream code paths.
|
| 114 |
+
if not hasattr(self, "depth_writer"):
|
| 115 |
+
self.depth_writer = None # type: ignore[attribute-defined-outside-init]
|
| 116 |
+
|
| 117 |
+
def close(self):
|
| 118 |
+
super().close()
|
| 119 |
+
depth_writer = getattr(self, "depth_writer", None)
|
| 120 |
+
try:
|
| 121 |
+
if depth_writer is not None:
|
| 122 |
+
depth_writer.close()
|
| 123 |
+
except Exception:
|
| 124 |
+
pass
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
@contextmanager
|
| 128 |
+
def _patched_sharp_videowriter():
|
| 129 |
+
"""Temporarily patch `sharp.utils.io.VideoWriter` used by `sharp.cli.render`."""
|
| 130 |
+
original = io.VideoWriter
|
| 131 |
+
io.VideoWriter = _PatchedVideoWriter # type: ignore[assignment]
|
| 132 |
+
try:
|
| 133 |
+
yield
|
| 134 |
+
finally:
|
| 135 |
+
io.VideoWriter = original # type: ignore[assignment]
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
# -----------------------------------------------------------------------------
|
| 139 |
+
# Model wrapper
|
| 140 |
+
# -----------------------------------------------------------------------------
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
class ModelWrapper:
|
| 144 |
+
"""Cached SHARP model wrapper for Gradio/Spaces."""
|
| 145 |
+
|
| 146 |
+
def __init__(
|
| 147 |
+
self,
|
| 148 |
+
*,
|
| 149 |
+
outputs_dir: str | Path = "outputs",
|
| 150 |
+
checkpoint_url: str = DEFAULT_MODEL_URL,
|
| 151 |
+
checkpoint_path: str | Path | None = None,
|
| 152 |
+
device_preference: str = "auto",
|
| 153 |
+
keep_model_on_device: bool | None = None,
|
| 154 |
+
hf_repo_id: str | None = None,
|
| 155 |
+
hf_filename: str | None = None,
|
| 156 |
+
hf_revision: str | None = None,
|
| 157 |
+
) -> None:
|
| 158 |
+
self.outputs_dir = _ensure_dir(Path(outputs_dir))
|
| 159 |
+
self.checkpoint_url = checkpoint_url
|
| 160 |
+
|
| 161 |
+
env_ckpt = os.getenv("SHARP_CHECKPOINT_PATH") or os.getenv("SHARP_CHECKPOINT")
|
| 162 |
+
if checkpoint_path:
|
| 163 |
+
self.checkpoint_path = Path(checkpoint_path)
|
| 164 |
+
elif env_ckpt:
|
| 165 |
+
self.checkpoint_path = Path(env_ckpt)
|
| 166 |
+
else:
|
| 167 |
+
self.checkpoint_path = None
|
| 168 |
+
|
| 169 |
+
# Optional Hugging Face Hub fallback (useful when direct CDN download fails).
|
| 170 |
+
self.hf_repo_id = hf_repo_id or os.getenv("SHARP_HF_REPO_ID", "apple/Sharp")
|
| 171 |
+
self.hf_filename = hf_filename or os.getenv(
|
| 172 |
+
"SHARP_HF_FILENAME", "sharp_2572gikvuh.pt"
|
| 173 |
+
)
|
| 174 |
+
self.hf_revision = hf_revision or os.getenv("SHARP_HF_REVISION") or None
|
| 175 |
+
|
| 176 |
+
self.device_preference = device_preference
|
| 177 |
+
|
| 178 |
+
# For ZeroGPU, it's safer to not keep large tensors on CUDA across calls.
|
| 179 |
+
if keep_model_on_device is None:
|
| 180 |
+
keep_env = (
|
| 181 |
+
os.getenv("SHARP_KEEP_MODEL_ON_DEVICE")
|
| 182 |
+
)
|
| 183 |
+
self.keep_model_on_device = keep_env == "1"
|
| 184 |
+
else:
|
| 185 |
+
self.keep_model_on_device = keep_model_on_device
|
| 186 |
+
|
| 187 |
+
self._lock = threading.RLock()
|
| 188 |
+
self._predictor: torch.nn.Module | None = None
|
| 189 |
+
self._predictor_device: torch.device | None = None
|
| 190 |
+
self._state_dict: dict | None = None
|
| 191 |
+
|
| 192 |
+
def has_cuda(self) -> bool:
|
| 193 |
+
return torch.cuda.is_available()
|
| 194 |
+
|
| 195 |
+
def _load_state_dict(self) -> dict:
|
| 196 |
+
with self._lock:
|
| 197 |
+
if self._state_dict is not None:
|
| 198 |
+
return self._state_dict
|
| 199 |
+
|
| 200 |
+
# 1) Explicit local checkpoint path
|
| 201 |
+
if self.checkpoint_path is not None:
|
| 202 |
+
try:
|
| 203 |
+
self._state_dict = torch.load(
|
| 204 |
+
self.checkpoint_path,
|
| 205 |
+
weights_only=True,
|
| 206 |
+
map_location="cpu",
|
| 207 |
+
)
|
| 208 |
+
return self._state_dict
|
| 209 |
+
except Exception as e:
|
| 210 |
+
raise RuntimeError(
|
| 211 |
+
"Failed to load SHARP checkpoint from local path.\n\n"
|
| 212 |
+
f"Path:\n {self.checkpoint_path}\n\n"
|
| 213 |
+
f"Original error:\n {type(e).__name__}: {e}"
|
| 214 |
+
) from e
|
| 215 |
+
|
| 216 |
+
# 2) HF cache (no-network): best match for Spaces `preload_from_hub`.
|
| 217 |
+
hf_cache_error: Exception | None = None
|
| 218 |
+
if try_to_load_from_cache is not None:
|
| 219 |
+
try:
|
| 220 |
+
cached = try_to_load_from_cache(
|
| 221 |
+
repo_id=self.hf_repo_id,
|
| 222 |
+
filename=self.hf_filename,
|
| 223 |
+
revision=self.hf_revision,
|
| 224 |
+
repo_type="model",
|
| 225 |
+
)
|
| 226 |
+
except TypeError:
|
| 227 |
+
cached = try_to_load_from_cache(self.hf_repo_id, self.hf_filename) # type: ignore[misc]
|
| 228 |
+
|
| 229 |
+
try:
|
| 230 |
+
if isinstance(cached, str) and Path(cached).exists():
|
| 231 |
+
self._state_dict = torch.load(
|
| 232 |
+
cached, weights_only=True, map_location="cpu"
|
| 233 |
+
)
|
| 234 |
+
return self._state_dict
|
| 235 |
+
except Exception as e:
|
| 236 |
+
hf_cache_error = e
|
| 237 |
+
|
| 238 |
+
# 3) HF Hub download (reuse cache when available; may download otherwise).
|
| 239 |
+
hf_error: Exception | None = None
|
| 240 |
+
if hf_hub_download is not None:
|
| 241 |
+
# Attempt "local only" mode if supported (avoids network).
|
| 242 |
+
try:
|
| 243 |
+
import inspect
|
| 244 |
+
|
| 245 |
+
if "local_files_only" in inspect.signature(hf_hub_download).parameters:
|
| 246 |
+
ckpt_path = hf_hub_download(
|
| 247 |
+
repo_id=self.hf_repo_id,
|
| 248 |
+
filename=self.hf_filename,
|
| 249 |
+
revision=self.hf_revision,
|
| 250 |
+
local_files_only=True,
|
| 251 |
+
)
|
| 252 |
+
if Path(ckpt_path).exists():
|
| 253 |
+
self._state_dict = torch.load(
|
| 254 |
+
ckpt_path, weights_only=True, map_location="cpu"
|
| 255 |
+
)
|
| 256 |
+
return self._state_dict
|
| 257 |
+
except Exception:
|
| 258 |
+
pass
|
| 259 |
+
|
| 260 |
+
try:
|
| 261 |
+
ckpt_path = hf_hub_download(
|
| 262 |
+
repo_id=self.hf_repo_id,
|
| 263 |
+
filename=self.hf_filename,
|
| 264 |
+
revision=self.hf_revision,
|
| 265 |
+
)
|
| 266 |
+
self._state_dict = torch.load(
|
| 267 |
+
ckpt_path,
|
| 268 |
+
weights_only=True,
|
| 269 |
+
map_location="cpu",
|
| 270 |
+
)
|
| 271 |
+
return self._state_dict
|
| 272 |
+
except Exception as e:
|
| 273 |
+
hf_error = e
|
| 274 |
+
|
| 275 |
+
# 4) Default upstream CDN (torch hub cache). Last resort.
|
| 276 |
+
url_error: Exception | None = None
|
| 277 |
+
try:
|
| 278 |
+
self._state_dict = torch.hub.load_state_dict_from_url(
|
| 279 |
+
self.checkpoint_url,
|
| 280 |
+
progress=True,
|
| 281 |
+
map_location="cpu",
|
| 282 |
+
)
|
| 283 |
+
return self._state_dict
|
| 284 |
+
except Exception as e:
|
| 285 |
+
url_error = e
|
| 286 |
+
|
| 287 |
+
# If we got here: all options failed.
|
| 288 |
+
hint_lines = [
|
| 289 |
+
"Failed to load SHARP checkpoint.",
|
| 290 |
+
"",
|
| 291 |
+
"Tried (in order):",
|
| 292 |
+
f" 1) HF cache (preload_from_hub): repo_id={self.hf_repo_id}, filename={self.hf_filename}, revision={self.hf_revision or 'None'}",
|
| 293 |
+
f" 2) HF Hub download: repo_id={self.hf_repo_id}, filename={self.hf_filename}, revision={self.hf_revision or 'None'}",
|
| 294 |
+
f" 3) URL (torch hub): {self.checkpoint_url}",
|
| 295 |
+
"",
|
| 296 |
+
"If network access is restricted, set a local checkpoint path:",
|
| 297 |
+
" - SHARP_CHECKPOINT_PATH=/path/to/sharp_2572gikvuh.pt",
|
| 298 |
+
"",
|
| 299 |
+
"Original errors:",
|
| 300 |
+
]
|
| 301 |
+
if try_to_load_from_cache is None:
|
| 302 |
+
hint_lines.append(" HF cache: huggingface_hub not installed")
|
| 303 |
+
elif hf_cache_error is not None:
|
| 304 |
+
hint_lines.append(
|
| 305 |
+
f" HF cache: {type(hf_cache_error).__name__}: {hf_cache_error}"
|
| 306 |
+
)
|
| 307 |
+
else:
|
| 308 |
+
hint_lines.append(" HF cache: (not found in cache)")
|
| 309 |
+
|
| 310 |
+
if hf_hub_download is None:
|
| 311 |
+
hint_lines.append(" HF download: huggingface_hub not installed")
|
| 312 |
+
else:
|
| 313 |
+
hint_lines.append(f" HF download: {type(hf_error).__name__}: {hf_error}")
|
| 314 |
+
|
| 315 |
+
hint_lines.append(f" URL: {type(url_error).__name__}: {url_error}")
|
| 316 |
+
|
| 317 |
+
raise RuntimeError("\n".join(hint_lines))
|
| 318 |
+
|
| 319 |
+
def _get_predictor(self, device: torch.device) -> torch.nn.Module:
|
| 320 |
+
with self._lock:
|
| 321 |
+
if self._predictor is None:
|
| 322 |
+
state_dict = self._load_state_dict()
|
| 323 |
+
predictor = create_predictor(PredictorParams())
|
| 324 |
+
predictor.load_state_dict(state_dict)
|
| 325 |
+
predictor.eval()
|
| 326 |
+
self._predictor = predictor
|
| 327 |
+
self._predictor_device = torch.device("cpu")
|
| 328 |
+
|
| 329 |
+
assert self._predictor is not None
|
| 330 |
+
assert self._predictor_device is not None
|
| 331 |
+
|
| 332 |
+
if self._predictor_device != device:
|
| 333 |
+
self._predictor.to(device)
|
| 334 |
+
self._predictor_device = device
|
| 335 |
+
|
| 336 |
+
return self._predictor
|
| 337 |
+
|
| 338 |
+
def _maybe_move_model_back_to_cpu(self) -> None:
|
| 339 |
+
if self.keep_model_on_device:
|
| 340 |
+
return
|
| 341 |
+
with self._lock:
|
| 342 |
+
if self._predictor is not None and self._predictor_device is not None:
|
| 343 |
+
if self._predictor_device.type != "cpu":
|
| 344 |
+
self._predictor.to("cpu")
|
| 345 |
+
self._predictor_device = torch.device("cpu")
|
| 346 |
+
if torch.cuda.is_available():
|
| 347 |
+
torch.cuda.empty_cache()
|
| 348 |
+
|
| 349 |
+
def _make_output_stem(self, input_path: Path) -> str:
|
| 350 |
+
return f"{input_path.stem}-{_now_ms()}-{uuid.uuid4().hex[:8]}"
|
| 351 |
+
|
| 352 |
+
def predict_to_ply(self, image_path: str | Path) -> PredictionOutputs:
|
| 353 |
+
"""Run SHARP inference and export a .ply file."""
|
| 354 |
+
image_path = Path(image_path)
|
| 355 |
+
if not image_path.exists():
|
| 356 |
+
raise FileNotFoundError(f"Image does not exist: {image_path}")
|
| 357 |
+
|
| 358 |
+
device = _select_device(self.device_preference)
|
| 359 |
+
predictor = self._get_predictor(device)
|
| 360 |
+
|
| 361 |
+
image_np, _, f_px = io.load_rgb(image_path)
|
| 362 |
+
height, width = image_np.shape[:2]
|
| 363 |
+
|
| 364 |
+
with torch.no_grad():
|
| 365 |
+
gaussians = predict_image(predictor, image_np, f_px, device)
|
| 366 |
+
|
| 367 |
+
stem = self._make_output_stem(image_path)
|
| 368 |
+
ply_path = self.outputs_dir / f"{stem}.ply"
|
| 369 |
+
|
| 370 |
+
# save_ply expects (height, width).
|
| 371 |
+
save_ply(gaussians, f_px, (height, width), ply_path)
|
| 372 |
+
|
| 373 |
+
# SceneMetaData expects (width, height) for resolution.
|
| 374 |
+
metadata_for_render = SceneMetaData(
|
| 375 |
+
focal_length_px=float(f_px),
|
| 376 |
+
resolution_px=(int(width), int(height)),
|
| 377 |
+
color_space="linearRGB",
|
| 378 |
+
)
|
| 379 |
+
|
| 380 |
+
self._maybe_move_model_back_to_cpu()
|
| 381 |
+
|
| 382 |
+
return PredictionOutputs(
|
| 383 |
+
ply_path=ply_path,
|
| 384 |
+
gaussians=gaussians,
|
| 385 |
+
metadata_for_render=metadata_for_render,
|
| 386 |
+
input_resolution_hw=(int(height), int(width)),
|
| 387 |
+
focal_length_px=float(f_px),
|
| 388 |
+
)
|
| 389 |
+
|
| 390 |
+
def _render_video_impl(
|
| 391 |
+
self,
|
| 392 |
+
*,
|
| 393 |
+
gaussians: Gaussians3D,
|
| 394 |
+
metadata: SceneMetaData,
|
| 395 |
+
output_path: Path,
|
| 396 |
+
trajectory_type: TrajectoryType,
|
| 397 |
+
num_frames: int,
|
| 398 |
+
fps: int,
|
| 399 |
+
output_long_side: int | None,
|
| 400 |
+
) -> Path:
|
| 401 |
+
if not torch.cuda.is_available():
|
| 402 |
+
raise RuntimeError("Rendering requires CUDA (gsplat).")
|
| 403 |
+
|
| 404 |
+
if num_frames < 2:
|
| 405 |
+
raise ValueError("num_frames must be >= 2")
|
| 406 |
+
if fps < 1:
|
| 407 |
+
raise ValueError("fps must be >= 1")
|
| 408 |
+
|
| 409 |
+
# Keep aligned with upstream CLI pipeline where possible.
|
| 410 |
+
if output_long_side is None and int(fps) == 30:
|
| 411 |
+
params = camera.TrajectoryParams(
|
| 412 |
+
type=trajectory_type,
|
| 413 |
+
num_steps=int(num_frames),
|
| 414 |
+
num_repeats=1,
|
| 415 |
+
)
|
| 416 |
+
with _patched_sharp_videowriter():
|
| 417 |
+
sharp_render_gaussians(
|
| 418 |
+
gaussians=gaussians,
|
| 419 |
+
metadata=metadata,
|
| 420 |
+
params=params,
|
| 421 |
+
output_path=output_path,
|
| 422 |
+
)
|
| 423 |
+
depth_path = output_path.with_suffix(".depth.mp4")
|
| 424 |
+
try:
|
| 425 |
+
if depth_path.exists():
|
| 426 |
+
depth_path.unlink()
|
| 427 |
+
except Exception:
|
| 428 |
+
pass
|
| 429 |
+
return output_path
|
| 430 |
+
|
| 431 |
+
# Adapted pipeline for custom output resolution / FPS.
|
| 432 |
+
src_w, src_h = metadata.resolution_px
|
| 433 |
+
src_f = float(metadata.focal_length_px)
|
| 434 |
+
|
| 435 |
+
if output_long_side is None:
|
| 436 |
+
out_w, out_h, out_f = src_w, src_h, src_f
|
| 437 |
+
else:
|
| 438 |
+
long_side = max(src_w, src_h)
|
| 439 |
+
scale = float(output_long_side) / float(long_side)
|
| 440 |
+
out_w = _make_even(max(2, int(round(src_w * scale))))
|
| 441 |
+
out_h = _make_even(max(2, int(round(src_h * scale))))
|
| 442 |
+
out_f = src_f * scale
|
| 443 |
+
|
| 444 |
+
traj_params = camera.TrajectoryParams(
|
| 445 |
+
type=trajectory_type,
|
| 446 |
+
num_steps=int(num_frames),
|
| 447 |
+
num_repeats=1,
|
| 448 |
+
)
|
| 449 |
+
|
| 450 |
+
device = torch.device("cuda")
|
| 451 |
+
gaussians_cuda = gaussians.to(device)
|
| 452 |
+
|
| 453 |
+
intrinsics = torch.tensor(
|
| 454 |
+
[
|
| 455 |
+
[out_f, 0.0, (out_w - 1) / 2.0, 0.0],
|
| 456 |
+
[0.0, out_f, (out_h - 1) / 2.0, 0.0],
|
| 457 |
+
[0.0, 0.0, 1.0, 0.0],
|
| 458 |
+
[0.0, 0.0, 0.0, 1.0],
|
| 459 |
+
],
|
| 460 |
+
device=device,
|
| 461 |
+
dtype=torch.float32,
|
| 462 |
+
)
|
| 463 |
+
|
| 464 |
+
cam_model = camera.create_camera_model(
|
| 465 |
+
gaussians_cuda,
|
| 466 |
+
intrinsics,
|
| 467 |
+
resolution_px=(out_w, out_h),
|
| 468 |
+
lookat_mode=traj_params.lookat_mode,
|
| 469 |
+
)
|
| 470 |
+
|
| 471 |
+
trajectory = camera.create_eye_trajectory(
|
| 472 |
+
gaussians_cuda,
|
| 473 |
+
traj_params,
|
| 474 |
+
resolution_px=(out_w, out_h),
|
| 475 |
+
f_px=out_f,
|
| 476 |
+
)
|
| 477 |
+
|
| 478 |
+
renderer = GSplatRenderer(color_space=metadata.color_space)
|
| 479 |
+
|
| 480 |
+
# IMPORTANT: Keep render_depth=True (avoids upstream AttributeError).
|
| 481 |
+
video_writer = _PatchedVideoWriter(output_path, fps=float(fps), render_depth=True)
|
| 482 |
+
|
| 483 |
+
for eye_position in trajectory:
|
| 484 |
+
cam_info = cam_model.compute(eye_position)
|
| 485 |
+
rendering = renderer(
|
| 486 |
+
gaussians_cuda,
|
| 487 |
+
extrinsics=cam_info.extrinsics[None].to(device),
|
| 488 |
+
intrinsics=cam_info.intrinsics[None].to(device),
|
| 489 |
+
image_width=cam_info.width,
|
| 490 |
+
image_height=cam_info.height,
|
| 491 |
+
)
|
| 492 |
+
color = (rendering.color[0].permute(1, 2, 0) * 255.0).to(dtype=torch.uint8)
|
| 493 |
+
depth = rendering.depth[0]
|
| 494 |
+
video_writer.add_frame(color, depth)
|
| 495 |
+
|
| 496 |
+
video_writer.close()
|
| 497 |
+
|
| 498 |
+
depth_path = output_path.with_suffix(".depth.mp4")
|
| 499 |
+
try:
|
| 500 |
+
if depth_path.exists():
|
| 501 |
+
depth_path.unlink()
|
| 502 |
+
except Exception:
|
| 503 |
+
pass
|
| 504 |
+
|
| 505 |
+
return output_path
|
| 506 |
+
|
| 507 |
+
def render_video(
|
| 508 |
+
self,
|
| 509 |
+
*,
|
| 510 |
+
gaussians: Gaussians3D,
|
| 511 |
+
metadata: SceneMetaData,
|
| 512 |
+
output_stem: str,
|
| 513 |
+
trajectory_type: TrajectoryType = "rotate_forward",
|
| 514 |
+
num_frames: int = 60,
|
| 515 |
+
fps: int = 30,
|
| 516 |
+
output_long_side: int | None = None,
|
| 517 |
+
) -> Path:
|
| 518 |
+
"""Render a camera trajectory as an MP4 (CUDA-only)."""
|
| 519 |
+
output_path = self.outputs_dir / f"{output_stem}.mp4"
|
| 520 |
+
return self._render_video_impl(
|
| 521 |
+
gaussians=gaussians,
|
| 522 |
+
metadata=metadata,
|
| 523 |
+
output_path=output_path,
|
| 524 |
+
trajectory_type=trajectory_type,
|
| 525 |
+
num_frames=num_frames,
|
| 526 |
+
fps=fps,
|
| 527 |
+
output_long_side=output_long_side,
|
| 528 |
+
)
|
| 529 |
+
|
| 530 |
+
def predict_and_maybe_render(
|
| 531 |
+
self,
|
| 532 |
+
image_path: str | Path,
|
| 533 |
+
*,
|
| 534 |
+
trajectory_type: TrajectoryType,
|
| 535 |
+
num_frames: int,
|
| 536 |
+
fps: int,
|
| 537 |
+
output_long_side: int | None,
|
| 538 |
+
render_video: bool = True,
|
| 539 |
+
) -> tuple[Path | None, Path]:
|
| 540 |
+
"""One-shot helper for the UI: returns (video_path, ply_path)."""
|
| 541 |
+
pred = self.predict_to_ply(image_path)
|
| 542 |
+
|
| 543 |
+
if not render_video:
|
| 544 |
+
return None, pred.ply_path
|
| 545 |
+
|
| 546 |
+
if not torch.cuda.is_available():
|
| 547 |
+
return None, pred.ply_path
|
| 548 |
+
|
| 549 |
+
output_stem = pred.ply_path.with_suffix("").name
|
| 550 |
+
video_path = self.render_video(
|
| 551 |
+
gaussians=pred.gaussians,
|
| 552 |
+
metadata=pred.metadata_for_render,
|
| 553 |
+
output_stem=output_stem,
|
| 554 |
+
trajectory_type=trajectory_type,
|
| 555 |
+
num_frames=num_frames,
|
| 556 |
+
fps=fps,
|
| 557 |
+
output_long_side=output_long_side,
|
| 558 |
+
)
|
| 559 |
+
return video_path, pred.ply_path
|
| 560 |
+
|
| 561 |
+
|
| 562 |
+
# -----------------------------------------------------------------------------
|
| 563 |
+
# ZeroGPU entrypoints
|
| 564 |
+
# -----------------------------------------------------------------------------
|
| 565 |
+
#
|
| 566 |
+
# IMPORTANT: Do NOT decorate bound instance methods with `@spaces.GPU` on ZeroGPU.
|
| 567 |
+
# The wrapper uses multiprocessing queues and pickles args/kwargs. If `self` is
|
| 568 |
+
# included, Python will try to pickle the whole instance. ModelWrapper contains
|
| 569 |
+
# a threading.RLock (not pickleable) and the model itself should not be pickled.
|
| 570 |
+
#
|
| 571 |
+
# Expose module-level functions that accept only pickleable arguments and
|
| 572 |
+
# create/cache the ModelWrapper inside the GPU worker process.
|
| 573 |
+
|
| 574 |
+
DEFAULT_OUTPUTS_DIR: Final[Path] = _ensure_dir(Path(__file__).resolve().parent / "outputs")
|
| 575 |
+
|
| 576 |
+
_GLOBAL_MODEL: ModelWrapper | None = None
|
| 577 |
+
_GLOBAL_MODEL_INIT_LOCK: Final[threading.Lock] = threading.Lock()
|
| 578 |
+
|
| 579 |
+
|
| 580 |
+
def get_global_model(*, outputs_dir: str | Path = DEFAULT_OUTPUTS_DIR) -> ModelWrapper:
|
| 581 |
+
global _GLOBAL_MODEL
|
| 582 |
+
with _GLOBAL_MODEL_INIT_LOCK:
|
| 583 |
+
if _GLOBAL_MODEL is None:
|
| 584 |
+
_GLOBAL_MODEL = ModelWrapper(outputs_dir=outputs_dir)
|
| 585 |
+
return _GLOBAL_MODEL
|
| 586 |
+
|
| 587 |
+
|
| 588 |
+
def predict_and_maybe_render(
|
| 589 |
+
image_path: str | Path,
|
| 590 |
+
*,
|
| 591 |
+
trajectory_type: TrajectoryType,
|
| 592 |
+
num_frames: int,
|
| 593 |
+
fps: int,
|
| 594 |
+
output_long_side: int | None,
|
| 595 |
+
render_video: bool = True,
|
| 596 |
+
) -> tuple[Path | None, Path]:
|
| 597 |
+
model = get_global_model()
|
| 598 |
+
return model.predict_and_maybe_render(
|
| 599 |
+
image_path,
|
| 600 |
+
trajectory_type=trajectory_type,
|
| 601 |
+
num_frames=num_frames,
|
| 602 |
+
fps=fps,
|
| 603 |
+
output_long_side=output_long_side,
|
| 604 |
+
render_video=render_video,
|
| 605 |
+
)
|
| 606 |
+
|
| 607 |
+
|
| 608 |
+
# Export the GPU-wrapped callable (or a no-op wrapper locally).
|
| 609 |
+
if spaces is not None:
|
| 610 |
+
predict_and_maybe_render_gpu = spaces.GPU(duration=180)(predict_and_maybe_render)
|
| 611 |
+
else: # pragma: no cover
|
| 612 |
+
predict_and_maybe_render_gpu = predict_and_maybe_render
|
pyproject.toml
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[project]
|
| 2 |
+
name = "ml-sharp"
|
| 3 |
+
version = "1.0.0"
|
| 4 |
+
description = "Sharp Monocular View Synthesis in Less Than a Second"
|
| 5 |
+
readme = "README.md"
|
| 6 |
+
requires-python = ">=3.13"
|
| 7 |
+
dependencies = [
|
| 8 |
+
"gradio==6.1.0",
|
| 9 |
+
"huggingface-hub>=1.2.3",
|
| 10 |
+
"sharp",
|
| 11 |
+
"spaces==0.44.0",
|
| 12 |
+
"torch>=2.9.1",
|
| 13 |
+
"torchvision>=0.24.1",
|
| 14 |
+
]
|
| 15 |
+
|
| 16 |
+
[tool.uv.sources]
|
| 17 |
+
sharp = { git = "https://github.com/apple/ml-sharp.git", rev = "cdb4ddc6796402bee5487c7312260f2edd8bd5f0" }
|
| 18 |
+
|
| 19 |
+
[dependency-groups]
|
| 20 |
+
dev = [
|
| 21 |
+
"hf>=1.2.3",
|
| 22 |
+
"ruff>=0.14.9",
|
| 23 |
+
]
|
requirements.txt
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
gradio==6.1.0
|
| 2 |
+
spaces==0.44.0
|
| 3 |
+
huggingface_hub>=1.2.3
|
| 4 |
+
torch
|
| 5 |
+
torchvision
|
| 6 |
+
sharp @ git+https://github.com/apple/ml-sharp.git@cdb4ddc6796402bee5487c7312260f2edd8bd5f0
|
uv.lock
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|