Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+examples/bathroom_shelf.png filter=lfs diff=lfs merge=lfs -text
+examples/kitchen_counter.png filter=lfs diff=lfs merge=lfs -text
+examples/robot_table.png filter=lfs diff=lfs merge=lfs -text
+examples/shelf_mugs.png filter=lfs diff=lfs merge=lfs -text
+examples/so101_pickplace.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,13 +1,36 @@
 ---
-title: Physbrain1 5 8b Demo
-emoji: 🚀
-colorFrom: pink
-colorTo: red
+title: PhysBrain 1.5-8B
+emoji: 🦾
+colorFrom: indigo
+colorTo: blue
 sdk: gradio
 sdk_version: 6.26.0
-python_version: '3.12'
 app_file: app.py
-pinned: false
+short_description: Embodied spatial reasoning, pointing and arm traces
+python_version: "3.12"
+startup_duration_timeout: 1h
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# PhysBrain 1.5-8B
+
+Demo for [DeepCybo/PhysBrain1.5-8B](https://huggingface.co/DeepCybo/PhysBrain1.5-8B),
+an embodied vision-language model built on Qwen3-VL-8B-Instruct.
+
+Three modes:
+
+- **Spatial Q&A** — open-ended embodied questions about a scene.
+- **Point / affordance** — referring expressions and free-space queries; points are
+  overlaid on the image.
+- **Manipulation trace** — the 2D trajectory a robot arm should follow to complete a task.
+
+Point / trajectory prompting follows the authors' evaluation harness
+([EmbodiedEvalKit](https://github.com/pickxiguapi/EmbodiedEvalKit), MIT): the model is asked for
+`[{"point_2d": [x, y]}]` and coordinates are decoded from a 0–1000 normalised canvas.
+
+The checkpoint also predicts action-chunk tokens and future-state VQ tokens; decoding
+those requires the action codebook / VQ image decoder, which are not published, so this
+demo exposes the embodied-understanding capabilities.
+
+Example scenes come from [BAAI/RefSpatial-Bench](https://huggingface.co/datasets/BAAI/RefSpatial-Bench)
+(Apache-2.0) and [lerobot/svla_so101_pickplace](https://huggingface.co/datasets/lerobot/svla_so101_pickplace)
+(Apache-2.0).

app.py

@@ -0,0 +1,343 @@
+import ast
+import json
+import os
+import re
+import time
+
+os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
+
+import spaces  # noqa: E402  (must precede torch)
+import torch  # noqa: E402
+import gradio as gr  # noqa: E402
+from PIL import Image, ImageDraw  # noqa: E402
+from transformers import AutoModelForImageTextToText, AutoProcessor  # noqa: E402
+
+MODEL_ID = "DeepCybo/PhysBrain1.5-8B"
+
+# Prompt suffix used by the authors' evaluation harness (EmbodiedEvalKit) for
+# Qwen3-VL-backboned models. Coordinates come back normalised to 0-1000.
+POINT_SUFFIX = 'The answer should be presented in JSON format as follows: [{"point_2d": [x, y]}].'
+
+MODE_ASK = "Spatial Q&A"
+MODE_POINT = "Point / affordance"
+MODE_TRACE = "Manipulation trace"
+MODES = [MODE_ASK, MODE_POINT, MODE_TRACE]
+
+# Cap the number of visual tokens so a 4K upload doesn't explode the context.
+MAX_PIXELS = 1024 * 1024
+
+processor = AutoProcessor.from_pretrained(MODEL_ID)
+model = AutoModelForImageTextToText.from_pretrained(
+    MODEL_ID,
+    dtype=torch.bfloat16,
+    attn_implementation="sdpa",
+).to("cuda")
+model.eval()
+
+
+# --------------------------------------------------------------------------- #
+# point parsing / drawing
+# --------------------------------------------------------------------------- #
+def _walk(data):
+    """Recursively pull [x, y] pairs out of a parsed JSON/py structure."""
+    pts = []
+    if isinstance(data, dict):
+        for key in ("point_2d", "points", "point", "coordinates", "position"):
+            if key in data:
+                return _walk(data[key])
+        return []
+    if isinstance(data, (list, tuple)):
+        if len(data) == 2 and all(isinstance(v, (int, float)) for v in data):
+            return [[float(data[0]), float(data[1])]]
+        for item in data:
+            pts.extend(_walk(item))
+    return pts
+
+
+def parse_points(text: str):
+    """Extract 2D points from a PhysBrain response.
+
+    Handles ``[{"point_2d": [x, y]}, ...]``, bare ``[[x, y], ...]``, markdown
+    code fences and points embedded in prose.
+    """
+    if not text or not text.strip():
+        return []
+
+    cleaned = re.sub(r"```(?:json|python)?\n?(.*?)\n?```", r"\1", text, flags=re.DOTALL)
+
+    # Try every bracketed blob, longest first (the full list beats a single pair).
+    blobs = re.findall(r"\[.*?\]\s*(?=$|[,\n]|\Z)", cleaned, flags=re.DOTALL)
+    blobs.sort(key=len, reverse=True)
+    for blob in blobs[:5]:
+        for loader in (json.loads, ast.literal_eval):
+            try:
+                pts = _walk(loader(blob))
+            except Exception:
+                continue
+            if pts:
+                return pts
+
+    # Last resort: raw "(x, y)" / "[x, y]" pairs anywhere in the text.
+    pairs = re.findall(r"[\[\(]\s*(-?\d+\.?\d*)\s*,\s*(-?\d+\.?\d*)\s*[\]\)]", cleaned)
+    return [[float(a), float(b)] for a, b in pairs]
+
+
+def to_pixels(points, width: int, height: int):
+    """PhysBrain emits coordinates normalised to a 0-1000 canvas."""
+    out = []
+    for pt in points:
+        x = float(pt[0]) / 1000.0 * width
+        y = float(pt[1]) / 1000.0 * height
+        out.append((max(0.0, min(width - 1.0, x)), max(0.0, min(height - 1.0, y))))
+    return out
+
+
+def _radius(image: Image.Image) -> int:
+    return max(5, int(0.011 * max(image.size)))
+
+
+def draw_points(image: Image.Image, pixels):
+    canvas = image.convert("RGB").copy()
+    draw = ImageDraw.Draw(canvas, "RGBA")
+    r = _radius(canvas)
+    for idx, (x, y) in enumerate(pixels):
+        draw.ellipse([x - r * 2.2, y - r * 2.2, x + r * 2.2, y + r * 2.2],
+                     fill=(255, 87, 34, 60))
+        draw.ellipse([x - r, y - r, x + r, y + r],
+                     fill=(255, 87, 34, 255), outline=(255, 255, 255, 255),
+                     width=max(2, r // 3))
+        if len(pixels) > 1:
+            draw.text((x + r * 1.6, y - r * 1.6), str(idx + 1),
+                      fill=(255, 255, 255, 255), stroke_width=3,
+                      stroke_fill=(0, 0, 0, 200))
+    return canvas
+
+
+def draw_trace(image: Image.Image, pixels):
+    canvas = image.convert("RGB").copy()
+    draw = ImageDraw.Draw(canvas, "RGBA")
+    r = _radius(canvas)
+    n = len(pixels)
+    if n > 1:
+        draw.line(pixels, fill=(0, 0, 0, 130), width=max(5, r), joint="curve")
+        for i in range(n - 1):
+            t = i / max(1, n - 2)
+            colour = (int(60 + 195 * t), int(220 - 130 * t), int(255 - 200 * t), 255)
+            draw.line([pixels[i], pixels[i + 1]], fill=colour,
+                      width=max(3, r - 2), joint="curve")
+    for idx, (x, y) in enumerate(pixels):
+        t = idx / max(1, n - 1)
+        colour = (int(60 + 195 * t), int(220 - 130 * t), int(255 - 200 * t), 255)
+        rr = r * (1.4 if idx in (0, n - 1) else 0.9)
+        draw.ellipse([x - rr, y - rr, x + rr, y + rr], fill=colour,
+                     outline=(255, 255, 255, 255), width=2)
+    return canvas
+
+
+# --------------------------------------------------------------------------- #
+# prompt construction
+# --------------------------------------------------------------------------- #
+def build_prompt(instruction: str, mode: str, num_points: int) -> str:
+    instruction = (instruction or "").strip()
+    if mode == MODE_POINT:
+        return f"{instruction}\n{POINT_SUFFIX}"
+    if mode == MODE_TRACE:
+        return (
+            "You are currently a robot performing robotic manipulation tasks. "
+            f"The task instruction is: {instruction}. Use 2D points to trace the "
+            "movement trajectory of the robotic arm (not the manipulated object) "
+            "as it moves to complete the task. You must provide the points in the "
+            "order of the trajectory, and the number of points must be "
+            f"{int(num_points)}.\n{POINT_SUFFIX}"
+        )
+    return instruction
+
+
+def prep_image(image: Image.Image) -> Image.Image:
+    image = image.convert("RGB")
+    w, h = image.size
+    if w * h > MAX_PIXELS:
+        scale = (MAX_PIXELS / (w * h)) ** 0.5
+        image = image.resize((max(32, int(w * scale)), max(32, int(h * scale))),
+                             Image.LANCZOS)
+    return image
+
+
+# --------------------------------------------------------------------------- #
+# inference
+# --------------------------------------------------------------------------- #
+@spaces.GPU(duration=90)
+def run(
+    image: Image.Image,
+    instruction: str,
+    mode: str = MODE_POINT,
+    num_points: int = 8,
+    max_new_tokens: int = 512,
+    temperature: float = 0.0,
+    progress=gr.Progress(track_tqdm=True),
+):
+    """Run PhysBrain 1.5-8B on an image plus an embodied instruction.
+
+    Args:
+        image: the scene the robot observes.
+        instruction: question, referring expression, or manipulation task.
+        mode: "Spatial Q&A", "Point / affordance", or "Manipulation trace".
+        num_points: number of waypoints requested in "Manipulation trace" mode.
+        max_new_tokens: generation length cap.
+        temperature: 0 for greedy decoding (what the authors evaluate with).
+
+    Returns:
+        The input image with any predicted points/trajectory overlaid, and the
+        raw model response.
+    """
+    if image is None:
+        raise gr.Error("Please provide an image.")
+    if not (instruction or "").strip():
+        raise gr.Error("Please provide an instruction or question.")
+
+    mode = mode if mode in MODES else MODE_POINT
+    original = image.convert("RGB")
+    vision_image = prep_image(original)
+    prompt = build_prompt(instruction, mode, num_points)
+
+    messages = [{
+        "role": "user",
+        "content": [{"type": "image", "image": vision_image},
+                    {"type": "text", "text": prompt}],
+    }]
+
+    inputs = processor.apply_chat_template(
+        messages,
+        tokenize=True,
+        add_generation_prompt=True,
+        return_dict=True,
+        return_tensors="pt",
+    ).to(model.device)
+
+    gen_kwargs = dict(max_new_tokens=int(max_new_tokens), do_sample=temperature > 0)
+    if temperature > 0:
+        gen_kwargs.update(temperature=float(temperature), top_p=0.9)
+
+    t0 = time.perf_counter()
+    with torch.inference_mode():
+        generated = model.generate(**inputs, **gen_kwargs)
+    trimmed = generated[:, inputs["input_ids"].shape[1]:]
+    answer = processor.batch_decode(trimmed, skip_special_tokens=True)[0].strip()
+    elapsed = time.perf_counter() - t0
+    print(f"[physbrain] mode={mode} vision={vision_image.size} "
+          f"grid={inputs.get('image_grid_thw')} {elapsed:.1f}s", flush=True)
+
+    overlay = original
+    points = parse_points(answer) if mode in (MODE_POINT, MODE_TRACE) else []
+    if points:
+        pixels = to_pixels(points, *original.size)
+        overlay = draw_trace(original, pixels) if mode == MODE_TRACE else draw_points(original, pixels)
+
+    return overlay, answer
+
+
+# --------------------------------------------------------------------------- #
+# UI
+# --------------------------------------------------------------------------- #
+CSS = """
+#col-container { max-width: 1200px; margin: 0 auto; }
+.dark .gradio-container { color: var(--body-text-color); }
+"""
+
+EXAMPLES = [
+    [
+        "examples/shelf_mugs.png",
+        "Please point out the orange box.",
+        MODE_POINT,
+    ],
+    [
+        "examples/kitchen_counter.png",
+        "Please point out the third object from left to right on the closest platform.",
+        MODE_POINT,
+    ],
+    [
+        "examples/bathroom_shelf.png",
+        "Please point out the free space on the second shelf of the wooden shelf.",
+        MODE_POINT,
+    ],
+    [
+        "examples/so101_pickplace.png",
+        "put the pink lego brick into the transparent box",
+        MODE_TRACE,
+    ],
+    [
+        "examples/robot_table.png",
+        "The robot arm has to pick up the yellow mustard bottle. Where is it "
+        "relative to the bowl, and what should the arm do first?",
+        MODE_ASK,
+    ],
+]
+
+with gr.Blocks(theme=gr.themes.Citrus(), css=CSS, title="PhysBrain 1.5-8B") as demo:
+    with gr.Column(elem_id="col-container"):
+        gr.Markdown(
+            """
+# PhysBrain 1.5-8B — embodied spatial reasoning
+
+[PhysBrain 1.5-8B](https://huggingface.co/DeepCybo/PhysBrain1.5-8B) is DeepCybo's
+embodied vision-language model (a Qwen3-VL-8B finetune). Give it a scene and ask a
+spatial question, ask it to **point** at a target or a free space to place something,
+or ask it to **trace** the trajectory a robot arm should follow.
+"""
+        )
+
+        with gr.Row():
+            with gr.Column(scale=1):
+                image_in = gr.Image(label="Scene", type="pil", height=340)
+                mode_in = gr.Radio(
+                    MODES, value=MODE_POINT, label="Task",
+                    info="Point / trace answers are overlaid on the image.",
+                )
+                instruction_in = gr.Textbox(
+                    label="Instruction",
+                    placeholder="e.g. Please point out the free space to the left of the mug.",
+                    lines=3,
+                )
+                run_btn = gr.Button("Run", variant="primary")
+            with gr.Column(scale=1):
+                image_out = gr.Image(label="Prediction", type="pil", height=340)
+                text_out = gr.Textbox(label="Model response", lines=8, show_copy_button=True)
+
+        with gr.Accordion("Advanced settings", open=False):
+            num_points_in = gr.Slider(2, 32, value=8, step=1,
+                                      label="Trajectory waypoints (trace mode)")
+            max_tokens_in = gr.Slider(64, 2048, value=512, step=64,
+                                      label="Max new tokens")
+            temperature_in = gr.Slider(0.0, 1.0, value=0.0, step=0.05,
+                                       label="Temperature (0 = greedy)")
+
+        gr.Examples(
+            examples=EXAMPLES,
+            inputs=[image_in, instruction_in, mode_in],
+            outputs=[image_out, text_out],
+            fn=run,
+            cache_examples=True,
+            cache_mode="lazy",
+            label="Examples",
+        )
+
+        gr.Markdown(
+            """
+Point coordinates follow the authors' evaluation format
+(`[{"point_2d": [x, y]}]`, normalised to a 0–1000 canvas). The model also has
+action-chunk and future-state token heads; those need the unreleased action /
+VQ detokenizers, so this demo covers the embodied-understanding side.
+
+Example scenes: [BAAI/RefSpatial-Bench](https://huggingface.co/datasets/BAAI/RefSpatial-Bench)
+and [lerobot/svla_so101_pickplace](https://huggingface.co/datasets/lerobot/svla_so101_pickplace) (both Apache-2.0).
+"""
+        )
+
+    run_btn.click(
+        run,
+        inputs=[image_in, instruction_in, mode_in, num_points_in, max_tokens_in, temperature_in],
+        outputs=[image_out, text_out],
+        api_name="predict",
+    )
+
+demo.queue().launch(mcp_server=True)

requirements.txt

@@ -0,0 +1,5 @@
+transformers>=5.12.1
+accelerate
+torchvision
+pillow
+numpy