Spaces:
Sleeping
Sleeping
Upload 2 files
Browse filessome small fixes
- app.py +63 -53
- inference.py +8 -2
app.py
CHANGED
|
@@ -79,6 +79,7 @@ examples = [
|
|
| 79 |
"[Brightness Perception](https://doi.org/10.1016/j.visres.2000.200.1)",
|
| 80 |
"[Edge Effects](https://doi.org/10.1016/j.tics.2003.08.003)"
|
| 81 |
],
|
|
|
|
| 82 |
"method": "Prior-Guided Drift Diffusion",
|
| 83 |
"reverse_diff": {
|
| 84 |
"model": "resnet50_robust",
|
|
@@ -101,12 +102,12 @@ examples = [
|
|
| 101 |
"method": "Prior-Guided Drift Diffusion",
|
| 102 |
"reverse_diff": {
|
| 103 |
"model": "resnet50_robust",
|
| 104 |
-
"layer": "
|
| 105 |
-
"initial_noise": 0.
|
| 106 |
-
"diffusion_noise": 0.
|
| 107 |
-
"step_size": 0.
|
| 108 |
-
"iterations":
|
| 109 |
-
"epsilon":
|
| 110 |
}
|
| 111 |
},
|
| 112 |
{
|
|
@@ -243,7 +244,8 @@ def apply_example(example):
|
|
| 243 |
example["reverse_diff"]["initial_noise"], # Initial noise
|
| 244 |
example["reverse_diff"]["diffusion_noise"], # Diffusion noise value (corrected)
|
| 245 |
example["reverse_diff"]["step_size"], # Step size (added)
|
| 246 |
-
example["reverse_diff"]["layer"] # Model layer
|
|
|
|
| 247 |
]
|
| 248 |
|
| 249 |
# Define the interface
|
|
@@ -273,39 +275,45 @@ with gr.Blocks(title="Generative Inference Demo", css="""
|
|
| 273 |
# Inputs
|
| 274 |
image_input = gr.Image(label="Input Image", type="pil", value=os.path.join("stimuli", "Neon_Color_Circle.jpg"))
|
| 275 |
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
choices=["resnet50_robust", "standard_resnet50"],
|
| 279 |
-
value="resnet50_robust",
|
| 280 |
-
label="Model"
|
| 281 |
-
)
|
| 282 |
-
|
| 283 |
-
inference_type = gr.Dropdown(
|
| 284 |
-
choices=["Prior-Guided Drift Diffusion", "IncreaseConfidence"],
|
| 285 |
-
value="Prior-Guided Drift Diffusion",
|
| 286 |
-
label="Inference Method"
|
| 287 |
-
)
|
| 288 |
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
iterations_slider = gr.Slider(minimum=1, maximum=600, value=101, step=1, label="Number of Iterations") # Updated max to 600
|
| 292 |
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
|
| 297 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 298 |
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
value=
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 309 |
|
| 310 |
with gr.Column(scale=2):
|
| 311 |
# Outputs
|
|
@@ -332,7 +340,7 @@ with gr.Blocks(title="Generative Inference Demo", css="""
|
|
| 332 |
image_input, model_choice, inference_type,
|
| 333 |
eps_slider, iterations_slider,
|
| 334 |
initial_noise_slider, diffusion_noise_slider,
|
| 335 |
-
step_size_slider, layer_choice
|
| 336 |
]
|
| 337 |
)
|
| 338 |
|
|
@@ -341,17 +349,10 @@ with gr.Blocks(title="Generative Inference Demo", css="""
|
|
| 341 |
gr.Markdown(f"### {ex['name']}")
|
| 342 |
gr.Markdown(f"[Read more on Wikipedia]({ex['wiki']})")
|
| 343 |
|
| 344 |
-
|
| 345 |
-
|
| 346 |
-
|
| 347 |
-
|
| 348 |
-
- **Initial Noise**: {ex['reverse_diff']['initial_noise']}
|
| 349 |
-
- **Diffusion Noise**: {ex['reverse_diff']['diffusion_noise']}
|
| 350 |
-
- **Step Size**: {ex['reverse_diff']['step_size']}
|
| 351 |
-
- **Iterations**: {ex['reverse_diff']['iterations']}
|
| 352 |
-
- **Epsilon**: {ex['reverse_diff']['epsilon']}
|
| 353 |
-
"""
|
| 354 |
-
gr.Markdown(params_md)
|
| 355 |
|
| 356 |
if i < len(examples) - 1: # Don't add separator after the last example
|
| 357 |
gr.Markdown("---")
|
|
@@ -368,6 +369,15 @@ with gr.Blocks(title="Generative Inference Demo", css="""
|
|
| 368 |
outputs=[output_image, output_frames]
|
| 369 |
)
|
| 370 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 371 |
# About section
|
| 372 |
gr.Markdown("""
|
| 373 |
## About Generative Inference
|
|
@@ -381,12 +391,12 @@ with gr.Blocks(title="Generative Inference Demo", css="""
|
|
| 381 |
Moving away from the least likely class identified at iteration 0 (fast perception)
|
| 382 |
|
| 383 |
### Parameters:
|
| 384 |
-
- **
|
| 385 |
-
- **Diffusion Noise
|
| 386 |
-
- **
|
| 387 |
- **Number of Iterations**: How many optimization steps to perform
|
| 388 |
- **Model Layer**: Select a specific layer of the ResNet50 model to extract features from
|
| 389 |
-
- **Epsilon**: Controls the size of perturbation during optimization
|
| 390 |
|
| 391 |
**Generative Inference was developed by [Tahereh Toosi](https://toosi.github.io).**
|
| 392 |
""")
|
|
|
|
| 79 |
"[Brightness Perception](https://doi.org/10.1016/j.visres.2000.200.1)",
|
| 80 |
"[Edge Effects](https://doi.org/10.1016/j.tics.2003.08.003)"
|
| 81 |
],
|
| 82 |
+
"instructions": "Both blocks are gray in color (the same), use your finger to cover the middle line. Hit 'Load Parameters' and then hit 'Run Generative Inference' to see how the model sees the blocks.",
|
| 83 |
"method": "Prior-Guided Drift Diffusion",
|
| 84 |
"reverse_diff": {
|
| 85 |
"model": "resnet50_robust",
|
|
|
|
| 102 |
"method": "Prior-Guided Drift Diffusion",
|
| 103 |
"reverse_diff": {
|
| 104 |
"model": "resnet50_robust",
|
| 105 |
+
"layer": "avgpool",
|
| 106 |
+
"initial_noise": 0.9,
|
| 107 |
+
"diffusion_noise": 0.003,
|
| 108 |
+
"step_size": 0.58,
|
| 109 |
+
"iterations": 100,
|
| 110 |
+
"epsilon": 0.81
|
| 111 |
}
|
| 112 |
},
|
| 113 |
{
|
|
|
|
| 244 |
example["reverse_diff"]["initial_noise"], # Initial noise
|
| 245 |
example["reverse_diff"]["diffusion_noise"], # Diffusion noise value (corrected)
|
| 246 |
example["reverse_diff"]["step_size"], # Step size (added)
|
| 247 |
+
example["reverse_diff"]["layer"], # Model layer
|
| 248 |
+
gr.Group(visible=True) # Show parameters section
|
| 249 |
]
|
| 250 |
|
| 251 |
# Define the interface
|
|
|
|
| 275 |
# Inputs
|
| 276 |
image_input = gr.Image(label="Input Image", type="pil", value=os.path.join("stimuli", "Neon_Color_Circle.jpg"))
|
| 277 |
|
| 278 |
+
# Run Inference button right below the image
|
| 279 |
+
run_button = gr.Button("🪄 Run Generative Inference", variant="primary", elem_classes="purple-button")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 280 |
|
| 281 |
+
# Parameters toggle button
|
| 282 |
+
params_button = gr.Button("⚙️ Play with the parameters", variant="secondary")
|
|
|
|
| 283 |
|
| 284 |
+
# Parameters section (initially hidden)
|
| 285 |
+
with gr.Group(visible=False) as params_section:
|
| 286 |
+
with gr.Row():
|
| 287 |
+
model_choice = gr.Dropdown(
|
| 288 |
+
choices=["resnet50_robust", "standard_resnet50"], # "resnet50_robust_face" - hidden for deployment
|
| 289 |
+
value="resnet50_robust",
|
| 290 |
+
label="Model"
|
| 291 |
+
)
|
| 292 |
+
|
| 293 |
+
inference_type = gr.Dropdown(
|
| 294 |
+
choices=["Prior-Guided Drift Diffusion", "IncreaseConfidence"],
|
| 295 |
+
value="Prior-Guided Drift Diffusion",
|
| 296 |
+
label="Inference Method"
|
| 297 |
+
)
|
| 298 |
|
| 299 |
+
with gr.Row():
|
| 300 |
+
eps_slider = gr.Slider(minimum=0.0, maximum=40.0, value=20.0, step=0.01, label="Epsilon (Stimulus Fidelity)")
|
| 301 |
+
iterations_slider = gr.Slider(minimum=1, maximum=600, value=101, step=1, label="Number of Iterations") # Updated max to 600
|
| 302 |
+
|
| 303 |
+
with gr.Row():
|
| 304 |
+
initial_noise_slider = gr.Slider(minimum=0.0, maximum=1.0, value=0.8, step=0.01,
|
| 305 |
+
label="Drift Noise")
|
| 306 |
+
diffusion_noise_slider = gr.Slider(minimum=0.0, maximum=0.05, value=0.003, step=0.001,
|
| 307 |
+
label="Diffusion Noise") # Corrected name
|
| 308 |
+
|
| 309 |
+
with gr.Row():
|
| 310 |
+
step_size_slider = gr.Slider(minimum=0.01, maximum=2.0, value=1.0, step=0.01,
|
| 311 |
+
label="Update Rate") # Added step size slider
|
| 312 |
+
layer_choice = gr.Dropdown(
|
| 313 |
+
choices=["all", "conv1", "bn1", "relu", "maxpool", "layer1", "layer2", "layer3", "layer4", "avgpool"],
|
| 314 |
+
value="layer3",
|
| 315 |
+
label="Model Layer"
|
| 316 |
+
)
|
| 317 |
|
| 318 |
with gr.Column(scale=2):
|
| 319 |
# Outputs
|
|
|
|
| 340 |
image_input, model_choice, inference_type,
|
| 341 |
eps_slider, iterations_slider,
|
| 342 |
initial_noise_slider, diffusion_noise_slider,
|
| 343 |
+
step_size_slider, layer_choice, params_section
|
| 344 |
]
|
| 345 |
)
|
| 346 |
|
|
|
|
| 349 |
gr.Markdown(f"### {ex['name']}")
|
| 350 |
gr.Markdown(f"[Read more on Wikipedia]({ex['wiki']})")
|
| 351 |
|
| 352 |
+
# Show instructions if they exist
|
| 353 |
+
if "instructions" in ex:
|
| 354 |
+
gr.Markdown(f"**Instructions:** {ex['instructions']}")
|
| 355 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 356 |
|
| 357 |
if i < len(examples) - 1: # Don't add separator after the last example
|
| 358 |
gr.Markdown("---")
|
|
|
|
| 369 |
outputs=[output_image, output_frames]
|
| 370 |
)
|
| 371 |
|
| 372 |
+
# Toggle parameters visibility
|
| 373 |
+
def toggle_params():
|
| 374 |
+
return gr.Group(visible=True)
|
| 375 |
+
|
| 376 |
+
params_button.click(
|
| 377 |
+
fn=toggle_params,
|
| 378 |
+
outputs=[params_section]
|
| 379 |
+
)
|
| 380 |
+
|
| 381 |
# About section
|
| 382 |
gr.Markdown("""
|
| 383 |
## About Generative Inference
|
|
|
|
| 391 |
Moving away from the least likely class identified at iteration 0 (fast perception)
|
| 392 |
|
| 393 |
### Parameters:
|
| 394 |
+
- **Drift Noise**: Controls the amount of noise added to the image at the beginning
|
| 395 |
+
- **Diffusion Noise**: Controls the amount of noise added at each optimization step
|
| 396 |
+
- **Update Rate**: Learning rate for the optimization process
|
| 397 |
- **Number of Iterations**: How many optimization steps to perform
|
| 398 |
- **Model Layer**: Select a specific layer of the ResNet50 model to extract features from
|
| 399 |
+
- **Epsilon (Stimulus Fidelity)**: Controls the size of perturbation during optimization
|
| 400 |
|
| 401 |
**Generative Inference was developed by [Tahereh Toosi](https://toosi.github.io).**
|
| 402 |
""")
|
inference.py
CHANGED
|
@@ -25,7 +25,8 @@ print(f"Using device: {device}")
|
|
| 25 |
# Constants
|
| 26 |
MODEL_URLS = {
|
| 27 |
'resnet50_robust': 'https://huggingface.co/madrylab/robust-imagenet-models/resolve/main/resnet50_l2_eps3.ckpt',
|
| 28 |
-
'resnet50_standard': 'https://huggingface.co/madrylab/robust-imagenet-models/resolve/main/resnet50_l2_eps0.ckpt'
|
|
|
|
| 29 |
}
|
| 30 |
|
| 31 |
IMAGENET_MEAN = [0.485, 0.456, 0.406]
|
|
@@ -162,7 +163,12 @@ def download_model(model_type):
|
|
| 162 |
if model_type not in MODEL_URLS or MODEL_URLS[model_type] is None:
|
| 163 |
return None # Use PyTorch's pretrained model
|
| 164 |
|
| 165 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 166 |
if not model_path.exists():
|
| 167 |
print(f"Downloading {model_type} model...")
|
| 168 |
url = MODEL_URLS[model_type]
|
|
|
|
| 25 |
# Constants
|
| 26 |
MODEL_URLS = {
|
| 27 |
'resnet50_robust': 'https://huggingface.co/madrylab/robust-imagenet-models/resolve/main/resnet50_l2_eps3.ckpt',
|
| 28 |
+
'resnet50_standard': 'https://huggingface.co/madrylab/robust-imagenet-models/resolve/main/resnet50_l2_eps0.ckpt',
|
| 29 |
+
'resnet50_robust_face': 'https://huggingface.co/ttoosi/resnet50_robust_face/blob/main/100_checkpoint.pt'
|
| 30 |
}
|
| 31 |
|
| 32 |
IMAGENET_MEAN = [0.485, 0.456, 0.406]
|
|
|
|
| 163 |
if model_type not in MODEL_URLS or MODEL_URLS[model_type] is None:
|
| 164 |
return None # Use PyTorch's pretrained model
|
| 165 |
|
| 166 |
+
# Handle special case for face model
|
| 167 |
+
if model_type == 'resnet50_robust_face':
|
| 168 |
+
model_path = Path("models/resnet50_robust_face_100_checkpoint.pt")
|
| 169 |
+
else:
|
| 170 |
+
model_path = Path(f"models/{model_type}.pt")
|
| 171 |
+
|
| 172 |
if not model_path.exists():
|
| 173 |
print(f"Downloading {model_type} model...")
|
| 174 |
url = MODEL_URLS[model_type]
|