


'''simple docstring'''




import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import (
    BertTokenizer,
    ViltConfig,
    ViltForImageAndTextRetrieval,
    ViltForImagesAndTextClassification,
    ViltForMaskedLM,
    ViltForQuestionAnswering,
    ViltImageProcessor,
    ViltProcessor,
)
from transformers.utils import logging


logging.set_verbosity_info()
_lowerCAmelCase  :		Dict		     =	logging.get_logger(__name__)

def 			__UpperCamelCase    (    _A    :				Optional[int]					,   _A    :				int=False					,   _A    :				Union[str, Any]=False					,   _A    :				List[Any]=False				)    ->      List[str]:




	"""simple docstring"""





	lowerCAmelCase					:    Optional[int]      							= []
	for i in range(config.num_hidden_layers				):
		# encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
		rename_keys.append((F"transformer.blocks.{i}.norm1.weight", F"vilt.encoder.layer.{i}.layernorm_before.weight")				)
		rename_keys.append((F"transformer.blocks.{i}.norm1.bias", F"vilt.encoder.layer.{i}.layernorm_before.bias")				)
		rename_keys.append(
		    (F"transformer.blocks.{i}.attn.proj.weight", F"vilt.encoder.layer.{i}.attention.output.dense.weight")				)
		rename_keys.append(
		    (F"transformer.blocks.{i}.attn.proj.bias", F"vilt.encoder.layer.{i}.attention.output.dense.bias")				)
		rename_keys.append((F"transformer.blocks.{i}.norm2.weight", F"vilt.encoder.layer.{i}.layernorm_after.weight")				)
		rename_keys.append((F"transformer.blocks.{i}.norm2.bias", F"vilt.encoder.layer.{i}.layernorm_after.bias")				)
		rename_keys.append(
		    (F"transformer.blocks.{i}.mlp.fc1.weight", F"vilt.encoder.layer.{i}.intermediate.dense.weight")				)
		rename_keys.append((F"transformer.blocks.{i}.mlp.fc1.bias", F"vilt.encoder.layer.{i}.intermediate.dense.bias")				)
		rename_keys.append((F"transformer.blocks.{i}.mlp.fc2.weight", F"vilt.encoder.layer.{i}.output.dense.weight")				)
		rename_keys.append((F"transformer.blocks.{i}.mlp.fc2.bias", F"vilt.encoder.layer.{i}.output.dense.bias")				)

	# embeddings
	rename_keys.extend(
	    [
	        # text embeddings
	        ('text_embeddings.word_embeddings.weight', 'vilt.embeddings.text_embeddings.word_embeddings.weight'),
	        (
	            'text_embeddings.position_embeddings.weight',
	            'vilt.embeddings.text_embeddings.position_embeddings.weight',
	        ),
	        ('text_embeddings.position_ids', 'vilt.embeddings.text_embeddings.position_ids'),
	        (
	            'text_embeddings.token_type_embeddings.weight',
	            'vilt.embeddings.text_embeddings.token_type_embeddings.weight',
	        ),
	        ('text_embeddings.LayerNorm.weight', 'vilt.embeddings.text_embeddings.LayerNorm.weight'),
	        ('text_embeddings.LayerNorm.bias', 'vilt.embeddings.text_embeddings.LayerNorm.bias'),
	        # patch embeddings
	        ('transformer.cls_token', 'vilt.embeddings.cls_token'),
	        ('transformer.patch_embed.proj.weight', 'vilt.embeddings.patch_embeddings.projection.weight'),
	        ('transformer.patch_embed.proj.bias', 'vilt.embeddings.patch_embeddings.projection.bias'),
	        ('transformer.pos_embed', 'vilt.embeddings.position_embeddings'),
	        # token type embeddings
	        ('token_type_embeddings.weight', 'vilt.embeddings.token_type_embeddings.weight'),
	    ]				)

	# final layernorm + pooler
	rename_keys.extend(
	    [
	        ('transformer.norm.weight', 'vilt.layernorm.weight'),
	        ('transformer.norm.bias', 'vilt.layernorm.bias'),
	        ('pooler.dense.weight', 'vilt.pooler.dense.weight'),
	        ('pooler.dense.bias', 'vilt.pooler.dense.bias'),
	    ]				)

	# classifier head(s)
	if vqa_model:
		# classification head
		rename_keys.extend(
		    [
		        ('vqa_classifier.0.weight', 'classifier.0.weight'),
		        ('vqa_classifier.0.bias', 'classifier.0.bias'),
		        ('vqa_classifier.1.weight', 'classifier.1.weight'),
		        ('vqa_classifier.1.bias', 'classifier.1.bias'),
		        ('vqa_classifier.3.weight', 'classifier.3.weight'),
		        ('vqa_classifier.3.bias', 'classifier.3.bias'),
		    ]				)
	elif nlvr_model:
		# classification head
		rename_keys.extend(
		    [
		        ('nlvr2_classifier.0.weight', 'classifier.0.weight'),
		        ('nlvr2_classifier.0.bias', 'classifier.0.bias'),
		        ('nlvr2_classifier.1.weight', 'classifier.1.weight'),
		        ('nlvr2_classifier.1.bias', 'classifier.1.bias'),
		        ('nlvr2_classifier.3.weight', 'classifier.3.weight'),
		        ('nlvr2_classifier.3.bias', 'classifier.3.bias'),
		    ]				)
	else:
		pass

	return rename_keys

def 			__UpperCamelCase    (    _A    :				List[Any]					,   _A    :				str				)    ->      Union[str, Any]:




	"""simple docstring"""





	for i in range(config.num_hidden_layers				):
		lowerCAmelCase					:    Union[str, Any]      							= 'vilt.'
		# read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
		lowerCAmelCase					:    str      							= state_dict.pop(F"transformer.blocks.{i}.attn.qkv.weight"				)
		lowerCAmelCase					:    List[Any]      							= state_dict.pop(F"transformer.blocks.{i}.attn.qkv.bias"				)
		# next, add query, keys and values (in that order) to the state dict
		lowerCAmelCase					:    Optional[Any]      							= in_proj_weight[
		    : config.hidden_size, :
		]
		lowerCAmelCase					:    Dict      							= in_proj_bias[: config.hidden_size]
		lowerCAmelCase					:    List[str]      							= in_proj_weight[
		    config.hidden_size : config.hidden_size * 2, :
		]
		lowerCAmelCase					:    Tuple      							= in_proj_bias[
		    config.hidden_size : config.hidden_size * 2
		]
		lowerCAmelCase					:    Dict      							= in_proj_weight[
		    -config.hidden_size :, :
		]
		lowerCAmelCase					:    int      							= in_proj_bias[-config.hidden_size :]

def 			__UpperCamelCase    (    _A    :				Optional[int]				)    ->      List[Any]:




	"""simple docstring"""





	lowerCAmelCase					:    Optional[Any]      							= ['head.weight', 'head.bias']
	for k in ignore_keys:
		state_dict.pop(_A					,   _A				)

def 			__UpperCamelCase    (    _A    :				Any					,   _A    :				Dict					,   _A    :				Optional[int]				)    ->      Optional[int]:




	"""simple docstring"""





	lowerCAmelCase					:    str      							= dct.pop(_A				)
	lowerCAmelCase					:    str      							= val

@torch.no_grad()
def 			__UpperCamelCase    (    _A    :				str					,   _A    :				List[Any]				)    ->      Dict:




	"""simple docstring"""





	lowerCAmelCase					:    Dict      							= ViltConfig(image_size=3_84					,   patch_size=32					,   tie_word_embeddings=_A				)
	lowerCAmelCase					:    Union[str, Any]      							= False
	lowerCAmelCase					:    Union[str, Any]      							= False
	lowerCAmelCase					:    int      							= False
	lowerCAmelCase					:    Any      							= False
	if "vqa" in checkpoint_url:
		lowerCAmelCase					:    Union[str, Any]      							= True
		lowerCAmelCase					:    int      							= 31_29
		lowerCAmelCase					:    Dict      							= 'huggingface/label-files'
		lowerCAmelCase					:    Optional[int]      							= 'vqa2-id2label.json'
		lowerCAmelCase					:    Any      							= json.load(open(hf_hub_download(_A					,   _A					,   repo_type='dataset'				)					,   'r'				)				)
		lowerCAmelCase					:    Union[str, Any]      							= {int(_A				): v for k, v in idalabel.items()}
		lowerCAmelCase					:    Union[str, Any]      							= idalabel
		lowerCAmelCase					:    Union[str, Any]      							= {v: k for k, v in idalabel.items()}
		lowerCAmelCase					:    str      							= ViltForQuestionAnswering(_A				)
	elif "nlvr" in checkpoint_url:
		lowerCAmelCase					:    Optional[int]      							= True
		lowerCAmelCase					:    str      							= 2
		lowerCAmelCase					:    Any      							= {0: 'False', 1: 'True'}
		lowerCAmelCase					:    Tuple      							= {v: k for k, v in config.idalabel.items()}
		lowerCAmelCase					:    str      							= 3
		lowerCAmelCase					:    Optional[Any]      							= ViltForImagesAndTextClassification(_A				)
	elif "irtr" in checkpoint_url:
		lowerCAmelCase					:    Union[str, Any]      							= True
		lowerCAmelCase					:    int      							= ViltForImageAndTextRetrieval(_A				)
	elif "mlm_itm" in checkpoint_url:
		lowerCAmelCase					:    Optional[int]      							= True
		lowerCAmelCase					:    str      							= ViltForMaskedLM(_A				)
	else:
		raise ValueError('Unknown model type'				)

	# load state_dict of original model, remove and rename some keys
	lowerCAmelCase					:    int      							= torch.hub.load_state_dict_from_url(_A					,   map_location='cpu'				)['state_dict']
	lowerCAmelCase					:    int      							= create_rename_keys(_A					,   _A					,   _A					,   _A				)
	for src, dest in rename_keys:
		rename_key(_A					,   _A					,   _A				)
	read_in_q_k_v(_A					,   _A				)
	if mlm_model or irtr_model:
		lowerCAmelCase					:    Tuple      							= ['itm_score.fc.weight', 'itm_score.fc.bias']
		for k in ignore_keys:
			state_dict.pop(_A					,   _A				)

    # load state dict into HuggingFace model
	model.eval()
	if mlm_model:
		lowerCAmelCase							,   lowerCAmelCase					:    Optional[Any]      							= model.load_state_dict(_A					,   strict=_A				)
		assert missing_keys == ["mlm_score.decoder.bias"]
	else:
		model.load_state_dict(_A				)

	# Define processor
	lowerCAmelCase					:    int      							= ViltImageProcessor(size=3_84				)
	lowerCAmelCase					:    int      							= BertTokenizer.from_pretrained('bert-base-uncased'				)
	lowerCAmelCase					:    List[str]      							= ViltProcessor(_A					,   _A				)

	# Forward pass on example inputs (image + text)
	if nlvr_model:
		lowerCAmelCase					:    Tuple      							= Image.open(requests.get('https://lil.nlp.cornell.edu/nlvr/exs/ex0_0.jpg'					,   stream=_A				).raw				)
		lowerCAmelCase					:    int      							= Image.open(requests.get('https://lil.nlp.cornell.edu/nlvr/exs/ex0_0.jpg'					,   stream=_A				).raw				)
		lowerCAmelCase					:    str      							= (
		    'The left image contains twice the number of dogs as the right image, and at least two dogs in total are'
		    ' standing.'
		)
		lowerCAmelCase					:    str      							= processor(_A					,   _A					,   return_tensors='pt'				)
		lowerCAmelCase					:    int      							= processor(_A					,   _A					,   return_tensors='pt'				)
		lowerCAmelCase					:    Union[str, Any]      							= model(
		    input_ids=encoding_a.input_ids					,   pixel_values=encoding_a.pixel_values					,   pixel_values_a=encoding_a.pixel_values					,   )
	else:
		lowerCAmelCase					:    str      							= Image.open(requests.get('http://images.cocodataset.org/val2017/000000039769.jpg'					,   stream=_A				).raw				)
		if mlm_model:
			lowerCAmelCase					:    Tuple      							= 'a bunch of [MASK] laying on a [MASK].'
		else:
			lowerCAmelCase					:    int      							= 'How many cats are there?'
		lowerCAmelCase					:    Tuple      							= processor(_A					,   _A					,   return_tensors='pt'				)
		lowerCAmelCase					:    Optional[int]      							= model(**_A				)

	# Verify outputs
	if mlm_model:
		lowerCAmelCase					:    Union[str, Any]      							= torch.Size([1, 11, 3_05_22]				)
		lowerCAmelCase					:    int      							= torch.tensor([-12.50_61, -12.51_23, -12.51_74]				)
		assert outputs.logits.shape == expected_shape
		assert torch.allclose(outputs.logits[0, 0, :3]					,   _A					,   atol=1e-4				)

		# verify masked token prediction equals "cats"
		lowerCAmelCase					:    Optional[int]      							= outputs.logits[0, 4, :].argmax(-1				).item()
		assert tokenizer.decode([predicted_id]				) == "cats"
	elif vqa_model:
		lowerCAmelCase					:    Any      							= torch.Size([1, 31_29]				)
		lowerCAmelCase					:    Union[str, Any]      							= torch.tensor([-15.94_95, -18.14_72, -10.30_41]				)
		assert torch.allclose(outputs.logits[0, :3]					,   _A					,   atol=1e-4				)
		assert outputs.logits.shape == expected_shape
		assert torch.allclose(outputs.logits[0, 0, :3]					,   _A					,   atol=1e-4				)

		# verify vqa prediction equals "2"
		lowerCAmelCase					:    Optional[Any]      							= outputs.logits.argmax(-1				).item()
		assert model.config.idalabel[predicted_idx] == "2"
	elif nlvr_model:
		lowerCAmelCase					:    List[str]      							= torch.Size([1, 2]				)
		lowerCAmelCase					:    Tuple      							= torch.tensor([-2.87_21, 2.12_91]				)
		assert torch.allclose(outputs.logits[0, :3]					,   _A					,   atol=1e-4				)
		assert outputs.logits.shape == expected_shape

	Path(_A				).mkdir(exist_ok=_A				)
	print(F"Saving model and processor to {pytorch_dump_folder_path}"				)
	model.save_pretrained(_A				)
	processor.save_pretrained(_A				)


if __name__ == "__main__":
				_lowerCAmelCase  :		Dict		     =	argparse.ArgumentParser()
				# Required parameters
				parser.add_argument(
				    '--checkpoint_url',
				    default='https://github.com/dandelin/ViLT/releases/download/200k/vilt_200k_mlm_itm.ckpt',
				    type=str,
				    help='URL of the checkpoint you\'d like to convert.',
				)
				parser.add_argument(
				    '--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model directory.'
				)

				_lowerCAmelCase  :		str		     =	parser.parse_args()
				convert_vilt_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path)




'''simple docstring'''




import gc
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import AutoencoderKL, DDIMScheduler, LDMTextToImagePipeline, UNetaDConditionModel
from diffusers.utils.testing_utils import (
    enable_full_determinism,
    load_numpy,
    nightly,
    require_torch_gpu,
    slow,
    torch_device,
)

from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_PARAMS
from ..test_pipelines_common import PipelineTesterMixin


enable_full_determinism()



class  lowerCAmelCase				(				a      ,				unittest.TestCase							):
	_lowerCamelCase   :					List[Any]			      = LDMTextToImagePipeline
	_lowerCamelCase   :					Optional[Any]			      = TEXT_TO_IMAGE_PARAMS - {
	    """negative_prompt""",
	    """negative_prompt_embeds""",
	    """cross_attention_kwargs""",
	    """prompt_embeds""",
	}
	_lowerCamelCase   :					List[str]			      = PipelineTesterMixin.required_optional_params - {
	    """num_images_per_prompt""",
	    """callback""",
	    """callback_steps""",
	}
	_lowerCamelCase   :					Union[str, Any]			      = TEXT_TO_IMAGE_BATCH_PARAMS
	_lowerCamelCase   :					Optional[int]			      = False



	def 				lowercase     (					self    ):
		torch.manual_seed(0    )
		lowerCAmelCase					:    Dict      							= UNetaDConditionModel(
		    block_out_channels=(32, 64) ,					layers_per_block=2 ,					sample_size=32 ,					in_channels=4 ,					out_channels=4 ,					down_block_types=('DownBlock2D', 'CrossAttnDownBlock2D') ,					up_block_types=('CrossAttnUpBlock2D', 'UpBlock2D') ,					cross_attention_dim=32 ,					)
		lowerCAmelCase					:    int      							= DDIMScheduler(
		    beta_start=0.0_0_0_8_5 ,					beta_end=0.0_1_2 ,					beta_schedule='scaled_linear' ,					clip_sample=snake_case__ ,					set_alpha_to_one=snake_case__ ,					)
		torch.manual_seed(0    )
		lowerCAmelCase					:    str      							= AutoencoderKL(
		    block_out_channels=(32, 64) ,					in_channels=3 ,					out_channels=3 ,					down_block_types=('DownEncoderBlock2D', 'DownEncoderBlock2D') ,					up_block_types=('UpDecoderBlock2D', 'UpDecoderBlock2D') ,					latent_channels=4 ,					)
		torch.manual_seed(0    )
		lowerCAmelCase					:    Any      							= CLIPTextConfig(
		    bos_token_id=0 ,					eos_token_id=2 ,					hidden_size=32 ,					intermediate_size=37 ,					layer_norm_eps=1e-0_5 ,					num_attention_heads=4 ,					num_hidden_layers=5 ,					pad_token_id=1 ,					vocab_size=1000 ,					)
		lowerCAmelCase					:    str      							= CLIPTextModel(snake_case__    )
		lowerCAmelCase					:    str      							= CLIPTokenizer.from_pretrained('hf-internal-testing/tiny-random-clip'    )

		lowerCAmelCase					:    List[Any]      							= {
		    'unet': unet,
		    'scheduler': scheduler,
		    'vqvae': vae,
		    'bert': text_encoder,
		    'tokenizer': tokenizer,
		}
		return components



	def 				lowercase     (					self ,					snake_case__ ,					snake_case__=0    ):
		if str(snake_case__    ).startswith('mps'    ):
			lowerCAmelCase					:    Optional[int]      							= torch.manual_seed(snake_case__    )
		else:
			lowerCAmelCase					:    str      							= torch.Generator(device=snake_case__    ).manual_seed(snake_case__    )
		lowerCAmelCase					:    Tuple      							= {
		    'prompt': 'A painting of a squirrel eating a burger',
		    'generator': generator,
		    'num_inference_steps': 2,
		    'guidance_scale': 6.0,
		    'output_type': 'numpy',
		}
		return inputs



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    List[str]      							= 'cpu'  # ensure determinism for the device-dependent torch.Generator

		lowerCAmelCase					:    Optional[Any]      							= self.get_dummy_components()
		lowerCAmelCase					:    Optional[Any]      							= LDMTextToImagePipeline(**snake_case__    )
		pipe.to(snake_case__    )
		pipe.set_progress_bar_config(disable=snake_case__    )

		lowerCAmelCase					:    Tuple      							= self.get_dummy_inputs(snake_case__    )
		lowerCAmelCase					:    Union[str, Any]      							= pipe(**snake_case__    ).images
		lowerCAmelCase					:    str      							= image[0, -3:, -3:, -1]

		assert image.shape == (1, 16, 16, 3)
		lowerCAmelCase					:    List[Any]      							= np.array([0.6_1_0_1, 0.6_1_5_6, 0.5_6_2_2, 0.4_8_9_5, 0.6_6_6_1, 0.3_8_0_4, 0.5_7_4_8, 0.6_1_3_6, 0.5_0_1_4]    )

		assert np.abs(image_slice.flatten() - expected_slice    ).max() < 1e-3



@slow
@require_torch_gpu
class  lowerCAmelCase				(				unittest.TestCase							):



	def 				lowercase     (					self    ):
		super().tearDown()
		gc.collect()
		torch.cuda.empty_cache()



	def 				lowercase     (					self ,					snake_case__ ,					snake_case__=torch.floataa ,					snake_case__=0    ):
		lowerCAmelCase					:    List[str]      							= torch.manual_seed(snake_case__    )
		lowerCAmelCase					:    int      							= np.random.RandomState(snake_case__    ).standard_normal((1, 4, 32, 32)    )
		lowerCAmelCase					:    Optional[Any]      							= torch.from_numpy(snake_case__    ).to(device=snake_case__ ,					dtype=snake_case__    )
		lowerCAmelCase					:    List[str]      							= {
		    'prompt': 'A painting of a squirrel eating a burger',
		    'latents': latents,
		    'generator': generator,
		    'num_inference_steps': 3,
		    'guidance_scale': 6.0,
		    'output_type': 'numpy',
		}
		return inputs



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    Tuple      							= LDMTextToImagePipeline.from_pretrained('CompVis/ldm-text2im-large-256'    ).to(snake_case__    )
		pipe.set_progress_bar_config(disable=snake_case__    )

		lowerCAmelCase					:    Optional[Any]      							= self.get_inputs(snake_case__    )
		lowerCAmelCase					:    List[Any]      							= pipe(**snake_case__    ).images
		lowerCAmelCase					:    str      							= image[0, -3:, -3:, -1].flatten()

		assert image.shape == (1, 256, 256, 3)
		lowerCAmelCase					:    Tuple      							= np.array([0.5_1_8_2_5, 0.5_2_8_5_0, 0.5_2_5_4_3, 0.5_4_2_5_8, 0.5_2_3_0_4, 0.5_2_5_6_9, 0.5_4_3_6_3, 0.5_5_2_7_6, 0.5_6_8_7_8]    )
		lowerCAmelCase					:    int      							= np.abs(expected_slice - image_slice    ).max()
		assert max_diff < 1e-3



@nightly
@require_torch_gpu
class  lowerCAmelCase				(				unittest.TestCase							):



	def 				lowercase     (					self    ):
		super().tearDown()
		gc.collect()
		torch.cuda.empty_cache()



	def 				lowercase     (					self ,					snake_case__ ,					snake_case__=torch.floataa ,					snake_case__=0    ):
		lowerCAmelCase					:    List[str]      							= torch.manual_seed(snake_case__    )
		lowerCAmelCase					:    Any      							= np.random.RandomState(snake_case__    ).standard_normal((1, 4, 32, 32)    )
		lowerCAmelCase					:    List[Any]      							= torch.from_numpy(snake_case__    ).to(device=snake_case__ ,					dtype=snake_case__    )
		lowerCAmelCase					:    List[str]      							= {
		    'prompt': 'A painting of a squirrel eating a burger',
		    'latents': latents,
		    'generator': generator,
		    'num_inference_steps': 50,
		    'guidance_scale': 6.0,
		    'output_type': 'numpy',
		}
		return inputs



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    Optional[int]      							= LDMTextToImagePipeline.from_pretrained('CompVis/ldm-text2im-large-256'    ).to(snake_case__    )
		pipe.set_progress_bar_config(disable=snake_case__    )

		lowerCAmelCase					:    int      							= self.get_inputs(snake_case__    )
		lowerCAmelCase					:    Optional[int]      							= pipe(**snake_case__    ).images[0]

		lowerCAmelCase					:    Optional[int]      							= load_numpy(
		    'https://huggingface.co/datasets/diffusers/test-arrays/resolve/main/ldm_text2img/ldm_large_256_ddim.npy'    )
		lowerCAmelCase					:    List[str]      							= np.abs(expected_image - image    ).max()
		assert max_diff < 1e-3
