
import argparse

import requests
import torch

# pip3 install salesforce-lavis
# I'm actually installing a slightly modified version: pip3 install git+https://github.com/nielsrogge/LAVIS.git@fix_lavis
from lavis.models import load_model_and_preprocess
from PIL import Image

from transformers import (
    AutoTokenizer,
    BlipaConfig,
    BlipaForConditionalGeneration,
    BlipaProcessor,
    BlipaVisionConfig,
    BlipImageProcessor,
    OPTConfig,
    TaConfig,
)
from transformers.utils.constants import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD







def        A__					(						)  ->	Union[str, Any]:
				UpperCamelCase_:							List[str]	=						"""https://storage.googleapis.com/sfr-vision-language-research/LAVIS/assets/merlion.png"""
				UpperCamelCase_:							Any	=						Image.open(requests.get(lowerCamelCase     ,    stream=lowerCamelCase						).raw						).convert("""RGB"""						)

				return image







def        A__					(						lowerCamelCase						)  ->	str:
				UpperCamelCase_:							Optional[Any]	=						[]
				# fmt: off

				# vision encoder
				rename_keys.append(("""visual_encoder.cls_token""", """vision_model.embeddings.class_embedding""")						)
				rename_keys.append(("""visual_encoder.pos_embed""", """vision_model.embeddings.position_embedding""")						)
				rename_keys.append(("""visual_encoder.patch_embed.proj.weight""", """vision_model.embeddings.patch_embedding.weight""")						)
				rename_keys.append(("""visual_encoder.patch_embed.proj.bias""", """vision_model.embeddings.patch_embedding.bias""")						)
				rename_keys.append(("""ln_vision.weight""", """vision_model.post_layernorm.weight""")						)
				rename_keys.append(("""ln_vision.bias""", """vision_model.post_layernorm.bias""")						)

				for i in range(config.vision_config.num_hidden_layers						):
								rename_keys.append((F'''visual_encoder.blocks.{i}.norm1.weight''', F'''vision_model.encoder.layers.{i}.layer_norm1.weight''')						)
								rename_keys.append((F'''visual_encoder.blocks.{i}.norm1.bias''', F'''vision_model.encoder.layers.{i}.layer_norm1.bias''')						)
								rename_keys.append((F'''visual_encoder.blocks.{i}.norm2.weight''', F'''vision_model.encoder.layers.{i}.layer_norm2.weight''')						)
								rename_keys.append((F'''visual_encoder.blocks.{i}.norm2.bias''', F'''vision_model.encoder.layers.{i}.layer_norm2.bias''')						)
								rename_keys.append((F'''visual_encoder.blocks.{i}.attn.qkv.weight''', F'''vision_model.encoder.layers.{i}.self_attn.qkv.weight''')						)
								rename_keys.append((F'''visual_encoder.blocks.{i}.attn.proj.weight''', F'''vision_model.encoder.layers.{i}.self_attn.projection.weight''',)						)
								rename_keys.append((F'''visual_encoder.blocks.{i}.attn.proj.bias''', F'''vision_model.encoder.layers.{i}.self_attn.projection.bias''')						)
								rename_keys.append((F'''visual_encoder.blocks.{i}.mlp.fc1.weight''', F'''vision_model.encoder.layers.{i}.mlp.fc1.weight''')						)
								rename_keys.append((F'''visual_encoder.blocks.{i}.mlp.fc1.bias''', F'''vision_model.encoder.layers.{i}.mlp.fc1.bias''')						)
								rename_keys.append((F'''visual_encoder.blocks.{i}.mlp.fc2.weight''', F'''vision_model.encoder.layers.{i}.mlp.fc2.weight''')						)
								rename_keys.append((F'''visual_encoder.blocks.{i}.mlp.fc2.bias''', F'''vision_model.encoder.layers.{i}.mlp.fc2.bias''')						)

				# QFormer
				rename_keys.append(("""Qformer.bert.embeddings.LayerNorm.weight""", """qformer.layernorm.weight""")						)
				rename_keys.append(("""Qformer.bert.embeddings.LayerNorm.bias""", """qformer.layernorm.bias""")						)

				# fmt: on
				return rename_keys







def        A__					(						lowerCamelCase     ,    lowerCamelCase     ,    lowerCamelCase						)  ->	Any:
				UpperCamelCase_:							Any	=						dct.pop(lowerCamelCase						)
				UpperCamelCase_:							Any	=						val







def        A__					(						lowerCamelCase     ,    lowerCamelCase						)  ->	Optional[int]:
				for i in range(config.vision_config.num_hidden_layers						):
								# read in original q and v biases
								UpperCamelCase_:							List[str]	=						state_dict.pop(F'''visual_encoder.blocks.{i}.attn.q_bias'''						)
								UpperCamelCase_:							Optional[Any]	=						state_dict.pop(F'''visual_encoder.blocks.{i}.attn.v_bias'''						)

								# next, set bias in the state dict
								UpperCamelCase_:							List[Any]	=						torch.cat((q_bias, torch.zeros_like(lowerCamelCase     ,    requires_grad=lowerCamelCase						), v_bias)						)
								UpperCamelCase_:							int	=						qkv_bias







def        A__					(						lowerCamelCase     ,    lowerCamelCase						)  ->	Optional[int]:
				UpperCamelCase_:							Optional[int]	=						3_64 if """coco""" in model_name else 2_24
				UpperCamelCase_:							Optional[int]	=						BlipaVisionConfig(image_size=lowerCamelCase						).to_dict()

				# make sure the models have proper bos_token_id and eos_token_id set (important for generation)
				# seems like flan-T5 models don't have bos_token_id properly set?
				if "opt-2.7b" in model_name:
								UpperCamelCase_:							Optional[Any]	=						OPTConfig.from_pretrained("""facebook/opt-2.7b"""     ,    eos_token_id=lowerCamelCase						).to_dict()
				elif "opt-6.7b" in model_name:
								UpperCamelCase_:							int	=						OPTConfig.from_pretrained("""facebook/opt-6.7b"""     ,    eos_token_id=lowerCamelCase						).to_dict()
				elif "t5-xl" in model_name:
								UpperCamelCase_:							Optional[int]	=						TaConfig.from_pretrained("""google/flan-t5-xl"""     ,    dense_act_fn="""gelu"""     ,    bos_token_id=1						).to_dict()
				elif "t5-xxl" in model_name:
								UpperCamelCase_:							str	=						TaConfig.from_pretrained("""google/flan-t5-xxl"""     ,    dense_act_fn="""gelu"""     ,    bos_token_id=1						).to_dict()

				UpperCamelCase_:							Optional[Any]	=						BlipaConfig(vision_config=lowerCamelCase     ,    text_config=lowerCamelCase						)

				return config, image_size







@torch.no_grad()
def        A__					(						lowerCamelCase     ,    lowerCamelCase=None     ,    lowerCamelCase=False						)  ->	str:
				UpperCamelCase_:							List[str]	=						(
				    AutoTokenizer.from_pretrained("""facebook/opt-2.7b"""						)
				    if """opt""" in model_name
				    else AutoTokenizer.from_pretrained("""google/flan-t5-xl"""						)
				)
				UpperCamelCase_:							List[Any]	=						tokenizer("""\n"""     ,    add_special_tokens=lowerCamelCase						).input_ids[0]
				UpperCamelCase_,   UpperCamelCase_:							List[str]	=						get_blipa_config(lowerCamelCase     ,    eos_token_id=lowerCamelCase						)

				UpperCamelCase_:							Any	=						BlipaForConditionalGeneration(lowerCamelCase						).eval()

				UpperCamelCase_:							List[Any]	=						{
				    """blip2-opt-2.7b""": ("""blip2_opt""", """pretrain_opt2.7b"""),
				    """blip2-opt-6.7b""": ("""blip2_opt""", """pretrain_opt6.7b"""),
				    """blip2-opt-2.7b-coco""": ("""blip2_opt""", """caption_coco_opt2.7b"""),
				    """blip2-opt-6.7b-coco""": ("""blip2_opt""", """caption_coco_opt6.7b"""),
				    """blip2-flan-t5-xl""": ("""blip2_t5""", """pretrain_flant5xl"""),
				    """blip2-flan-t5-xl-coco""": ("""blip2_t5""", """caption_coco_flant5xl"""),
				    """blip2-flan-t5-xxl""": ("""blip2_t5""", """pretrain_flant5xxl"""),
				}

				UpperCamelCase_,   UpperCamelCase_:							List[Any]	=						model_name_to_original[model_name]

				# load original model
				print("""Loading original model..."""						)
				UpperCamelCase_:							str	=						"""cuda""" if torch.cuda.is_available() else """cpu"""
				UpperCamelCase_,   UpperCamelCase_,   UpperCamelCase_:							Any	=						load_model_and_preprocess(
				    name=lowerCamelCase     ,    model_type=lowerCamelCase     ,    is_eval=lowerCamelCase     ,    device=lowerCamelCase						)
				original_model.eval()
				print("""Done!"""						)

				# update state dict keys
				UpperCamelCase_:							List[Any]	=						original_model.state_dict()
				UpperCamelCase_:							Union[str, Any]	=						create_rename_keys(lowerCamelCase						)
				for src, dest in rename_keys:
								rename_key(lowerCamelCase     ,    lowerCamelCase     ,    lowerCamelCase						)

				# some keys can be renamed efficiently
				for key, val in state_dict.copy().items():
								UpperCamelCase_:							List[Any]	=						state_dict.pop(lowerCamelCase						)
								if key.startswith("""Qformer.bert"""						):
												UpperCamelCase_:							Dict	=						key.replace("""Qformer.bert"""     ,    """qformer"""						)
								if "attention.self" in key:
												UpperCamelCase_:							int	=						key.replace("""self"""     ,    """attention"""						)
								if "opt_proj" in key:
												UpperCamelCase_:							int	=						key.replace("""opt_proj"""     ,    """language_projection"""						)
								if "t5_proj" in key:
												UpperCamelCase_:							Optional[int]	=						key.replace("""t5_proj"""     ,    """language_projection"""						)
								if key.startswith("""opt"""						):
												UpperCamelCase_:							Any	=						key.replace("""opt"""     ,    """language"""						)
								if key.startswith("""t5"""						):
												UpperCamelCase_:							Dict	=						key.replace("""t5"""     ,    """language"""						)
								UpperCamelCase_:							Optional[int]	=						val

				# read in qv biases
				read_in_q_v_bias(lowerCamelCase     ,    lowerCamelCase						)

				UpperCamelCase_,   UpperCamelCase_:							Dict	=						hf_model.load_state_dict(lowerCamelCase     ,    strict=lowerCamelCase						)
				assert len(lowerCamelCase						) == 0
				assert unexpected_keys == ["qformer.embeddings.position_ids"]

				UpperCamelCase_:							List[Any]	=						load_demo_image()
				UpperCamelCase_:							Optional[int]	=						vis_processors["""eval"""](lowerCamelCase						).unsqueeze(0						).to(lowerCamelCase						)
				UpperCamelCase_:							Tuple	=						tokenizer(["""\n"""]     ,    return_tensors="""pt"""						).input_ids.to(lowerCamelCase						)

				# create processor
				UpperCamelCase_:							str	=						BlipImageProcessor(
				    size={"""height""": image_size, """width""": image_size}     ,    image_mean=lowerCamelCase     ,    image_std=lowerCamelCase						)
				UpperCamelCase_:							Dict	=						BlipaProcessor(image_processor=lowerCamelCase     ,    tokenizer=lowerCamelCase						)
				UpperCamelCase_:							Optional[Any]	=						processor(images=lowerCamelCase     ,    return_tensors="""pt"""						).pixel_values.to(lowerCamelCase						)

				# make sure processor creates exact same pixel values
				assert torch.allclose(lowerCamelCase     ,    lowerCamelCase						)

				original_model.to(lowerCamelCase						)
				hf_model.to(lowerCamelCase						)
				with torch.no_grad():
								if "opt" in model_name:
												UpperCamelCase_:							Any	=						original_model({"""image""": original_pixel_values, """text_input""": [""""""]}						).logits
												UpperCamelCase_:							Dict	=						hf_model(lowerCamelCase     ,    lowerCamelCase						).logits
								else:
												UpperCamelCase_:							List[str]	=						original_model(
												    {"""image""": original_pixel_values, """text_input""": ["""\n"""], """text_output""": ["""\n"""]}						).logits
												UpperCamelCase_:							Tuple	=						input_ids.masked_fill(input_ids == tokenizer.pad_token_id     ,    -1_00						)
												UpperCamelCase_:							Union[str, Any]	=						hf_model(lowerCamelCase     ,    lowerCamelCase     ,    labels=lowerCamelCase						).logits

				assert original_logits.shape == logits.shape
				print("""First values of original logits:"""     ,    original_logits[0, :3, :3]						)
				print("""First values of HF logits:"""     ,    logits[0, :3, :3]						)

				# assert values
				if model_name == "blip2-flan-t5-xl":
								UpperCamelCase_:							int	=						torch.tensor(
								    [[-41.5850, -4.4440, -8.9922], [-47.4322, -5.9143, -1.7340]]     ,    device=lowerCamelCase						)
								assert torch.allclose(logits[0, :3, :3]     ,    lowerCamelCase     ,    atol=1E-4						)
				elif model_name == "blip2-flan-t5-xl-coco":
								UpperCamelCase_:							Dict	=						torch.tensor(
								    [[-57.0109, -9.8967, -12.6280], [-68.6578, -12.7191, -10.5065]]     ,    device=lowerCamelCase						)
				else:
								# cast to same type
								UpperCamelCase_:							str	=						logits.dtype
								assert torch.allclose(original_logits.to(lowerCamelCase						)     ,    lowerCamelCase     ,    atol=1E-2						)
				print("""Looks ok!"""						)

				print("""Generating a caption..."""						)
				UpperCamelCase_:							Optional[Any]	=						""""""
				UpperCamelCase_:							Union[str, Any]	=						tokenizer(lowerCamelCase     ,    return_tensors="""pt"""						).input_ids.to(lowerCamelCase						)

				UpperCamelCase_:							List[str]	=						original_model.generate({"""image""": original_pixel_values}						)
				UpperCamelCase_:							Any	=						hf_model.generate(
				    lowerCamelCase     ,    lowerCamelCase     ,    do_sample=lowerCamelCase     ,    num_beams=5     ,    max_length=30     ,    min_length=1     ,    top_p=0.9     ,    repetition_penalty=1.0     ,    length_penalty=1.0     ,    temperature=1     ,    )
				print("""Original generation:"""     ,    lowerCamelCase						)
				UpperCamelCase_:							Any	=						input_ids.shape[1]
				UpperCamelCase_:							Dict	=						processor.batch_decode(outputs[:, prompt_length:]     ,    skip_special_tokens=lowerCamelCase						)
				UpperCamelCase_:							Any	=						[text.strip() for text in output_text]
				print("""HF generation:"""     ,    lowerCamelCase						)

				if pytorch_dump_folder_path is not None:
								processor.save_pretrained(lowerCamelCase						)
								hf_model.save_pretrained(lowerCamelCase						)

				if push_to_hub:
								processor.push_to_hub(F'''nielsr/{model_name}'''						)
								hf_model.push_to_hub(F'''nielsr/{model_name}'''						)


if __name__ == "__main__":
							lowerCamelCase_						: Union[str, Any]         =  argparse.ArgumentParser()
							lowerCamelCase_						: Dict         =  [
							    """blip2-opt-2.7b""",
							    """blip2-opt-6.7b""",
							    """blip2-opt-2.7b-coco""",
							    """blip2-opt-6.7b-coco""",
							    """blip2-flan-t5-xl""",
							    """blip2-flan-t5-xl-coco""",
							    """blip2-flan-t5-xxl""",
							]
							parser.add_argument(
							    """--model_name""",
							    default="""blip2-opt-2.7b""",
							    choices=choices,
							    type=str,
							    help="""Path to hf config.json of model to convert""",
							)
							parser.add_argument("""--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model.""")
							parser.add_argument(
							    """--push_to_hub""",
							    action="""store_true""",
							    help="""Whether to push the model and processor to the hub after converting""",
							)

							lowerCamelCase_						: Optional[Any]         =  parser.parse_args()

							convert_blipa_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)








from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowerCamelCase_						: str         =  {
    """configuration_roformer""": ["""ROFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP""", """RoFormerConfig""", """RoFormerOnnxConfig"""],
    """tokenization_roformer""": ["""RoFormerTokenizer"""],
}

try:
							if not is_tokenizers_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowerCamelCase_						: Union[str, Any]         =  ["""RoFormerTokenizerFast"""]

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowerCamelCase_						: Any         =  [
							    """ROFORMER_PRETRAINED_MODEL_ARCHIVE_LIST""",
							    """RoFormerForCausalLM""",
							    """RoFormerForMaskedLM""",
							    """RoFormerForMultipleChoice""",
							    """RoFormerForQuestionAnswering""",
							    """RoFormerForSequenceClassification""",
							    """RoFormerForTokenClassification""",
							    """RoFormerLayer""",
							    """RoFormerModel""",
							    """RoFormerPreTrainedModel""",
							    """load_tf_weights_in_roformer""",
							]


try:
							if not is_tf_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowerCamelCase_						: Dict         =  [
							    """TF_ROFORMER_PRETRAINED_MODEL_ARCHIVE_LIST""",
							    """TFRoFormerForCausalLM""",
							    """TFRoFormerForMaskedLM""",
							    """TFRoFormerForMultipleChoice""",
							    """TFRoFormerForQuestionAnswering""",
							    """TFRoFormerForSequenceClassification""",
							    """TFRoFormerForTokenClassification""",
							    """TFRoFormerLayer""",
							    """TFRoFormerModel""",
							    """TFRoFormerPreTrainedModel""",
							]


try:
							if not is_flax_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowerCamelCase_						: Optional[Any]         =  [
							    """FLAX_ROFORMER_PRETRAINED_MODEL_ARCHIVE_LIST""",
							    """FlaxRoFormerForMaskedLM""",
							    """FlaxRoFormerForMultipleChoice""",
							    """FlaxRoFormerForQuestionAnswering""",
							    """FlaxRoFormerForSequenceClassification""",
							    """FlaxRoFormerForTokenClassification""",
							    """FlaxRoFormerModel""",
							    """FlaxRoFormerPreTrainedModel""",
							]


if TYPE_CHECKING:
							from .configuration_roformer import ROFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, RoFormerConfig, RoFormerOnnxConfig
							from .tokenization_roformer import RoFormerTokenizer

							try:
														if not is_tokenizers_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .tokenization_roformer_fast import RoFormerTokenizerFast

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_roformer import (
														    ROFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
														    RoFormerForCausalLM,
														    RoFormerForMaskedLM,
														    RoFormerForMultipleChoice,
														    RoFormerForQuestionAnswering,
														    RoFormerForSequenceClassification,
														    RoFormerForTokenClassification,
														    RoFormerLayer,
														    RoFormerModel,
														    RoFormerPreTrainedModel,
														    load_tf_weights_in_roformer,
														)

							try:
														if not is_tf_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_tf_roformer import (
														    TF_ROFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
														    TFRoFormerForCausalLM,
														    TFRoFormerForMaskedLM,
														    TFRoFormerForMultipleChoice,
														    TFRoFormerForQuestionAnswering,
														    TFRoFormerForSequenceClassification,
														    TFRoFormerForTokenClassification,
														    TFRoFormerLayer,
														    TFRoFormerModel,
														    TFRoFormerPreTrainedModel,
														)

							try:
														if not is_flax_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_flax_roformer import (
														    FLAX_ROFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
														    FlaxRoFormerForMaskedLM,
														    FlaxRoFormerForMultipleChoice,
														    FlaxRoFormerForQuestionAnswering,
														    FlaxRoFormerForSequenceClassification,
														    FlaxRoFormerForTokenClassification,
														    FlaxRoFormerModel,
														    FlaxRoFormerPreTrainedModel,
														)


else:
							import sys

							lowerCamelCase_						: Tuple         =  _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)






