

"""simple docstring"""


import argparse
import logging
import os
import re

import tensorflow as tf

from transformers import (
    AutoConfig,
    AutoTokenizer,
    DataCollatorForLanguageModeling,
    PushToHubCallback,
    TFAutoModelForMaskedLM,
    create_optimizer,
)


a_       							=    logging.getLogger(__name__)

a_       							=    tf.data.AUTOTUNE






def      a__ (				)	->	int:
				_A					  =			argparse.ArgumentParser(description="Train a masked language model on TPU."   )
				parser.add_argument(
				    "--pretrained_model_config"     ,			type=lowerCAmelCase__     ,			default="roberta-base"     ,			help="The model config to use. Note that we don\'t copy the model\'s weights, only the config!"     ,			)
				parser.add_argument(
				    "--tokenizer"     ,			type=lowerCAmelCase__     ,			default="unigram-tokenizer-wikitext"     ,			help="The name of the tokenizer to load. We use the pretrained tokenizer to initialize the model\'s vocab size."     ,			)

				parser.add_argument(
				    "--per_replica_batch_size"     ,			type=lowerCAmelCase__     ,			default=8     ,			help="Batch size per TPU core."     ,			)

				parser.add_argument(
				    "--no_tpu"     ,			action="store_true"     ,			help="If set, run on CPU and don\'t try to initialize a TPU. Useful for debugging on non-TPU instances."     ,			)

				parser.add_argument(
				    "--tpu_name"     ,			type=lowerCAmelCase__     ,			help="Name of TPU resource to initialize. Should be blank on Colab, and \'local\' on TPU VMs."     ,			default="local"     ,			)

				parser.add_argument(
				    "--tpu_zone"     ,			type=lowerCAmelCase__     ,			help="Google cloud zone that TPU resource is located in. Only used for non-Colab TPU nodes."     ,			)

				parser.add_argument(
				    "--gcp_project"     ,			type=lowerCAmelCase__     ,			help="Google cloud project name. Only used for non-Colab TPU nodes."   )

				parser.add_argument(
				    "--bfloat16"     ,			action="store_true"     ,			help="Use mixed-precision bfloat16 for training. This is the recommended lower-precision format for TPU."     ,			)

				parser.add_argument(
				    "--train_dataset"     ,			type=lowerCAmelCase__     ,			help="Path to training dataset to load. If the path begins with `gs://`"
				    " then the dataset will be loaded from a Google Cloud Storage bucket."     ,			)

				parser.add_argument(
				    "--shuffle_buffer_size"     ,			type=lowerCAmelCase__     ,			default=2**18     ,			help="Size of the shuffle buffer (in samples)"     ,			)

				parser.add_argument(
				    "--eval_dataset"     ,			type=lowerCAmelCase__     ,			help="Path to evaluation dataset to load. If the path begins with `gs://`"
				    " then the dataset will be loaded from a Google Cloud Storage bucket."     ,			)

				parser.add_argument(
				    "--num_epochs"     ,			type=lowerCAmelCase__     ,			default=1     ,			help="Number of epochs to train for."     ,			)

				parser.add_argument(
				    "--learning_rate"     ,			type=lowerCAmelCase__     ,			default=1E-4     ,			help="Learning rate to use for training."     ,			)

				parser.add_argument(
				    "--weight_decay_rate"     ,			type=lowerCAmelCase__     ,			default=1E-3     ,			help="Weight decay rate to use for training."     ,			)

				parser.add_argument(
				    "--max_length"     ,			type=lowerCAmelCase__     ,			default=512     ,			help="Maximum length of tokenized sequences. Should match the setting used in prepare_tfrecord_shards.py"     ,			)

				parser.add_argument(
				    "--mlm_probability"     ,			type=lowerCAmelCase__     ,			default=0.15     ,			help="Fraction of tokens to mask during training."     ,			)

				parser.add_argument("--output_dir"     ,			type=lowerCAmelCase__     ,			required=lowerCAmelCase__     ,			help="Path to save model checkpoints to."   )
				parser.add_argument("--hub_model_id"     ,			type=lowerCAmelCase__     ,			help="Model ID to upload to on the Hugging Face Hub."   )

				_A					  =			parser.parse_args()
				return args






def      a__ (				__lowercase   )	->	Dict:







				try:
								if args.tpu_name:
												_A					  =			tf.distribute.cluster_resolver.TPUClusterResolver(
												    args.tpu_name     ,			zone=args.tpu_zone     ,			project=args.gcp_project   )
								else:
												_A					  =			tf.distribute.cluster_resolver.TPUClusterResolver()
				except ValueError:
								raise RuntimeError(
								    "Couldn\'t connect to TPU! Most likely you need to specify --tpu_name, --tpu_zone, or "
								    "--gcp_project. When running on a TPU VM, use --tpu_name local."   )

				tf.config.experimental_connect_to_cluster(lowerCAmelCase__   )
				tf.tpu.experimental.initialize_tpu_system(lowerCAmelCase__   )

				return tpu






def      a__ (				__lowercase   )	->	Any:
				_A					  =			0
				for file in file_list:
								_A					  =			file.split("/"   )[-1]
								_A					  =			re.search(R"-\d+-(\d+)\.tfrecord"     ,			lowerCAmelCase__   ).group(1   )
								_A					  =			int(lowerCAmelCase__   )
								num_samples += sample_count

				return num_samples






def      a__ (				__lowercase     ,			__lowercase     ,			__lowercase     ,			__lowercase     ,			__lowercase     ,			__lowercase=None   )	->	Union[str, Any]:
				_A					  =			count_samples(lowerCAmelCase__   )
				_A					  =			tf.data.Dataset.from_tensor_slices(lowerCAmelCase__   )
				if shuffle:
								_A					  =			dataset.shuffle(len(lowerCAmelCase__   )   )
				_A					  =			tf.data.TFRecordDataset(lowerCAmelCase__     ,			num_parallel_reads=lowerCAmelCase__   )
				# TF can't infer the total sample count because it doesn't read all the records yet, so we assert it here
				_A					  =			dataset.apply(tf.data.experimental.assert_cardinality(lowerCAmelCase__   )   )
				_A					  =			dataset.map(lowerCAmelCase__     ,			num_parallel_calls=lowerCAmelCase__   )
				if shuffle:
								assert shuffle_buffer_size is not None
								_A					  =			dataset.shuffle(args.shuffle_buffer_size   )
				_A					  =			dataset.batch(lowerCAmelCase__     ,			drop_remainder=lowerCAmelCase__   )
				_A					  =			dataset.map(lowerCAmelCase__     ,			num_parallel_calls=lowerCAmelCase__   )
				_A					  =			dataset.prefetch(lowerCAmelCase__   )
				return dataset






def      a__ (				__lowercase   )	->	Union[str, Any]:







				if not args.no_tpu:
								_A					  =			initialize_tpu(lowerCAmelCase__   )
								_A					  =			tf.distribute.TPUStrategy(lowerCAmelCase__   )
				else:
								_A					  =			tf.distribute.OneDeviceStrategy(device="/gpu:0"   )

				if args.bfloataa:
								tf.keras.mixed_precision.set_global_policy("mixed_bfloat16"   )

				_A					  =			AutoTokenizer.from_pretrained(args.tokenizer   )
				_A					  =			AutoConfig.from_pretrained(args.pretrained_model_config   )
				_A					  =			tokenizer.vocab_size

				_A					  =			tf.io.gfile.glob(os.path.join(args.train_dataset     ,			"*.tfrecord"   )   )
				if not training_records:
								raise ValueError(f"""No .tfrecord files found in {args.train_dataset}."""   )
				_A					  =			tf.io.gfile.glob(os.path.join(args.eval_dataset     ,			"*.tfrecord"   )   )
				if not eval_records:
								raise ValueError(f"""No .tfrecord files found in {args.eval_dataset}."""   )

				_A					  =			count_samples(lowerCAmelCase__   )

				_A					  =			num_train_samples // (args.per_replica_batch_size * strategy.num_replicas_in_sync)
				_A					  =			steps_per_epoch * args.num_epochs

				with strategy.scope():
								_A					  =			TFAutoModelForMaskedLM.from_config(lowerCAmelCase__   )
								model(model.dummy_inputs   )  # Pass some dummy inputs through the model to ensure all the weights are built
								_A					  =			create_optimizer(
								    num_train_steps=lowerCAmelCase__     ,			num_warmup_steps=total_train_steps // 20     ,			init_lr=args.learning_rate     ,			weight_decay_rate=args.weight_decay_rate     ,			)

								# Transformers models compute the right loss for their task by default when labels are passed, and will
								# use this for training unless you specify your own loss function in compile().
								model.compile(optimizer=lowerCAmelCase__     ,			metrics=["accuracy"]   )

				def decode_fn(__lowercase   ):
								_A					  =			{
								    'input_ids': tf.io.FixedLenFeature(dtype=tf.intaa     ,			shape=(args.max_length,)   ),
								    'attention_mask': tf.io.FixedLenFeature(dtype=tf.intaa     ,			shape=(args.max_length,)   ),
								}
								return tf.io.parse_single_example(lowerCAmelCase__     ,			lowerCAmelCase__   )

				# Many of the data collators in Transformers are TF-compilable when return_tensors == "tf", so we can
				# use their methods in our data pipeline.
				_A					  =			DataCollatorForLanguageModeling(
				    tokenizer=lowerCAmelCase__     ,			mlm_probability=args.mlm_probability     ,			mlm=lowerCAmelCase__     ,			return_tensors="tf"   )

				def mask_with_collator(__lowercase   ):
								# TF really needs an isin() function
								_A					  =			(
								    ~tf.cast(batch["attention_mask"]     ,			tf.bool   )
								    | (batch['input_ids'] == tokenizer.cls_token_id)
								    | (batch['input_ids'] == tokenizer.sep_token_id)
								)
								_A					  =			data_collator.tf_mask_tokens(
								    batch["input_ids"]     ,			vocab_size=len(lowerCAmelCase__   )     ,			mask_token_id=tokenizer.mask_token_id     ,			special_tokens_mask=lowerCAmelCase__     ,			)
								return batch

				_A					  =			args.per_replica_batch_size * strategy.num_replicas_in_sync

				_A					  =			prepare_dataset(
				    lowerCAmelCase__     ,			decode_fn=lowerCAmelCase__     ,			mask_fn=lowerCAmelCase__     ,			batch_size=lowerCAmelCase__     ,			shuffle=lowerCAmelCase__     ,			shuffle_buffer_size=args.shuffle_buffer_size     ,			)

				_A					  =			prepare_dataset(
				    lowerCAmelCase__     ,			decode_fn=lowerCAmelCase__     ,			mask_fn=lowerCAmelCase__     ,			batch_size=lowerCAmelCase__     ,			shuffle=lowerCAmelCase__     ,			)

				_A					  =			[]
				if args.hub_model_id:
								callbacks.append(
								    PushToHubCallback(output_dir=args.output_dir     ,			hub_model_id=args.hub_model_id     ,			tokenizer=lowerCAmelCase__   )   )

				model.fit(
				    lowerCAmelCase__     ,			validation_data=lowerCAmelCase__     ,			epochs=args.num_epochs     ,			callbacks=lowerCAmelCase__     ,			)

				model.save_pretrained(args.output_dir   )


if __name__ == "__main__":
		a_       							=    parse_args()
		main(args)



"""simple docstring"""


import copy
import os
from typing import Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


a_       							=    logging.get_logger(__name__)

a_       							=    {
    "Salesforce/blip-vqa-base": "https://huggingface.co/Salesforce/blip-vqa-base/resolve/main/config.json",
    "Salesforce/blip-vqa-capfit-large": (
        "https://huggingface.co/Salesforce/blip-vqa-base-capfit/resolve/main/config.json"
    ),
    "Salesforce/blip-image-captioning-base": (
        "https://huggingface.co/Salesforce/blip-image-captioning-base/resolve/main/config.json"
    ),
    "Salesforce/blip-image-captioning-large": (
        "https://huggingface.co/Salesforce/blip-image-captioning-large/resolve/main/config.json"
    ),
    "Salesforce/blip-itm-base-coco": "https://huggingface.co/Salesforce/blip-itm-base-coco/resolve/main/config.json",
    "Salesforce/blip-itm-large-coco": "https://huggingface.co/Salesforce/blip-itm-large-coco/resolve/main/config.json",
    "Salesforce/blip-itm-base-flikr": "https://huggingface.co/Salesforce/blip-itm-base-flikr/resolve/main/config.json",
    "Salesforce/blip-itm-large-flikr": (
        "https://huggingface.co/Salesforce/blip-itm-large-flikr/resolve/main/config.json"
    ),
}
class snake_case      (  _UpperCamelCase):
		__UpperCamelCase           =			'blip_text_model'







		def __init__(					self      :       int    ,   a__      :       List[str]=3_05_24    ,   a__      :       List[str]=7_68    ,   a__      :       List[Any]=7_68    ,   a__      :       int=30_72    ,   a__      :       List[str]=7_68    ,   a__      :       Dict=12    ,   a__      :       Optional[int]=8    ,   a__      :       Optional[Any]=5_12    ,   a__      :       List[Any]="gelu"    ,   a__      :       Optional[Any]=1E-1_2    ,   a__      :       Any=0.0    ,   a__      :       int=0.0    ,   a__      :       Dict=0.0_2    ,   a__      :       Optional[Any]=3_05_22    ,   a__      :       Any=2    ,   a__      :       int=0    ,   a__      :       Union[str, Any]=1_02    ,   a__      :       Tuple=True    ,   a__      :       Optional[int]=True    ,   **a__      :       Any    ,   )       ->						List[Any]:





						'''simple docstring'''







						super().__init__(
						    pad_token_id=a__    ,   bos_token_id=a__    ,   eos_token_id=a__    ,   sep_token_id=a__    ,   **a__    ,   )

						_A					  =			vocab_size
						_A					  =			hidden_size
						_A					  =			encoder_hidden_size
						_A					  =			intermediate_size
						_A					  =			projection_dim
						_A					  =			hidden_dropout_prob
						_A					  =			num_hidden_layers
						_A					  =			num_attention_heads
						_A					  =			max_position_embeddings
						_A					  =			layer_norm_eps
						_A					  =			hidden_act
						_A					  =			initializer_range
						_A					  =			attention_probs_dropout_prob
						_A					  =			is_decoder
						_A					  =			use_cache







		@classmethod
		def  a_ (					cls      :       Optional[Any]    ,   a__      :       Union[str, os.PathLike]    ,   **a__      :       Optional[Any]		)       ->						"PretrainedConfig":





						'''simple docstring'''







						cls._set_token_in_kwargs(a__		)

						_A  ,		_A					  =			cls.get_config_dict(a__    ,   **a__		)

						# get the text config dict if we are loading from BlipConfig
						if config_dict.get("model_type"		) == "blip":
										_A					  =			config_dict["text_config"]

						if "model_type" in config_dict and hasattr(cls    ,   "model_type"		) and config_dict["model_type"] != cls.model_type:
										logger.warning(
										    F"""You are using a model of type {config_dict['model_type']} to instantiate a model of type """
										    F"""{cls.model_type}. This is not supported for all configurations of models and can yield errors."""		)

						return cls.from_dict(a__    ,   **a__		)





class snake_case      (  _UpperCamelCase):
		__UpperCamelCase           =			'blip_vision_model'







		def __init__(					self      :       Optional[Any]    ,   a__      :       Any=7_68    ,   a__      :       List[str]=30_72    ,   a__      :       str=5_12    ,   a__      :       Any=12    ,   a__      :       int=12    ,   a__      :       int=3_84    ,   a__      :       Tuple=16    ,   a__      :       str="gelu"    ,   a__      :       Tuple=1E-5    ,   a__      :       List[str]=0.0    ,   a__      :       List[Any]=1E-1_0    ,   **a__      :       int    ,   )       ->						List[str]:





						'''simple docstring'''







						super().__init__(**a__		)

						_A					  =			hidden_size
						_A					  =			intermediate_size
						_A					  =			projection_dim
						_A					  =			num_hidden_layers
						_A					  =			num_attention_heads
						_A					  =			patch_size
						_A					  =			image_size
						_A					  =			initializer_range
						_A					  =			attention_dropout
						_A					  =			layer_norm_eps
						_A					  =			hidden_act







		@classmethod
		def  a_ (					cls      :       Any    ,   a__      :       Union[str, os.PathLike]    ,   **a__      :       int		)       ->						"PretrainedConfig":





						'''simple docstring'''







						cls._set_token_in_kwargs(a__		)

						_A  ,		_A					  =			cls.get_config_dict(a__    ,   **a__		)

						# get the vision config dict if we are loading from BlipConfig
						if config_dict.get("model_type"		) == "blip":
										_A					  =			config_dict["vision_config"]

						if "model_type" in config_dict and hasattr(cls    ,   "model_type"		) and config_dict["model_type"] != cls.model_type:
										logger.warning(
										    F"""You are using a model of type {config_dict['model_type']} to instantiate a model of type """
										    F"""{cls.model_type}. This is not supported for all configurations of models and can yield errors."""		)

						return cls.from_dict(a__    ,   **a__		)





class snake_case      (  _UpperCamelCase):
		__UpperCamelCase           =			'blip'
		__UpperCamelCase           =			True







		def __init__(					self      :       List[Any]    ,   a__      :       Optional[int]=None    ,   a__      :       str=None    ,   a__      :       List[str]=5_12    ,   a__      :       Any=2.6_5_9_2    ,   a__      :       str=2_56    ,   **a__      :       Optional[int]    ,   )       ->						Dict:





						'''simple docstring'''







						super().__init__(**a__		)

						if text_config is None:
										_A					  =			{}
										logger.info("`text_config` is `None`. Initializing the `BlipTextConfig` with default values."		)

						if vision_config is None:
										_A					  =			{}
										logger.info("`vision_config` is `None`. Initializing the `BlipVisionConfig` with default values."		)

						_A					  =			BlipTextConfig(**a__		)
						_A					  =			BlipVisionConfig(**a__		)

						_A					  =			self.vision_config.hidden_size

						_A					  =			projection_dim
						_A					  =			logit_scale_init_value
						_A					  =			1.0
						_A					  =			0.0_2
						_A					  =			image_text_hidden_size







		@classmethod
		def  a_ (					cls      :       Tuple    ,   a__      :       BlipTextConfig    ,   a__      :       BlipVisionConfig    ,   **a__      :       Optional[int]		)       ->						str:





						'''simple docstring'''







						return cls(text_config=text_config.to_dict()    ,   vision_config=vision_config.to_dict()    ,   **a__		)







		def  a_ (					self      :       Union[str, Any]		)       ->						List[Any]:





						'''simple docstring'''







						_A					  =			copy.deepcopy(self.__dict__		)
						_A					  =			self.text_config.to_dict()
						_A					  =			self.vision_config.to_dict()
						_A					  =			self.__class__.model_type
						return output