






import logging
import os
import sys
from dataclasses import dataclass, field
from typing import Optional

import evaluate
import numpy as np
import torch
from datasets import load_dataset
from PIL import Image
from torchvision.transforms import (
    CenterCrop,
    Compose,
    Normalize,
    RandomHorizontalFlip,
    RandomResizedCrop,
    Resize,
    ToTensor,
)

import transformers
from transformers import (
    MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
    AutoConfig,
    AutoImageProcessor,
    AutoModelForImageClassification,
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    set_seed,
)
from transformers.trainer_utils import get_last_checkpoint
from transformers.utils import check_min_version, send_example_telemetry
from transformers.utils.versions import require_version
lowerCamelCase : Optional[Any]						   =       logging.getLogger(__name__)

# Will error if the minimal version of Transformers is not installed. Remove at your own risks.
check_min_version('''4.31.0''')

require_version('''datasets>=1.8.0''', '''To fix: pip install -r examples/pytorch/image-classification/requirements.txt''')

lowerCamelCase : Tuple						   =       list(MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING.keys())
lowerCamelCase : str						   =       tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
def       snake_case_		(     lowerCAmelCase_       :						str      ):
							with open(lowerCAmelCase_       ,       """rb"""      ) as f:
														__lowercase      :							Tuple				=     Image.open(lowerCAmelCase_      )
														return im.convert("""RGB"""      )

@dataclass
class 		lowerCAmelCase     :

					'''simple docstring'''

					_A						:			Optional[str]									=      field(
					    default=__a      ,       metadata={
					        '''help''': '''Name of a dataset from the hub (could be your own, possibly private dataset hosted on the hub).'''
					    }      ,       )
					_A						:			Optional[str]									=      field(
					    default=__a      ,       metadata={'''help''': '''The configuration name of the dataset to use (via the datasets library).'''}			)
					_A						:			Optional[str]									=      field(default=__a      ,       metadata={'''help''': '''A folder containing the training data.'''}			)
					_A						:			Optional[str]									=      field(default=__a      ,       metadata={'''help''': '''A folder containing the validation data.'''}			)
					_A						:			Optional[float]									=      field(
					    default=0.1_5      ,       metadata={'''help''': '''Percent to split off of train for validation.'''}			)
					_A						:			Optional[int]									=      field(
					    default=__a      ,       metadata={
					        '''help''': (
					            '''For debugging purposes or quicker training, truncate the number of training examples to this '''
					            '''value if set.'''
					        )
					    }      ,       )
					_A						:			Optional[int]									=      field(
					    default=__a      ,       metadata={
					        '''help''': (
					            '''For debugging purposes or quicker training, truncate the number of evaluation examples to this '''
					            '''value if set.'''
					        )
					    }      ,       )



					def       lowerCAmelCase				(  self				:	str							)			->  str:


												"""simple docstring"""

												if self.dataset_name is None and (self.train_dir is None and self.validation_dir is None):
																			raise ValueError(
																			    """You must specify either a dataset name from the hub or a train and/or validation directory."""							)







@dataclass
class 		lowerCAmelCase     :

					'''simple docstring'''

					_A						:			str									=      field(
					    default='''google/vit-base-patch16-224-in21k'''      ,       metadata={'''help''': '''Path to pretrained model or model identifier from huggingface.co/models'''}      ,       )
					_A						:			Optional[str]									=      field(
					    default=__a      ,       metadata={'''help''': '''If training from scratch, pass a model type from the list: ''' + ''', '''.join(__a			)}      ,       )
					_A						:			Optional[str]									=      field(
					    default=__a      ,       metadata={'''help''': '''Pretrained config name or path if not the same as model_name'''}			)
					_A						:			Optional[str]									=      field(
					    default=__a      ,       metadata={'''help''': '''Where do you want to store the pretrained models downloaded from s3'''}			)
					_A						:			str									=      field(
					    default='''main'''      ,       metadata={'''help''': '''The specific model version to use (can be a branch name, tag name or commit id).'''}      ,       )
					_A						:			str									=      field(default=__a      ,       metadata={'''help''': '''Name or path of preprocessor config.'''}			)
					_A						:			bool									=      field(
					    default=__a      ,       metadata={
					        '''help''': (
					            '''Will use the token generated when running `huggingface-cli login` (necessary to use this script '''
					            '''with private models).'''
					        )
					    }      ,       )
					_A						:			bool									=      field(
					    default=__a      ,       metadata={'''help''': '''Will enable to load a pretrained model whose head dimensions are different.'''}      ,       )







def       snake_case_		(     lowerCAmelCase_       :						Optional[Any]      ):
							__lowercase      :							Tuple				=     torch.stack([example["""pixel_values"""] for example in examples]      )
							__lowercase      :							Dict				=     torch.tensor([example["""labels"""] for example in examples]      )
							return {"pixel_values": pixel_values, "labels": labels}
def       snake_case_		(     ):
							# See all possible arguments in src/transformers/training_args.py
							# or by passing the --help flag to this script.
							# We now keep distinct sets of args, for a cleaner separation of concerns.

							__lowercase      :							Any				=     HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)      )
							if len(sys.argv      ) == 2 and sys.argv[1].endswith(""".json"""      ):
														# If we pass only one argument to the script and it's the path to a json file,
														# let's parse it to get our arguments.
														__lowercase			,			__lowercase			,			__lowercase      :							List[str]				=     parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]      )      )
							else:
														__lowercase			,			__lowercase			,			__lowercase      :							Optional[int]				=     parser.parse_args_into_dataclasses()

							# Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The
							# information sent is the one passed as arguments along with your Python/PyTorch versions.
							send_example_telemetry("""run_image_classification"""       ,       lowerCAmelCase_       ,       lowerCAmelCase_      )

							# Setup logging
							logging.basicConfig(
							    format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s"""       ,       datefmt="""%m/%d/%Y %H:%M:%S"""       ,       handlers=[logging.StreamHandler(sys.stdout      )]       ,       )

							if training_args.should_log:
														# The default of training_args.log_level is passive, so we set log level at info here to have that default.
														transformers.utils.logging.set_verbosity_info()

							__lowercase      :							Any				=     training_args.get_process_log_level()
							logger.setLevel(lowerCAmelCase_      )
							transformers.utils.logging.set_verbosity(lowerCAmelCase_      )
							transformers.utils.logging.enable_default_handler()
							transformers.utils.logging.enable_explicit_format()

							# Log on each process the small summary:
							logger.warning(
							    F"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}"
							    + F"distributed training: {bool(training_args.local_rank != -1      )}, 16-bits training: {training_args.fpaa}"      )
							logger.info(F"Training/evaluation parameters {training_args}"      )

							# Detecting last checkpoint.
							__lowercase      :							int				=     None
							if os.path.isdir(training_args.output_dir      ) and training_args.do_train and not training_args.overwrite_output_dir:
														__lowercase      :							Dict				=     get_last_checkpoint(training_args.output_dir      )
														if last_checkpoint is None and len(os.listdir(training_args.output_dir      )      ) > 0:
																					raise ValueError(
																					    F"Output directory ({training_args.output_dir}) already exists and is not empty. "
																					    """Use --overwrite_output_dir to overcome."""      )
														elif last_checkpoint is not None and training_args.resume_from_checkpoint is None:
																					logger.info(
																					    F"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change "
																					    """the `--output_dir` or add `--overwrite_output_dir` to train from scratch."""      )

    # Set seed before initializing model.
							set_seed(training_args.seed      )

							# Initialize our dataset and prepare it for the 'image-classification' task.
							if data_args.dataset_name is not None:
														__lowercase      :							Tuple				=     load_dataset(
														    data_args.dataset_name       ,       data_args.dataset_config_name       ,       cache_dir=model_args.cache_dir       ,       task="""image-classification"""       ,       use_auth_token=True if model_args.use_auth_token else None       ,       )
							else:
														__lowercase      :							Tuple				=     {}
														if data_args.train_dir is not None:
																					__lowercase      :							Tuple				=     os.path.join(data_args.train_dir       ,       """**"""      )
														if data_args.validation_dir is not None:
																					__lowercase      :							Tuple				=     os.path.join(data_args.validation_dir       ,       """**"""      )
														__lowercase      :							Dict				=     load_dataset(
														    """imagefolder"""       ,       data_files=lowerCAmelCase_       ,       cache_dir=model_args.cache_dir       ,       task="""image-classification"""       ,       )

							# If we don't have a validation split, split off a percentage of train as validation.
							__lowercase      :							Union[str, Any]				=     None if """validation""" in dataset.keys() else data_args.train_val_split
							if isinstance(data_args.train_val_split       ,       lowerCAmelCase_      ) and data_args.train_val_split > 0.0:
														__lowercase      :							Optional[int]				=     dataset["""train"""].train_test_split(data_args.train_val_split      )
														__lowercase      :							List[Any]				=     split["""train"""]
														__lowercase      :							List[Any]				=     split["""test"""]

							# Prepare label mappings.
							# We'll include these in the model's config to get human readable labels in the Inference API.
							__lowercase      :							Union[str, Any]				=     dataset["""train"""].features["""labels"""].names
							__lowercase			,			__lowercase      :							List[str]				=     {}, {}
							for i, label in enumerate(lowerCAmelCase_      ):
														__lowercase      :							Union[str, Any]				=     str(lowerCAmelCase_      )
														__lowercase      :							Any				=     label

							# Load the accuracy metric from the datasets package
							__lowercase      :							str				=     evaluate.load("""accuracy"""      )

							# Define our compute_metrics function. It takes an `EvalPrediction` object (a namedtuple with a
							# predictions and label_ids field) and has to return a dictionary string to float.
							def compute_metrics(lowerCAmelCase_       :						str      ):
														return metric.compute(predictions=np.argmax(p.predictions       ,       axis=1      )       ,       references=p.label_ids      )

							__lowercase      :							Tuple				=     AutoConfig.from_pretrained(
							    model_args.config_name or model_args.model_name_or_path       ,       num_labels=len(lowerCAmelCase_      )       ,       labelaid=lowerCAmelCase_       ,       idalabel=lowerCAmelCase_       ,       finetuning_task="""image-classification"""       ,       cache_dir=model_args.cache_dir       ,       revision=model_args.model_revision       ,       use_auth_token=True if model_args.use_auth_token else None       ,       )
							__lowercase      :							Any				=     AutoModelForImageClassification.from_pretrained(
							    model_args.model_name_or_path       ,       from_tf=bool(""".ckpt""" in model_args.model_name_or_path      )       ,       config=lowerCAmelCase_       ,       cache_dir=model_args.cache_dir       ,       revision=model_args.model_revision       ,       use_auth_token=True if model_args.use_auth_token else None       ,       ignore_mismatched_sizes=model_args.ignore_mismatched_sizes       ,       )
							__lowercase      :							Dict				=     AutoImageProcessor.from_pretrained(
							    model_args.image_processor_name or model_args.model_name_or_path       ,       cache_dir=model_args.cache_dir       ,       revision=model_args.model_revision       ,       use_auth_token=True if model_args.use_auth_token else None       ,       )

							# Define torchvision transforms to be applied to each image.
							if "shortest_edge" in image_processor.size:
														__lowercase      :							str				=     image_processor.size["""shortest_edge"""]
							else:
														__lowercase      :							List[str]				=     (image_processor.size["""height"""], image_processor.size["""width"""])
							__lowercase      :							int				=     Normalize(mean=image_processor.image_mean       ,       std=image_processor.image_std      )
							__lowercase      :							Dict				=     Compose(
							    [
							        RandomResizedCrop(lowerCAmelCase_      ),
							        RandomHorizontalFlip(),
							        ToTensor(),
							        normalize,
							    ]      )
							__lowercase      :							List[Any]				=     Compose(
							    [
							        Resize(lowerCAmelCase_      ),
							        CenterCrop(lowerCAmelCase_      ),
							        ToTensor(),
							        normalize,
							    ]      )

							def train_transforms(lowerCAmelCase_       :						Tuple      ):
														__lowercase      :							int				=     [
														    _train_transforms(pil_img.convert("""RGB"""      )      ) for pil_img in example_batch["""image"""]
														]
														return example_batch

							def val_transforms(lowerCAmelCase_       :						int      ):
														__lowercase      :							Dict				=     [_val_transforms(pil_img.convert("""RGB"""      )      ) for pil_img in example_batch["""image"""]]
														return example_batch

							if training_args.do_train:
														if "train" not in dataset:
																					raise ValueError("""--do_train requires a train dataset"""      )
														if data_args.max_train_samples is not None:
																					__lowercase      :							List[str]				=     (
																					    dataset["""train"""].shuffle(seed=training_args.seed      ).select(range(data_args.max_train_samples      )      )
																					)
														# Set the training transforms
														dataset["train"].set_transform(lowerCAmelCase_      )

							if training_args.do_eval:
														if "validation" not in dataset:
																					raise ValueError("""--do_eval requires a validation dataset"""      )
														if data_args.max_eval_samples is not None:
																					__lowercase      :							Optional[Any]				=     (
																					    dataset["""validation"""].shuffle(seed=training_args.seed      ).select(range(data_args.max_eval_samples      )      )
																					)
														# Set the validation transforms
														dataset["validation"].set_transform(lowerCAmelCase_      )

							# Initalize our trainer
							__lowercase      :							List[Any]				=     Trainer(
							    model=lowerCAmelCase_       ,       args=lowerCAmelCase_       ,       train_dataset=dataset["""train"""] if training_args.do_train else None       ,       eval_dataset=dataset["""validation"""] if training_args.do_eval else None       ,       compute_metrics=lowerCAmelCase_       ,       tokenizer=lowerCAmelCase_       ,       data_collator=lowerCAmelCase_       ,       )

							# Training
							if training_args.do_train:
														__lowercase      :							int				=     None
														if training_args.resume_from_checkpoint is not None:
																					__lowercase      :							str				=     training_args.resume_from_checkpoint
														elif last_checkpoint is not None:
																					__lowercase      :							List[Any]				=     last_checkpoint
														__lowercase      :							Any				=     trainer.train(resume_from_checkpoint=lowerCAmelCase_      )
														trainer.save_model()
														trainer.log_metrics("""train"""       ,       train_result.metrics      )
														trainer.save_metrics("""train"""       ,       train_result.metrics      )
														trainer.save_state()

							# Evaluation
							if training_args.do_eval:
														__lowercase      :							Union[str, Any]				=     trainer.evaluate()
														trainer.log_metrics("""eval"""       ,       lowerCAmelCase_      )
														trainer.save_metrics("""eval"""       ,       lowerCAmelCase_      )

							# Write model card and (optionally) push to hub
							__lowercase      :							Optional[int]				=     {
							    """finetuned_from""": model_args.model_name_or_path,
							    """tasks""": """image-classification""",
							    """dataset""": data_args.dataset_name,
							    """tags""": ["""image-classification""", """vision"""],
							}
							if training_args.push_to_hub:
														trainer.push_to_hub(**lowerCAmelCase_      )
							else:
														trainer.create_model_card(**lowerCAmelCase_      )


if __name__ == "__main__":
	main()







from __future__ import annotations

import os
import tempfile
import unittest

from transformers import ConvBertConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
	import tensorflow as tf

	from transformers import (
	    TFConvBertForMaskedLM,
	    TFConvBertForMultipleChoice,
	    TFConvBertForQuestionAnswering,
	    TFConvBertForSequenceClassification,
	    TFConvBertForTokenClassification,
	    TFConvBertModel,
	)

class 		lowerCAmelCase     :

					'''simple docstring'''

					def __init__(  self				:	Union[str, Any]		,     __a				:	Dict		,     __a				:	Union[str, Any]=13		,     __a				:	Dict=7		,     __a				:	Dict=True		,     __a				:	Dict=True		,     __a				:	Any=True		,     __a				:	List[str]=True		,     __a				:	int=99		,     __a				:	Optional[int]=32		,     __a				:	str=2		,     __a				:	int=4		,     __a				:	List[str]=37		,     __a				:	Union[str, Any]="gelu"		,     __a				:	Union[str, Any]=0.1		,     __a				:	Union[str, Any]=0.1		,     __a				:	List[Any]=512		,     __a				:	int=16		,     __a				:	Union[str, Any]=2		,     __a				:	Union[str, Any]=0.02		,     __a				:	List[str]=3		,     __a				:	Dict=4		,     __a				:	Optional[Any]=None		,     )			->  Optional[Any]:


												"""simple docstring"""

												__lowercase      :							Any				=     parent
												__lowercase      :							Tuple				=     13
												__lowercase      :							Dict				=     7
												__lowercase      :							List[Any]				=     True
												__lowercase      :							Tuple				=     True
												__lowercase      :							List[str]				=     True
												__lowercase      :							Any				=     True
												__lowercase      :							Optional[int]				=     99
												__lowercase      :							str				=     384
												__lowercase      :							Optional[Any]				=     2
												__lowercase      :							Dict				=     4
												__lowercase      :							str				=     37
												__lowercase      :							Optional[int]				=     """gelu"""
												__lowercase      :							int				=     0.1
												__lowercase      :							Union[str, Any]				=     0.1
												__lowercase      :							Tuple				=     512
												__lowercase      :							Tuple				=     16
												__lowercase      :							Optional[int]				=     2
												__lowercase      :							Optional[Any]				=     0.02
												__lowercase      :							Dict				=     3
												__lowercase      :							Union[str, Any]				=     4
												__lowercase      :							Tuple				=     128
												__lowercase      :							Optional[Any]				=     2
												__lowercase      :							int				=     9
												__lowercase      :							List[Any]				=     1
												__lowercase      :							Union[str, Any]				=     None



					def       lowerCAmelCase				(  self				:	str							)			->  List[Any]:


												"""simple docstring"""

												__lowercase      :							Any				=     ids_tensor([self.batch_size, self.seq_length]		,     self.vocab_size							)

												__lowercase      :							Optional[Any]				=     None
												if self.use_input_mask:
																			__lowercase      :							Dict				=     random_attention_mask([self.batch_size, self.seq_length]							)

												__lowercase      :							Dict				=     None
												if self.use_token_type_ids:
																			__lowercase      :							Dict				=     ids_tensor([self.batch_size, self.seq_length]		,     self.type_vocab_size							)

												__lowercase      :							Optional[Any]				=     None
												__lowercase      :							str				=     None
												__lowercase      :							Tuple				=     None
												if self.use_labels:
																			__lowercase      :							List[str]				=     ids_tensor([self.batch_size]		,     self.type_sequence_label_size							)
																			__lowercase      :							List[Any]				=     ids_tensor([self.batch_size, self.seq_length]		,     self.num_labels							)
																			__lowercase      :							str				=     ids_tensor([self.batch_size]		,     self.num_choices							)

												__lowercase      :							Optional[int]				=     ConvBertConfig(
												    vocab_size=self.vocab_size		,     hidden_size=self.hidden_size		,     num_hidden_layers=self.num_hidden_layers		,     num_attention_heads=self.num_attention_heads		,     intermediate_size=self.intermediate_size		,     hidden_act=self.hidden_act		,     hidden_dropout_prob=self.hidden_dropout_prob		,     attention_probs_dropout_prob=self.attention_probs_dropout_prob		,     max_position_embeddings=self.max_position_embeddings		,     type_vocab_size=self.type_vocab_size		,     initializer_range=self.initializer_range		,     return_dict=__a		,     )

												return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



					def       lowerCAmelCase				(  self				:	Dict		,     __a				:	List[Any]		,     __a				:	List[str]		,     __a				:	Union[str, Any]		,     __a				:	str		,     __a				:	Union[str, Any]		,     __a				:	Tuple		,     __a				:	Tuple							)			->  Dict:


												"""simple docstring"""

												__lowercase      :							Dict				=     TFConvBertModel(config=__a							)
												__lowercase      :							Tuple				=     {"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}

												__lowercase      :							Any				=     [input_ids, input_mask]
												__lowercase      :							Dict				=     model(__a							)

												__lowercase      :							str				=     model(__a							)

												self.parent.assertEqual(result.last_hidden_state.shape		,     (self.batch_size, self.seq_length, self.hidden_size)							)



					def       lowerCAmelCase				(  self				:	Tuple		,     __a				:	Union[str, Any]		,     __a				:	Any		,     __a				:	Tuple		,     __a				:	Union[str, Any]		,     __a				:	str		,     __a				:	Dict		,     __a				:	str							)			->  Dict:


												"""simple docstring"""

												__lowercase      :							Optional[int]				=     TFConvBertForMaskedLM(config=__a							)
												__lowercase      :							List[Any]				=     {
												    """input_ids""": input_ids,
												    """attention_mask""": input_mask,
												    """token_type_ids""": token_type_ids,
												}
												__lowercase      :							Any				=     model(__a							)
												self.parent.assertEqual(result.logits.shape		,     (self.batch_size, self.seq_length, self.vocab_size)							)



					def       lowerCAmelCase				(  self				:	Optional[int]		,     __a				:	int		,     __a				:	Any		,     __a				:	Optional[int]		,     __a				:	int		,     __a				:	int		,     __a				:	List[Any]		,     __a				:	Optional[int]							)			->  List[Any]:


												"""simple docstring"""

												__lowercase      :							str				=     self.num_labels
												__lowercase      :							List[Any]				=     TFConvBertForSequenceClassification(config=__a							)
												__lowercase      :							int				=     {
												    """input_ids""": input_ids,
												    """attention_mask""": input_mask,
												    """token_type_ids""": token_type_ids,
												}

												__lowercase      :							List[str]				=     model(__a							)
												self.parent.assertEqual(result.logits.shape		,     (self.batch_size, self.num_labels)							)



					def       lowerCAmelCase				(  self				:	Optional[int]		,     __a				:	Any		,     __a				:	Optional[Any]		,     __a				:	int		,     __a				:	Optional[int]		,     __a				:	Tuple		,     __a				:	int		,     __a				:	int							)			->  Dict:


												"""simple docstring"""

												__lowercase      :							Tuple				=     self.num_choices
												__lowercase      :							Dict				=     TFConvBertForMultipleChoice(config=__a							)
												__lowercase      :							List[str]				=     tf.tile(tf.expand_dims(__a		,     1							)		,     (1, self.num_choices, 1)							)
												__lowercase      :							int				=     tf.tile(tf.expand_dims(__a		,     1							)		,     (1, self.num_choices, 1)							)
												__lowercase      :							str				=     tf.tile(tf.expand_dims(__a		,     1							)		,     (1, self.num_choices, 1)							)
												__lowercase      :							str				=     {
												    """input_ids""": multiple_choice_inputs_ids,
												    """attention_mask""": multiple_choice_input_mask,
												    """token_type_ids""": multiple_choice_token_type_ids,
												}
												__lowercase      :							Dict				=     model(__a							)
												self.parent.assertEqual(result.logits.shape		,     (self.batch_size, self.num_choices)							)



					def       lowerCAmelCase				(  self				:	List[str]		,     __a				:	str		,     __a				:	List[str]		,     __a				:	List[str]		,     __a				:	List[str]		,     __a				:	Any		,     __a				:	Tuple		,     __a				:	Union[str, Any]							)			->  Optional[int]:


												"""simple docstring"""

												__lowercase      :							Tuple				=     self.num_labels
												__lowercase      :							Tuple				=     TFConvBertForTokenClassification(config=__a							)
												__lowercase      :							Dict				=     {
												    """input_ids""": input_ids,
												    """attention_mask""": input_mask,
												    """token_type_ids""": token_type_ids,
												}
												__lowercase      :							str				=     model(__a							)
												self.parent.assertEqual(result.logits.shape		,     (self.batch_size, self.seq_length, self.num_labels)							)



					def       lowerCAmelCase				(  self				:	List[Any]		,     __a				:	Optional[int]		,     __a				:	List[str]		,     __a				:	Optional[Any]		,     __a				:	int		,     __a				:	Tuple		,     __a				:	Any		,     __a				:	Union[str, Any]							)			->  List[Any]:


												"""simple docstring"""

												__lowercase      :							Any				=     TFConvBertForQuestionAnswering(config=__a							)
												__lowercase      :							str				=     {
												    """input_ids""": input_ids,
												    """attention_mask""": input_mask,
												    """token_type_ids""": token_type_ids,
												}

												__lowercase      :							List[Any]				=     model(__a							)
												self.parent.assertEqual(result.start_logits.shape		,     (self.batch_size, self.seq_length)							)
												self.parent.assertEqual(result.end_logits.shape		,     (self.batch_size, self.seq_length)							)



					def       lowerCAmelCase				(  self				:	Tuple							)			->  Tuple:


												"""simple docstring"""

												__lowercase      :							Tuple				=     self.prepare_config_and_inputs()
												(
												    (
												    __lowercase
												)			,			(
												    __lowercase
												)			,			(
												    __lowercase
												)			,			(
												    __lowercase
												)			,			(
												    __lowercase
												)			,			(
												    __lowercase
												)			,			(
												    __lowercase
												)			,			
												)      :							int				=     config_and_inputs
												__lowercase      :							Union[str, Any]				=     {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
												return config, inputs_dict







@require_tf
class 		lowerCAmelCase     ( __a      ,       __a      ,       unittest.TestCase			):

					'''simple docstring'''

					_A						:			Dict									=      (
					    (
					        TFConvBertModel,
					        TFConvBertForMaskedLM,
					        TFConvBertForQuestionAnswering,
					        TFConvBertForSequenceClassification,
					        TFConvBertForTokenClassification,
					        TFConvBertForMultipleChoice,
					    )
					    if is_tf_available()
					    else ()
					)
					_A						:			str									=      (
					    {
					        '''feature-extraction''': TFConvBertModel,
					        '''fill-mask''': TFConvBertForMaskedLM,
					        '''question-answering''': TFConvBertForQuestionAnswering,
					        '''text-classification''': TFConvBertForSequenceClassification,
					        '''token-classification''': TFConvBertForTokenClassification,
					        '''zero-shot''': TFConvBertForSequenceClassification,
					    }
					    if is_tf_available()
					    else {}
					)
					_A						:			Union[str, Any]									=      False
					_A						:			List[str]									=      False
					_A						:			Dict									=      False



					def       lowerCAmelCase				(  self				:	List[str]							)			->  int:


												"""simple docstring"""

												__lowercase      :							int				=     TFConvBertModelTester(self							)
												__lowercase      :							Tuple				=     ConfigTester(self		,     config_class=__a		,     hidden_size=37							)



					def       lowerCAmelCase				(  self				:	Optional[int]							)			->  Optional[Any]:


												"""simple docstring"""

												self.config_tester.run_common_tests()



					def       lowerCAmelCase				(  self				:	Optional[Any]							)			->  Union[str, Any]:


												"""simple docstring"""

												__lowercase      :							Optional[Any]				=     self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_model(*__a							)



					def       lowerCAmelCase				(  self				:	Dict							)			->  Union[str, Any]:


												"""simple docstring"""

												__lowercase      :							Union[str, Any]				=     self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_masked_lm(*__a							)



					def       lowerCAmelCase				(  self				:	Union[str, Any]							)			->  Union[str, Any]:


												"""simple docstring"""

												__lowercase      :							int				=     self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_multiple_choice(*__a							)



					def       lowerCAmelCase				(  self				:	str							)			->  Any:


												"""simple docstring"""

												__lowercase      :							List[str]				=     self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_question_answering(*__a							)



					def       lowerCAmelCase				(  self				:	str							)			->  str:


												"""simple docstring"""

												__lowercase      :							List[Any]				=     self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_sequence_classification(*__a							)



					def       lowerCAmelCase				(  self				:	str							)			->  Any:


												"""simple docstring"""

												__lowercase      :							Optional[int]				=     self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_token_classification(*__a							)



					@slow
					def       lowerCAmelCase				(  self				:	str							)			->  Any:


												"""simple docstring"""

												__lowercase			,			__lowercase      :							Optional[int]				=     self.model_tester.prepare_config_and_inputs_for_common()
												__lowercase      :							Union[str, Any]				=     True
												__lowercase      :							List[Any]				=     True

												if hasattr(__a		,     """use_cache"""							):
																			__lowercase      :							Optional[Any]				=     True

												__lowercase      :							List[str]				=     getattr(self.model_tester		,     """encoder_seq_length"""		,     self.model_tester.seq_length							)
												__lowercase      :							int				=     getattr(self.model_tester		,     """key_length"""		,     __a							)

												for model_class in self.all_model_classes:
																			__lowercase      :							Optional[Any]				=     self._prepare_for_class(__a		,     __a							)
																			__lowercase      :							Tuple				=     model_class(__a							)
																			__lowercase      :							Tuple				=     len(model(__a							)							)

																			with tempfile.TemporaryDirectory() as tmpdirname:
																										model.save_pretrained(__a		,     saved_model=__a							)
																										__lowercase      :							List[Any]				=     os.path.join(__a		,     """saved_model"""		,     """1"""							)
																										__lowercase      :							str				=     tf.keras.models.load_model(__a							)
																										__lowercase      :							Optional[int]				=     model(__a							)

																										if self.is_encoder_decoder:
																																	__lowercase      :							Union[str, Any]				=     outputs["""encoder_hidden_states"""]
																																	__lowercase      :							Union[str, Any]				=     outputs["""encoder_attentions"""]
																										else:
																																	__lowercase      :							Union[str, Any]				=     outputs["""hidden_states"""]
																																	__lowercase      :							List[str]				=     outputs["""attentions"""]

																										self.assertEqual(len(__a							)		,     __a							)

																										__lowercase      :							List[Any]				=     getattr(
																										    self.model_tester		,     """expected_num_hidden_layers"""		,     self.model_tester.num_hidden_layers + 1							)

																										self.assertEqual(len(__a							)		,     __a							)
																										self.assertListEqual(
																										    list(output_hidden_states[0].shape[-2:]							)		,     [self.model_tester.seq_length, self.model_tester.hidden_size]		,     )

																										self.assertEqual(len(__a							)		,     self.model_tester.num_hidden_layers							)
																										self.assertListEqual(
																										    list(output_attentions[0].shape[-3:]							)		,     [self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]		,     )



					@slow
					def       lowerCAmelCase				(  self				:	Union[str, Any]							)			->  List[Any]:


												"""simple docstring"""

												__lowercase      :							str				=     TFConvBertModel.from_pretrained("""YituTech/conv-bert-base"""							)
												self.assertIsNotNone(__a							)



					def       lowerCAmelCase				(  self				:	Any							)			->  List[Any]:


												"""simple docstring"""

												__lowercase			,			__lowercase      :							int				=     self.model_tester.prepare_config_and_inputs_for_common()
												__lowercase      :							List[str]				=     True
												__lowercase      :							List[Any]				=     getattr(self.model_tester		,     """decoder_seq_length"""		,     self.model_tester.seq_length							)
												__lowercase      :							Optional[int]				=     getattr(self.model_tester		,     """encoder_seq_length"""		,     self.model_tester.seq_length							)
												__lowercase      :							List[str]				=     getattr(self.model_tester		,     """key_length"""		,     __a							)
												__lowercase      :							List[Any]				=     getattr(self.model_tester		,     """key_length"""		,     __a							)

												def check_decoder_attentions_output(__a				:	List[str]							):
																			__lowercase      :							Union[str, Any]				=     len(__a							)
																			self.assertEqual(out_len % 2		,     0							)
																			__lowercase      :							Any				=     outputs.decoder_attentions
																			self.assertEqual(len(__a							)		,     self.model_tester.num_hidden_layers							)
																			self.assertListEqual(
																			    list(decoder_attentions[0].shape[-3:]							)		,     [self.model_tester.num_attention_heads / 2, decoder_seq_length, decoder_key_length]		,     )

												def check_encoder_attentions_output(__a				:	str							):
																			__lowercase      :							str				=     [
																			    t.numpy() for t in (outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions)
																			]
																			self.assertEqual(len(__a							)		,     self.model_tester.num_hidden_layers							)
																			self.assertListEqual(
																			    list(attentions[0].shape[-3:]							)		,     [self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]		,     )

												for model_class in self.all_model_classes:
																			__lowercase      :							int				=     True
																			__lowercase      :							Any				=     False
																			__lowercase      :							List[Any]				=     model_class(__a							)
																			__lowercase      :							Tuple				=     model(self._prepare_for_class(__a		,     __a							)							)
																			__lowercase      :							Dict				=     len(__a							)
																			self.assertEqual(config.output_hidden_states		,     __a							)
																			check_encoder_attentions_output(__a							)

																			if self.is_encoder_decoder:
																										__lowercase      :							Any				=     model_class(__a							)
																										__lowercase      :							List[str]				=     model(self._prepare_for_class(__a		,     __a							)							)
																										self.assertEqual(config.output_hidden_states		,     __a							)
																										check_decoder_attentions_output(__a							)

																			# Check that output attentions can also be changed via the config
																			del inputs_dict["output_attentions"]
																			__lowercase      :							Dict				=     True
																			__lowercase      :							Optional[Any]				=     model_class(__a							)
																			__lowercase      :							Optional[int]				=     model(self._prepare_for_class(__a		,     __a							)							)
																			self.assertEqual(config.output_hidden_states		,     __a							)
																			check_encoder_attentions_output(__a							)

																			# Check attention is always last and order is fine
																			__lowercase      :							List[str]				=     True
																			__lowercase      :							List[Any]				=     True
																			__lowercase      :							Any				=     model_class(__a							)
																			__lowercase      :							Optional[int]				=     model(self._prepare_for_class(__a		,     __a							)							)

																			self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1)		,     len(__a							)							)
																			self.assertEqual(model.config.output_hidden_states		,     __a							)
																			check_encoder_attentions_output(__a							)







@require_tf
class 		lowerCAmelCase     ( unittest.TestCase			):

					'''simple docstring'''

					@slow
					def       lowerCAmelCase				(  self				:	Dict							)			->  Union[str, Any]:


												"""simple docstring"""

												__lowercase      :							List[str]				=     TFConvBertModel.from_pretrained("""YituTech/conv-bert-base"""							)
												__lowercase      :							str				=     tf.constant([[0, 1, 2, 3, 4, 5]]							)
												__lowercase      :							Tuple				=     model(__a							)[0]

												__lowercase      :							Any				=     [1, 6, 768]
												self.assertEqual(output.shape		,     __a							)

												__lowercase      :							Optional[Any]				=     tf.constant(
												    [
												        [
												            [-0.03475493, -0.4686034, -0.30638832],
												            [0.22637248, -0.26988646, -0.7423424],
												            [0.10324868, -0.45013508, -0.58280784],
												        ]
												    ]							)
												tf.debugging.assert_near(output[:, :3, :3]		,     __a		,     atol=1E-4							)