






'''simple docstring'''







from __future__ import annotations

import unittest

import numpy as np

from transformers import OPTConfig, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
		import tensorflow as tf

		from transformers import GPTaTokenizer, TFOPTForCausalLM, TFOPTModel





def 		_UpperCAmelCase   (							__A							:   Optional[int]						,		__A							:   Dict						,		__A							:   List[Any]=None						,		__A							:   str=None      ):







				if attention_mask is None:
								a_		:  Any											=      tf.cast(tf.math.not_equal(lowercase__						,		config.pad_token_id      )						,		tf.inta      )
				return {"input_ids": input_ids, "attention_mask": attention_mask}





@require_tf
class 	SCREAMING_SNAKE_CASE					:
			snake_case__  =				OPTConfig
			snake_case__  =				{}
			snake_case__  =				"gelu"






			def __init__( self      :			Dict  ,						__SCREAMING_SNAKE_CASE      :			Optional[int]  ,						__SCREAMING_SNAKE_CASE      :			Union[str, Any]=13  ,						__SCREAMING_SNAKE_CASE      :			List[Any]=7  ,						__SCREAMING_SNAKE_CASE      :			List[Any]=True  ,						__SCREAMING_SNAKE_CASE      :			Any=False  ,						__SCREAMING_SNAKE_CASE      :			Any=99  ,						__SCREAMING_SNAKE_CASE      :			Optional[Any]=16  ,						__SCREAMING_SNAKE_CASE      :			Tuple=2  ,						__SCREAMING_SNAKE_CASE      :			Any=4  ,						__SCREAMING_SNAKE_CASE      :			Any=4  ,						__SCREAMING_SNAKE_CASE      :			Dict="gelu"  ,						__SCREAMING_SNAKE_CASE      :			Dict=0.1  ,						__SCREAMING_SNAKE_CASE      :			Dict=0.1  ,						__SCREAMING_SNAKE_CASE      :			Any=20  ,						__SCREAMING_SNAKE_CASE      :			int=2  ,						__SCREAMING_SNAKE_CASE      :			str=1  ,						__SCREAMING_SNAKE_CASE      :			List[Any]=0  ,						__SCREAMING_SNAKE_CASE      :			Any=16  ,						__SCREAMING_SNAKE_CASE      :			Any=16  ,						)			->			Optional[int]:
							a_		:  Union[str, Any]											=      parent
							a_		:  Dict											=      batch_size
							a_		:  str											=      seq_length
							a_		:  Dict											=      is_training
							a_		:  Dict											=      use_labels
							a_		:  List[Any]											=      vocab_size
							a_		:  Dict											=      hidden_size
							a_		:  int											=      num_hidden_layers
							a_		:  str											=      num_attention_heads
							a_		:  Dict											=      intermediate_size
							a_		:  Tuple											=      hidden_act
							a_		:  str											=      hidden_dropout_prob
							a_		:  str											=      attention_probs_dropout_prob
							a_		:  str											=      max_position_embeddings
							a_		:  Dict											=      eos_token_id
							a_		:  int											=      pad_token_id
							a_		:  str											=      bos_token_id
							a_		:  Optional[Any]											=      embed_dim
							a_		:  List[str]											=      word_embed_proj_dim
							a_		:  Any											=      False






			def 			SCREAMING_SNAKE_CASE      ( self      :			List[str]					)			->			Any:
							a_		:  List[Any]											=      ids_tensor([self.batch_size, self.seq_length - 1]  ,						self.vocab_size					)
							a_		:  Optional[int]											=      tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size					)  ,						1					)
							a_		:  Any											=      tf.concat([input_ids, eos_tensor]  ,						axis=1					)

							a_		:  int											=      self.config_cls(
							    vocab_size=self.vocab_size  ,						hidden_size=self.hidden_size  ,						num_hidden_layers=self.num_hidden_layers  ,						num_attention_heads=self.num_attention_heads  ,						ffn_dim=self.intermediate_size  ,						dropout=self.hidden_dropout_prob  ,						attention_dropout=self.attention_probs_dropout_prob  ,						max_position_embeddings=self.max_position_embeddings  ,						eos_token_id=self.eos_token_id  ,						bos_token_id=self.bos_token_id  ,						pad_token_id=self.pad_token_id  ,						embed_dim=self.embed_dim  ,						word_embed_proj_dim=self.word_embed_proj_dim  ,						is_encoder_decoder=lowerCAmelCase_  ,						**self.config_updates  ,						)
							a_		:  int											=      prepare_opt_inputs_dict(lowerCAmelCase_  ,						lowerCAmelCase_					)
							return config, inputs_dict







			def 			SCREAMING_SNAKE_CASE      ( self      :			Any  ,						__SCREAMING_SNAKE_CASE      :			Optional[int]  ,						__SCREAMING_SNAKE_CASE      :			Optional[Any]					)			->			Dict:
							a_		:  int											=      TFOPTModel(config=lowerCAmelCase_					)
							a_		:  Dict											=      inputs_dict['''input_ids''']

							a_		:  Dict											=      input_ids[:1, :]
							a_		:  Optional[int]											=      inputs_dict['''attention_mask'''][:1, :]
							a_		:  Dict											=      1

							# first forward pass
							a_		:  List[str]											=      model(lowerCAmelCase_  ,						attention_mask=lowerCAmelCase_  ,						use_cache=lowerCAmelCase_					)

							a_				,			a_		:  Optional[int]											=      outputs.to_tuple()

							# create hypothetical next token and extent to next_input_ids
							a_		:  Optional[Any]											=      ids_tensor((self.batch_size, 3)  ,						config.vocab_size					)
							a_		:  int											=      tf.cast(ids_tensor((self.batch_size, 3)  ,						2					)  ,						tf.inta					)

							# append to next input_ids and
							a_		:  Union[str, Any]											=      tf.concat([input_ids, next_tokens]  ,						axis=-1					)
							a_		:  str											=      tf.concat([attention_mask, next_attn_mask]  ,						axis=-1					)

							a_		:  List[str]											=      model(lowerCAmelCase_  ,						attention_mask=lowerCAmelCase_					)[0]
							a_		:  Dict											=      model(lowerCAmelCase_  ,						attention_mask=lowerCAmelCase_  ,						past_key_values=lowerCAmelCase_					)[0]

							self.parent.assertEqual(next_tokens.shape[1]  ,						output_from_past.shape[1]					)

							# select random slice
							a_		:  str											=      int(ids_tensor((1,)  ,						output_from_past.shape[-1]					)					)
							a_		:  Optional[Any]											=      output_from_no_past[:, -3:, random_slice_idx]
							a_		:  int											=      output_from_past[:, :, random_slice_idx]

							# test that outputs are equal for slice
							tf.debugging.assert_near(lowerCAmelCase_  ,						lowerCAmelCase_  ,						rtol=1e-3					)





@require_tf
class 	SCREAMING_SNAKE_CASE					(					__a	,   __a	,   unittest.TestCase       ):
			snake_case__  =				(TFOPTModel, TFOPTForCausalLM) if is_tf_available() else ()
			snake_case__  =				(TFOPTForCausalLM,) if is_tf_available() else ()
			snake_case__  =				(
			    {"feature-extraction": TFOPTModel, "text-generation": TFOPTForCausalLM} if is_tf_available() else {}
			)
			snake_case__  =				False
			snake_case__  =				False
			snake_case__  =				False
			snake_case__  =				10






			def 			SCREAMING_SNAKE_CASE      ( self      :			List[Any]					)			->			Tuple:
							a_		:  List[str]											=      TFOPTModelTester(self					)
							a_		:  Optional[Any]											=      ConfigTester(self  ,						config_class=lowerCAmelCase_					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Any					)			->			Tuple:
							self.config_tester.run_common_tests()






			def 			SCREAMING_SNAKE_CASE      ( self      :			Dict					)			->			List[str]:
							a_		:  Any											=      self.model_tester.prepare_config_and_inputs_for_common()
							self.model_tester.check_decoder_model_past_large_inputs(*lowerCAmelCase_					)







			def 			SCREAMING_SNAKE_CASE      ( self      :			Tuple					)			->			Union[str, Any]:
							a_				,			a_		:  Optional[Any]											=      self.model_tester.prepare_config_and_inputs_for_common()

							def _get_word_embedding_weight(__SCREAMING_SNAKE_CASE      :			Tuple  ,						__SCREAMING_SNAKE_CASE      :			str					):
											if hasattr(lowerCAmelCase_  ,						'''weight'''					):
															return embedding_layer.weight
											else:
															# Here we build the word embeddings weights if not exists.
															# And then we retry to get the attribute once built.
															model.build()
															if hasattr(lowerCAmelCase_  ,						'''weight'''					):
																			return embedding_layer.weight
															else:
																			return None

							for model_class in self.all_model_classes:
											for size in [config.vocab_size - 10, config.vocab_size + 10]:
															# build the embeddings
															a_		:  Optional[int]											=      model_class(config=lowerCAmelCase_					)
															a_		:  List[str]											=      _get_word_embedding_weight(lowerCAmelCase_  ,						model.get_input_embeddings()					)
															a_		:  Dict											=      _get_word_embedding_weight(lowerCAmelCase_  ,						model.get_output_embeddings()					)

															# reshape the embeddings
															model.resize_token_embeddings(lowerCAmelCase_					)
															a_		:  List[Any]											=      _get_word_embedding_weight(lowerCAmelCase_  ,						model.get_input_embeddings()					)
															a_		:  List[Any]											=      _get_word_embedding_weight(lowerCAmelCase_  ,						model.get_output_embeddings()					)

															# check that the resized embeddings size matches the desired size.
															a_		:  int											=      size if size is not None else config.vocab_size

															self.assertEqual(new_input_embeddings.shape[0]  ,						lowerCAmelCase_					)

															# check that weights remain the same after resizing
															a_		:  str											=      True
															for pa, pa in zip(old_input_embeddings.value()  ,						new_input_embeddings.value()					):
																			if tf.math.reduce_sum(tf.math.abs(pa - pa					)					) > 0:
																							a_		:  int											=      False
															self.assertTrue(lowerCAmelCase_					)

															if old_output_embeddings is not None and new_output_embeddings is not None:
																			self.assertEqual(new_output_embeddings.shape[0]  ,						lowerCAmelCase_					)

																			a_		:  Optional[Any]											=      True
																			for pa, pa in zip(old_output_embeddings.value()  ,						new_output_embeddings.value()					):
																							if tf.math.reduce_sum(tf.math.abs(pa - pa					)					) > 0:
																											a_		:  int											=      False
																			self.assertTrue(lowerCAmelCase_					)





def 		_UpperCAmelCase   (							__A							:   Any      ):
				return tf.constant(lowercase__						,		dtype=tf.intaa      )





@require_tf
class 	SCREAMING_SNAKE_CASE					(					unittest.TestCase       ):
			snake_case__  =				99






			def 			SCREAMING_SNAKE_CASE      ( self      :			Tuple					)			->			Optional[int]:
							a_		:  Union[str, Any]											=      tf.ones((4, 1)  ,						dtype=tf.intaa					) * 2
							a_		:  List[Any]											=      tf.concat([ids_tensor((4, 6)  ,						self.vocab_size - 3					) + 3, eos_column_vector]  ,						axis=1					)
							a_		:  Optional[Any]											=      input_ids.shape[0]
							a_		:  Any											=      OPTConfig(
							    vocab_size=self.vocab_size  ,						hidden_size=24  ,						num_hidden_layers=2  ,						num_attention_heads=2  ,						ffn_dim=32  ,						max_position_embeddings=48  ,						eos_token_id=2  ,						pad_token_id=1  ,						bos_token_id=0  ,						)
							return config, input_ids, batch_size





@require_sentencepiece
@require_tf
class 	SCREAMING_SNAKE_CASE					(					unittest.TestCase       ):






			@slow
			def 			SCREAMING_SNAKE_CASE      ( self      :			str					)			->			str:
							a_		:  Any											=      TFOPTModel.from_pretrained('''facebook/opt-350m'''					)
							a_		:  Union[str, Any]											=      _long_tensor([[0, 3_1414, 232, 328, 740, 1140, 1_2695, 69, 4_6078, 1588, 2]]					)
							a_		:  int											=      tf.not_equal(lowerCAmelCase_  ,						model.config.pad_token_id					)
							with tf.GradientTape():
											a_		:  List[str]											=      model(input_ids=lowerCAmelCase_  ,						attention_mask=lowerCAmelCase_					).last_hidden_state
							a_		:  Optional[int]											=      (1, 11, 512)
							self.assertEqual(output.shape  ,						lowerCAmelCase_					)
							a_		:  List[Any]											=      tf.constant(
							    [[-0.2873, -1.9218, -0.3033], [-1.2710, -0.1338, -0.1902], [0.4095, 0.1214, -1.3121]]					)
							self.assertTrue(np.allclose(output[:, :3, :3]  ,						lowerCAmelCase_  ,						atol=4e-3					)					)

							a_		:  List[str]											=      tf.function(lowerCAmelCase_  ,						jit_compile=lowerCAmelCase_					)
							a_		:  Union[str, Any]											=      xla_generate(lowerCAmelCase_  ,						lowerCAmelCase_					)[0]
							self.assertTrue(np.allclose(output[:, :3, :3]  ,						lowerCAmelCase_  ,						atol=4e-2					)					)





@require_tf
@slow
class 	SCREAMING_SNAKE_CASE					(					unittest.TestCase       ):






			def 			SCREAMING_SNAKE_CASE      ( self      :			Union[str, Any]					)			->			Optional[Any]:
							super().setUp()
							a_		:  str											=      '''facebook/opt-350m'''







			def 			SCREAMING_SNAKE_CASE      ( self      :			Union[str, Any]					)			->			Union[str, Any]:
							a_		:  Optional[int]											=      TFOPTForCausalLM.from_pretrained(self.path_model					)
							a_		:  List[str]											=      GPTaTokenizer.from_pretrained(self.path_model					)

							a_		:  Union[str, Any]											=      [
							    '''Today is a beautiful day and I want to''',
							    '''In the city of''',
							    '''Paris is the capital of France and''',
							    '''Computers and mobile phones have taken''',
							]
							# verify that prompt without BOS token is identical to Metaseq -> add_special_tokens=False
							a_		:  str											=      tokenizer(lowerCAmelCase_  ,						return_tensors='''tf'''  ,						padding=lowerCAmelCase_  ,						add_special_tokens=lowerCAmelCase_					)
							a_		:  Any											=      tf.math.reduce_mean(model(inputs.input_ids  ,						attention_mask=inputs.attention_mask					)[0]  ,						axis=-1					)
							a_		:  Optional[int]											=      tf.constant(
							    [
							        [1.3851, -13.8923, -10.5229, -10.7533, -0.2309, -10.2384, -0.5365, -9.0947, -5.1670],
							        [-4.7073, -10.6276, -3.9415, -21.5242, -0.2822, -0.2822, -0.2822, -0.2822, -0.2822],
							        [0.6247, -3.4229, -8.9179, -1.4297, -14.1650, 1.4146, -9.0218, -0.2703, -0.2703],
							        [6.4783, -1.9913, -10.7926, -2.3336, 1.5092, -0.9974, -6.8213, 1.3477, 1.3477],
							    ]					)
							self.assertTrue(np.allclose(lowerCAmelCase_  ,						lowerCAmelCase_  ,						atol=1e-4					)					)

							a_		:  str											=      tf.function(lowerCAmelCase_  ,						jit_compile=lowerCAmelCase_					)
							a_		:  Any											=      tf.math.reduce_mean(xla_generate(inputs.input_ids  ,						attention_mask=inputs.attention_mask					)[0]  ,						axis=-1					)
							self.assertTrue(np.allclose(lowerCAmelCase_  ,						lowerCAmelCase_  ,						atol=1e-4					)					)





@require_tf
@slow
class 	SCREAMING_SNAKE_CASE					(					unittest.TestCase       ):






			@property
			def 			SCREAMING_SNAKE_CASE      ( self      :			Dict					)			->			int:
							return [
							    "Today is a beautiful day and I want",
							    "In the city of",
							    "Paris is the capital of France and",
							    "Computers and mobile phones have taken",
							]






			def 			SCREAMING_SNAKE_CASE      ( self      :			Tuple					)			->			Any:
							a_		:  Dict											=      '''facebook/opt-125m'''

							a_		:  Tuple											=      [
							    '''Today is a beautiful day and I want to''',
							    '''In the city of New York, the city''',
							    '''Paris is the capital of France and the capital''',
							    '''Computers and mobile phones have taken over the''',
							]

							a_		:  Any											=      []
							a_		:  List[str]											=      GPTaTokenizer.from_pretrained(lowerCAmelCase_					)
							a_		:  List[str]											=      TFOPTForCausalLM.from_pretrained(lowerCAmelCase_					)

							for prompt in self.prompts:
											a_		:  List[str]											=      tokenizer(lowerCAmelCase_  ,						return_tensors='''tf'''					).input_ids

											a_		:  str											=      model.generate(lowerCAmelCase_  ,						max_length=10					)

											a_		:  List[Any]											=      tokenizer.batch_decode(lowerCAmelCase_  ,						skip_special_tokens=lowerCAmelCase_					)
											predicted_outputs += generated_string

							self.assertListEqual(lowerCAmelCase_  ,						lowerCAmelCase_					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Dict					)			->			Union[str, Any]:
							a_		:  Dict											=      '''facebook/opt-350m'''

							a_		:  Tuple											=      GPTaTokenizer.from_pretrained(lowerCAmelCase_					)
							a_		:  Tuple											=      TFOPTForCausalLM.from_pretrained(lowerCAmelCase_					)

							a_		:  List[Any]											=      '''left'''

							# use different length sentences to test batching
							a_		:  int											=      [
							    '''Hello, my dog is a little''',
							    '''Today, I''',
							]

							a_		:  Union[str, Any]											=      tokenizer(lowerCAmelCase_  ,						return_tensors='''tf'''  ,						padding=lowerCAmelCase_					)
							a_		:  Tuple											=      inputs['''input_ids''']

							a_		:  Any											=      model.generate(input_ids=lowerCAmelCase_  ,						attention_mask=inputs['''attention_mask''']					)

							a_		:  List[str]											=      tokenizer(sentences[0]  ,						return_tensors='''tf'''					).input_ids
							a_		:  List[str]											=      model.generate(input_ids=lowerCAmelCase_					)

							a_		:  List[str]											=      inputs_non_padded.shape[-1] - tf.math.reduce_sum(
							    tf.cast(inputs['''attention_mask'''][-1]  ,						tf.intaa					)					)
							a_		:  Union[str, Any]											=      tokenizer(sentences[1]  ,						return_tensors='''tf'''					).input_ids
							a_		:  Any											=      model.generate(input_ids=lowerCAmelCase_  ,						max_length=model.config.max_length - num_paddings					)

							a_		:  int											=      tokenizer.batch_decode(lowerCAmelCase_  ,						skip_special_tokens=lowerCAmelCase_					)
							a_		:  Union[str, Any]											=      tokenizer.decode(output_non_padded[0]  ,						skip_special_tokens=lowerCAmelCase_					)
							a_		:  Dict											=      tokenizer.decode(output_padded[0]  ,						skip_special_tokens=lowerCAmelCase_					)

							a_		:  Optional[int]											=      [
							    '''Hello, my dog is a little bit of a dork.\nI\'m a little bit''',
							    '''Today, I was in the middle of a conversation with a friend about the''',
							]
							self.assertListEqual(lowerCAmelCase_  ,						lowerCAmelCase_					)
							self.assertListEqual(lowerCAmelCase_  ,						[non_padded_sentence, padded_sentence]					)







			def 			SCREAMING_SNAKE_CASE      ( self      :			Tuple					)			->			List[str]:
							a_		:  int											=      '''facebook/opt-350m'''

							a_		:  Dict											=      [
							    '''Today is a beautiful day and I want to''',
							    '''In the city of San Francisco, the city''',
							    '''Paris is the capital of France and the capital''',
							    '''Computers and mobile phones have taken over the''',
							]

							a_		:  Optional[int]											=      []
							a_		:  List[Any]											=      GPTaTokenizer.from_pretrained(lowerCAmelCase_					)
							a_		:  int											=      TFOPTForCausalLM.from_pretrained(lowerCAmelCase_					)

							for prompt in self.prompts:
											a_		:  int											=      tokenizer(lowerCAmelCase_  ,						return_tensors='''tf'''					).input_ids

											a_		:  List[str]											=      model.generate(lowerCAmelCase_  ,						max_length=10					)

											a_		:  str											=      tokenizer.batch_decode(lowerCAmelCase_  ,						skip_special_tokens=lowerCAmelCase_					)
											predicted_outputs += generated_string

							self.assertListEqual(lowerCAmelCase_  ,						lowerCAmelCase_					)












'''simple docstring'''







from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Union

import pyarrow as pa


if TYPE_CHECKING:
		from .features import FeatureType





@dataclass
class 	SCREAMING_SNAKE_CASE					:
			snake_case__  =				42
			snake_case__  =				None
			# Automatically constructed
			snake_case__  =				"dict"
			snake_case__  =				None
			snake_case__  =				field(default="Translation"	,   init=SCREAMING_SNAKE_CASE_	,   repr=SCREAMING_SNAKE_CASE_       )






			def __call__( self      :			Dict					)			->			Tuple:
							return pa.struct({lang: pa.string() for lang in sorted(self.languages					)}					)







			def 			SCREAMING_SNAKE_CASE      ( self      :			Union[str, Any]					)			->			Union["FeatureType", Dict[str, "FeatureType"]]:
							from .features import Value

							return {k: Value('''string'''					) for k in sorted(self.languages					)}





@dataclass
class 	SCREAMING_SNAKE_CASE					:
			snake_case__  =				None
			snake_case__  =				None
			snake_case__  =				None
			# Automatically constructed
			snake_case__  =				"dict"
			snake_case__  =				None
			snake_case__  =				field(default="TranslationVariableLanguages"	,   init=SCREAMING_SNAKE_CASE_	,   repr=SCREAMING_SNAKE_CASE_       )






			def 			SCREAMING_SNAKE_CASE      ( self      :			Tuple					)			->			Dict:
							a_		:  List[str]											=      sorted(set(self.languages					)					) if self.languages else None
							a_		:  Optional[Any]											=      len(self.languages					) if self.languages else None






			def __call__( self      :			Any					)			->			Optional[Any]:
							return pa.struct({'''language''': pa.list_(pa.string()					), '''translation''': pa.list_(pa.string()					)}					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[int]  ,						__SCREAMING_SNAKE_CASE      :			Optional[Any]					)			->			Optional[Any]:
							a_		:  str											=      set(self.languages					)
							if self.languages and set(__SCREAMING_SNAKE_CASE					) - lang_set:
											raise ValueError(
											    f'Some languages in example ({", ".join(sorted(set(__SCREAMING_SNAKE_CASE					) - lang_set					)					)}) are not in valid set ({", ".join(__SCREAMING_SNAKE_CASE					)}).'					)

							# Convert dictionary into tuples, splitting out cases where there are
							# multiple translations for a single language.
							a_		:  int											=      []
							for lang, text in translation_dict.items():
											if isinstance(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					):
															translation_tuples.append((lang, text)					)
											else:
															translation_tuples.extend([(lang, el) for el in text]					)

        # Ensure translations are in ascending order by language code.
							a_				,			a_		:  List[Any]											=      zip(*sorted(__SCREAMING_SNAKE_CASE					)					)

							return {"language": languages, "translation": translations}







			def 			SCREAMING_SNAKE_CASE      ( self      :			Any					)			->			Union["FeatureType", Dict[str, "FeatureType"]]:
							from .features import Sequence, Value

							return {
							    "language": Sequence(Value('''string'''					)					),
							    "translation": Sequence(Value('''string'''					)					),
							}




