




import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase       :	Dict									=     get_tests_dir("fixtures/test_sentencepiece_with_bytefallback.model")







@require_sentencepiece
@require_tokenizers
class 						A(		UpperCamelCase							,   unittest.TestCase     ):



 '''simple docstring'''


 UpperCamelCase		     =		GPTSwaTokenizer
 UpperCamelCase		     =		False
 UpperCamelCase		     =		True
 UpperCamelCase		     =		False

 def        a__				(       self      :						Optional[int]						)     ->			Any:

       """simple docstring"""






       super().setUp()

       # We have a SentencePiece fixture for testing
       lowerCamelCase_	  =					GPTSwaTokenizer(A_		,							eos_token='<unk>'		,							bos_token='<unk>'		,							pad_token='<unk>'						)

       tokenizer.save_pretrained(self.tmpdirname						)

 def        a__				(       self      :						str		,							A_      :						Union[str, Any]						)     ->			str:

       """simple docstring"""






       lowerCamelCase_	  =					'This is a test'
       lowerCamelCase_	  =					'This is a test'
       return input_text, output_text

 def        a__				(       self      :						Union[str, Any]						)     ->			Any:

       """simple docstring"""






       lowerCamelCase_	  =					'<s>'
       lowerCamelCase_	  =					1

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(A_						)		,							A_						)
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(A_						)		,							A_						)

 def        a__				(       self      :						Optional[int]						)     ->			Any:

       """simple docstring"""






       lowerCamelCase_	  =					list(self.get_tokenizer().get_vocab().keys()						)

       self.assertEqual(vocab_keys[0]		,							'<unk>'						)
       self.assertEqual(vocab_keys[1]		,							'<s>'						)
       self.assertEqual(vocab_keys[-1]		,							'j'						)
       self.assertEqual(len(A_						)		,							2000						)

 def        a__				(       self      :						Optional[int]						)     ->			Any:

       """simple docstring"""






       self.assertEqual(self.get_tokenizer().vocab_size		,							2000						)

 def        a__				(       self      :						List[str]						)     ->			int:

       """simple docstring"""






       lowerCamelCase_	  =					GPTSwaTokenizer(A_						)

       lowerCamelCase_	  =					tokenizer.tokenize('This is a test'						)
       self.assertListEqual(A_		,							['▁This', '▁is', '▁a', '▁t', 'est']						)

       self.assertListEqual(tokenizer.convert_tokens_to_ids(A_						)		,							[465, 287, 265, 631, 842]						)

       lowerCamelCase_	  =					tokenizer.tokenize('I was born in 92000, and this is falsé.'						)
       # fmt: off
       self.assertListEqual(
           A_		,							['▁I', '▁was', '▁bor', 'n', '▁in', '▁', '<0x39>', '2', '0', '0', '0', ',', '▁and', '▁this', '▁is', '▁f', 'al', 's', '<0xC3>', '<0xA9>', '.']		,							)
       # fmt: on

       lowerCamelCase_	  =					tokenizer.convert_tokens_to_ids(A_						)
       self.assertListEqual(
           A_		,							[262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260]		,							)

       lowerCamelCase_	  =					tokenizer.convert_ids_to_tokens(A_						)
       # fmt: off
       self.assertListEqual(
           A_		,							['▁I', '▁was', '▁bor', 'n', '▁in', '▁', '<0x39>', '2', '0', '0', '0', ',', '▁and', '▁this', '▁is', '▁f', 'al', 's', '<0xC3>', '<0xA9>', '.']						)
       # fmt: on

 def        a__				(       self      :						List[str]						)     ->			List[str]:

       """simple docstring"""






       lowerCamelCase_	  =					GPTSwaTokenizer(A_						)
       lowerCamelCase_	  =					['This is a test', 'I was born in 92000, and this is falsé.']
       lowerCamelCase_	  =					[
           [465, 287, 265, 631, 842],
           [262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260],
       ]

       # Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
       for text, expected_ids in zip(A_		,							A_						):
             self.assertListEqual(tokenizer.encode_fast(A_						)		,							A_						)

       # Test that decode_fast returns the input text
       for text, token_ids in zip(A_		,							A_						):
             self.assertEqual(tokenizer.decode_fast(A_						)		,							A_						)





 @slow
 def        a__				(       self      :						Optional[int]						)     ->			Tuple:

       """simple docstring"""






       lowerCamelCase_	  =					[
           '<|python|>def fibonacci(n)\n    if n < 0:\n        print(\'Incorrect input\')',
           'Hey there, how are you doing this fine day?',
           'This is a text with a trailing spaces followed by a dot     .',
           'Häj sväjs lillebrör! =)',
           'Det är inget fel på Mr. Cool',
       ]

       # fmt: off
       lowerCamelCase_	  =					{'input_ids': [[63423, 5, 6811, 14954, 282, 816, 3821, 63466, 63425, 63462, 18, 63978, 678, 301, 1320, 63423, 63455, 63458, 18, 63982, 4246, 3940, 1901, 47789, 5547, 18994], [19630, 1100, 63446, 1342, 633, 544, 4488, 593, 5102, 2416, 63495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1652, 428, 268, 1936, 515, 268, 58593, 22413, 9106, 546, 268, 33213, 63979, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [55130, 63450, 924, 63449, 2249, 4062, 1558, 318, 63504, 21498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [509, 377, 2827, 2559, 332, 6575, 63443, 26801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
       # fmt: on
       self.tokenizer_integration_test_util(
           expected_encoding=A_		,							model_name='AI-Sweden/gpt-sw3-126m'		,							sequences=A_		,							)









from __future__ import annotations

import copy
import tempfile
import unittest

from transformers import CONFIG_MAPPING, AutoConfig, BertConfig, GPTaConfig, TaConfig, TapasConfig, is_tf_available
from transformers.testing_utils import (
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tensorflow_probability,
    require_tf,
    slow,
)

from ..bert.test_modeling_bert import BertModelTester


if is_tf_available():
 from transformers import (
     TFAutoModel,
     TFAutoModelForCausalLM,
     TFAutoModelForMaskedLM,
     TFAutoModelForPreTraining,
     TFAutoModelForQuestionAnswering,
     TFAutoModelForSeqaSeqLM,
     TFAutoModelForSequenceClassification,
     TFAutoModelForTableQuestionAnswering,
     TFAutoModelForTokenClassification,
     TFAutoModelWithLMHead,
     TFBertForMaskedLM,
     TFBertForPreTraining,
     TFBertForQuestionAnswering,
     TFBertForSequenceClassification,
     TFBertModel,
     TFFunnelBaseModel,
     TFFunnelModel,
     TFGPTaLMHeadModel,
     TFRobertaForMaskedLM,
     TFTaForConditionalGeneration,
     TFTapasForQuestionAnswering,
 )
 from transformers.models.auto.modeling_tf_auto import (
     TF_MODEL_FOR_CAUSAL_LM_MAPPING,
     TF_MODEL_FOR_MASKED_LM_MAPPING,
     TF_MODEL_FOR_PRETRAINING_MAPPING,
     TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
     TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
     TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
     TF_MODEL_MAPPING,
 )
 from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
 from transformers.models.gpta.modeling_tf_gpta import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
 from transformers.models.ta.modeling_tf_ta import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST
 from transformers.models.tapas.modeling_tf_tapas import TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST







class 						A(		UpperCamelCase     ):



 '''simple docstring'''


 UpperCamelCase		     =		'''new-model'''


if is_tf_available():







 class 						A(		UpperCamelCase     ):



  '''simple docstring'''


  UpperCamelCase		     =		NewModelConfig







@require_tf
class 						A(		unittest.TestCase     ):



 '''simple docstring'''


 @slow
 def        a__				(       self      :						str						)     ->			Union[str, Any]:

       """simple docstring"""






       lowerCamelCase_	  =					'bert-base-cased'
       lowerCamelCase_	  =					AutoConfig.from_pretrained(A_						)
       self.assertIsNotNone(A_						)
       self.assertIsInstance(A_		,							A_						)

       lowerCamelCase_	  =					TFAutoModel.from_pretrained(A_						)
       self.assertIsNotNone(A_						)
       self.assertIsInstance(A_		,							A_						)

 @slow
 def        a__				(       self      :						List[Any]						)     ->			Dict:

       """simple docstring"""






       lowerCamelCase_	  =					'bert-base-cased'
       lowerCamelCase_	  =					AutoConfig.from_pretrained(A_						)
       self.assertIsNotNone(A_						)
       self.assertIsInstance(A_		,							A_						)

       lowerCamelCase_	  =					TFAutoModelForPreTraining.from_pretrained(A_						)
       self.assertIsNotNone(A_						)
       self.assertIsInstance(A_		,							A_						)

 @slow
 def        a__				(       self      :						Union[str, Any]						)     ->			str:

       """simple docstring"""






       for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             lowerCamelCase_	  =					AutoConfig.from_pretrained(A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

             lowerCamelCase_	  =					TFAutoModelForCausalLM.from_pretrained(A_						)
             lowerCamelCase_  ,       lowerCamelCase_	  =					TFAutoModelForCausalLM.from_pretrained(A_		,							output_loading_info=A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

 @slow
 def        a__				(       self      :						List[Any]						)     ->			Dict:

       """simple docstring"""






       for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             lowerCamelCase_	  =					AutoConfig.from_pretrained(A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

             lowerCamelCase_	  =					TFAutoModelWithLMHead.from_pretrained(A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

 @slow
 def        a__				(       self      :						int						)     ->			str:

       """simple docstring"""






       for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             lowerCamelCase_	  =					AutoConfig.from_pretrained(A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

             lowerCamelCase_	  =					TFAutoModelForMaskedLM.from_pretrained(A_						)
             lowerCamelCase_  ,       lowerCamelCase_	  =					TFAutoModelForMaskedLM.from_pretrained(A_		,							output_loading_info=A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

 @slow
 def        a__				(       self      :						Any						)     ->			List[Any]:

       """simple docstring"""






       for model_name in TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             lowerCamelCase_	  =					AutoConfig.from_pretrained(A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

             lowerCamelCase_	  =					TFAutoModelForSeqaSeqLM.from_pretrained(A_						)
             lowerCamelCase_  ,       lowerCamelCase_	  =					TFAutoModelForSeqaSeqLM.from_pretrained(A_		,							output_loading_info=A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

 @slow
 def        a__				(       self      :						Tuple						)     ->			str:

       """simple docstring"""






       for model_name in ["bert-base-uncased"]:
             lowerCamelCase_	  =					AutoConfig.from_pretrained(A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

             lowerCamelCase_	  =					TFAutoModelForSequenceClassification.from_pretrained(A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

 @slow
 def        a__				(       self      :						List[Any]						)     ->			Any:

       """simple docstring"""






       for model_name in ["bert-base-uncased"]:
             lowerCamelCase_	  =					AutoConfig.from_pretrained(A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

             lowerCamelCase_	  =					TFAutoModelForQuestionAnswering.from_pretrained(A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

 @slow
 @require_tensorflow_probability
 def        a__				(       self      :						int						)     ->			Union[str, Any]:

       """simple docstring"""






       for model_name in TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST[5:6]:
             lowerCamelCase_	  =					AutoConfig.from_pretrained(A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

             lowerCamelCase_	  =					TFAutoModelForTableQuestionAnswering.from_pretrained(A_						)
             lowerCamelCase_  ,       lowerCamelCase_	  =					TFAutoModelForTableQuestionAnswering.from_pretrained(
                 A_		,							output_loading_info=A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

 def        a__				(       self      :						int						)     ->			Union[str, Any]:

       """simple docstring"""






       lowerCamelCase_	  =					TFAutoModelWithLMHead.from_pretrained(A_						)
       self.assertIsInstance(A_		,							A_						)
       self.assertEqual(model.num_parameters()		,							14410						)
       self.assertEqual(model.num_parameters(only_trainable=A_						)		,							14410						)

 def        a__				(       self      :						Optional[int]						)     ->			List[Any]:

       """simple docstring"""






       lowerCamelCase_	  =					TFAutoModelWithLMHead.from_pretrained(A_						)
       self.assertIsInstance(A_		,							A_						)
       self.assertEqual(model.num_parameters()		,							14410						)
       self.assertEqual(model.num_parameters(only_trainable=A_						)		,							14410						)

 def        a__				(       self      :						Tuple						)     ->			Optional[Any]:

       """simple docstring"""






       lowerCamelCase_	  =					TFAutoModel.from_pretrained('sgugger/funnel-random-tiny'						)
       self.assertIsInstance(A_		,							A_						)

       lowerCamelCase_	  =					copy.deepcopy(model.config						)
       lowerCamelCase_	  =					['FunnelBaseModel']
       lowerCamelCase_	  =					TFAutoModel.from_config(A_						)
       self.assertIsInstance(A_		,							A_						)

       with tempfile.TemporaryDirectory() as tmp_dir:
             model.save_pretrained(A_						)
             lowerCamelCase_	  =					TFAutoModel.from_pretrained(A_						)
             self.assertIsInstance(A_		,							A_						)

 def        a__				(       self      :						Any						)     ->			Tuple:

       """simple docstring"""






       try:
             AutoConfig.register('new-model'		,							A_						)

             lowerCamelCase_	  =					[
                 TFAutoModel,
                 TFAutoModelForCausalLM,
                 TFAutoModelForMaskedLM,
                 TFAutoModelForPreTraining,
                 TFAutoModelForQuestionAnswering,
                 TFAutoModelForSequenceClassification,
                 TFAutoModelForTokenClassification,
             ]

             for auto_class in auto_classes:
                   with self.subTest(auto_class.__name__						):
                         # Wrong config class will raise an error
                         with self.assertRaises(A_						):
                               auto_class.register(A_		,							A_						)
                         auto_class.register(A_		,							A_						)
                         # Trying to register something existing in the Transformers library will raise an error
                         with self.assertRaises(A_						):
                               auto_class.register(A_		,							A_						)

                         # Now that the config is registered, it can be used as any other config with the auto-API
                         lowerCamelCase_	  =					BertModelTester(self						).get_config()
                         lowerCamelCase_	  =					NewModelConfig(**tiny_config.to_dict()						)
                         lowerCamelCase_	  =					auto_class.from_config(A_						)
                         self.assertIsInstance(A_		,							A_						)

                         with tempfile.TemporaryDirectory() as tmp_dir:
                               model.save_pretrained(A_						)
                               lowerCamelCase_	  =					auto_class.from_pretrained(A_						)
                               self.assertIsInstance(A_		,							A_						)

       finally:
             if "new-model" in CONFIG_MAPPING._extra_content:
                   del CONFIG_MAPPING._extra_content["new-model"]
             for mapping in (
                 TF_MODEL_MAPPING,
                 TF_MODEL_FOR_PRETRAINING_MAPPING,
                 TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
                 TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
                 TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
                 TF_MODEL_FOR_CAUSAL_LM_MAPPING,
                 TF_MODEL_FOR_MASKED_LM_MAPPING,
             ):
                   if NewModelConfig in mapping._extra_content:
                         del mapping._extra_content[NewModelConfig]

 def        a__				(       self      :						int						)     ->			int:

       """simple docstring"""






       with self.assertRaisesRegex(
           A_		,							'bert-base is not a local folder and is not a valid model identifier'						):
             lowerCamelCase_	  =					TFAutoModel.from_pretrained('bert-base'						)

 def        a__				(       self      :						Any						)     ->			Dict:

       """simple docstring"""






       with self.assertRaisesRegex(
           A_		,							r'aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'						):
             lowerCamelCase_	  =					TFAutoModel.from_pretrained(A_		,							revision='aaaaaa'						)

 def        a__				(       self      :						str						)     ->			int:

       """simple docstring"""






       with self.assertRaisesRegex(
           A_		,							'hf-internal-testing/config-no-model does not appear to have a file named pytorch_model.bin'		,							):
             lowerCamelCase_	  =					TFAutoModel.from_pretrained('hf-internal-testing/config-no-model'						)

 def        a__				(       self      :						Any						)     ->			List[Any]:

       """simple docstring"""






       with self.assertRaisesRegex(A_		,							'Use `from_pt=True` to load this model'						):
             lowerCamelCase_	  =					TFAutoModel.from_pretrained('hf-internal-testing/tiny-bert-pt-only'						)





 def        a__				(       self      :						Tuple						)     ->			Union[str, Any]:

       """simple docstring"""






       lowerCamelCase_	  =					TFAutoModel.from_pretrained('hf-internal-testing/tiny-random-bert'						)
       with RequestCounter() as counter:
             lowerCamelCase_	  =					TFAutoModel.from_pretrained('hf-internal-testing/tiny-random-bert'						)
             self.assertEqual(counter.get_request_count		,							0						)
             self.assertEqual(counter.head_request_count		,							1						)
             self.assertEqual(counter.other_request_count		,							0						)

       # With a sharded checkpoint
       lowerCamelCase_	  =					TFAutoModel.from_pretrained('ArthurZ/tiny-random-bert-sharded'						)
       with RequestCounter() as counter:
             lowerCamelCase_	  =					TFAutoModel.from_pretrained('ArthurZ/tiny-random-bert-sharded'						)
             self.assertEqual(counter.get_request_count		,							0						)
             self.assertEqual(counter.head_request_count		,							1						)
             self.assertEqual(counter.other_request_count		,							0						)



