




import json
import os
import unittest
from typing import Tuple

from transformers import WavaVecaPhonemeCTCTokenizer
from transformers.models.wavaveca.tokenization_wavaveca import VOCAB_FILES_NAMES
from transformers.models.wavaveca_phoneme.tokenization_wavaveca_phoneme import WavaVecaPhonemeCTCTokenizerOutput
from transformers.testing_utils import require_phonemizer

from ...test_tokenization_common import TokenizerTesterMixin







@require_phonemizer
class 						A(		UpperCamelCase							,   unittest.TestCase     ):



 '''simple docstring'''


 UpperCamelCase		     =		WavaVecaPhonemeCTCTokenizer
 UpperCamelCase		     =		False

 def        a__				(       self      :						List[str]						)     ->			Tuple:

       """simple docstring"""






       super().setUp()

       lowerCamelCase_	  =					(
           '<s> <pad> </s> <unk> n s t ə l a i k d m ɛ ɾ e ɪ p o ɐ z ð f j v b ɹ ʁ ʊ iː r w ʌ u ɡ æ aɪ ʃ h ɔ ɑː '
           'ŋ ɚ eɪ β uː y ɑ̃ oʊ ᵻ eː θ aʊ ts oː ɔ̃ ɣ ɜ ɑ dʒ əl x ɜː ç ʒ tʃ ɔː ɑːɹ ɛ̃ ʎ ɔːɹ ʋ aː ɕ œ ø oːɹ ɲ yː '
           'ʔ iə i5 s. tɕ ?? nʲ ɛː œ̃ ɭ ɔø ʑ tʲ ɨ ɛɹ ts. rʲ ɪɹ ɭʲ i.5 ɔɪ q sʲ u5 ʊɹ iɜ a5 iɛ5 øː ʕ ja əɜ th ɑ5 '
           'oɪ dʲ ə5 tɕh ts.h mʲ ɯ dʑ vʲ e̞ tʃʲ ei5 o5 onɡ5 ɑu5 iɑ5 ai5 aɪɚ kh ə1 ʐ i2 ʉ ħ t[ aɪə ʲ ju ə2 u2 oɜ '
           'pː iɛɜ ou5 y5 uɜ tː uo5 d[ uoɜ tsh ɑɜ ɵ i̪5 uei5 ɟ aɜ ɑɨ i.ɜ eʊ o2 ɐ̃ ä pʲ kʲ n̩ ɒ ph ɑu2 uɨ əɪ ɫ ɬ '
           'yɜ bʲ ɑ2 s̪ aiɜ χ ɐ̃ʊ̃ 1 ə4 yæɜ a2 ɨː t̪ iouɜ ũ onɡɜ aɨ iɛ2 ɔɨ ɑuɜ o̞ ei2 iou2 c kː y2 ɖ oe dˤ yɛɜ '
           'əʊ S ɡʲ onɡ2 u" eiɜ ʈ ɯᵝ iou5 dZ r̝̊ i.2 tS s^ ʝ yə5 iɑɜ uə5 pf ɨu iɑ2 ou2 ər2 fʲ ai2 r̝ uəɜ ɳ əɨ '
           'ua5 uɪ ɽ bː yu5 uo2 yɛ5 l̩ ɻ ərɜ ʂ i̪2 ouɜ uaɜ a. a.ː yæ5 dː r̩ ee ɪu ər5 i̪ ɜ æi u: i.ː t^ o1 ɪ^ '
           'ai ueiɜ æː ɛɪ eə i. ɴ ie ua2 ɑ1 o4 tʃː o: ɑ: u1 N i̪1 au yæ2 u. qː yəɜ y: kʰ tʃʰ iʊ sx õ uo tʰ '
           'uai5 bʰ u.ː uə2 ʊə d^ s̪ː yiɜ dʰ r. oe: i1 ɟː yu2 nʲʲ i̪4 uei2 tsʲ ɸ ĩ ɑ4 t̪ː eɑ u4 e: tsː ʈʰ ɡʰ '
           'ɯɯ dʒʲ ʂʲ X ɵː uaiɜ tɕʲ ã t^ː ẽː yɛ2 cː i.1 ɛʊ dˤdˤ dʒː i4 ɡː yi ɕʲ ɟʰ pʰ dʑʲ yuɜ ua1 ua4 æiː ɐɐ '
           'ui iou1 ʊː a1 iou4 cʰ iɛ1 yə2 ɖʰ ẽ ʒʲ ää ər4 iːː ɪː iɑ1 ər1 œː øi ɪuː cʰcʰ əː1 iː1 ũ kʰː o̞o̞ xʲ '
           'ou1 iɛ4 e̞e̞ y1 dzː dʲʲ dʰː ɯᵝɯᵝ lː uo1 i.4 i: yɛ5ʲ a4'
       ).split(' '						)
       lowerCamelCase_	  =					dict(zip(A_		,							range(len(A_						)						)						)						)

       lowerCamelCase_	  =					{'pad_token': '<pad>', 'unk_token': '<unk>', 'bos_token': '<s>', 'eos_token': '</s>'}

       lowerCamelCase_	  =					os.path.join(self.tmpdirname		,							VOCAB_FILES_NAMES['vocab_file']						)
       with open(self.vocab_file		,							'w'		,							encoding='utf-8'						) as fp:
             fp.write(json.dumps(A_						) + '\n'						)

 def        a__				(       self      :						Optional[int]		,							A_      :						int		,							A_      :						Union[str, Any]=False		,							A_      :						List[Any]=20		,							A_      :						List[Any]=5						)     ->			Tuple[str, list]:

       """simple docstring"""






       lowerCamelCase_	  =					[(i, tokenizer.decode([i]		,							clean_up_tokenization_spaces=A_						)) for i in range(len(A_						)						)]
       lowerCamelCase_	  =					list(filter(lambda A_						: [t[0]] == tokenizer.encode(t[1]		,							do_phonemize=A_						)		,							A_						)						)
       if max_length is not None and len(A_						) > max_length:
             lowerCamelCase_	  =					toks[:max_length]
       if min_length is not None and len(A_						) < min_length and len(A_						) > 0:
             while len(A_						) < min_length:
                   lowerCamelCase_	  =					toks + toks
        # toks_str = [t[1] for t in toks]
       lowerCamelCase_	  =					[t[0] for t in toks]

       # Ensure consistency
       lowerCamelCase_	  =					tokenizer.decode(A_		,							clean_up_tokenization_spaces=A_						)
       if " " not in output_txt and len(A_						) > 1:
             lowerCamelCase_	  =					(
                 tokenizer.decode([toks_ids[0]]		,							clean_up_tokenization_spaces=A_						)
                 + ' '
                 + tokenizer.decode(toks_ids[1:]		,							clean_up_tokenization_spaces=A_						)
             )
       if with_prefix_space:
             lowerCamelCase_	  =					' ' + output_txt
       lowerCamelCase_	  =					tokenizer.encode(A_		,							add_special_tokens=A_						)
       return output_txt, output_ids

 def        a__				(       self      :						str		,							**A_      :						str						)     ->			Any:

       """simple docstring"""






       kwargs.update(self.special_tokens_map						)
       return WavaVecaPhonemeCTCTokenizer.from_pretrained(self.tmpdirname		,							**A_						)

 def        a__				(       self      :						List[Any]						)     ->			Optional[int]:

       """simple docstring"""






       lowerCamelCase_	  =					self.tokenizer_class.from_pretrained('facebook/wav2vec2-lv-60-espeak-cv-ft'						)

       # check adding a single token
       tokenizer.add_tokens('xxx'						)
       lowerCamelCase_	  =					tokenizer('m xxx ɪ'		,							do_phonemize=A_						).input_ids
       self.assertEqual(A_		,							[13, 392, 17]						)  # xxx should be last token

       tokenizer.add_tokens(['aaa', 'bbb', 'ccc']						)
       lowerCamelCase_	  =					tokenizer('m aaa ɪ ccc'		,							do_phonemize=A_						).input_ids
       self.assertEqual(A_		,							[13, 393, 17, 395]						)  # aaa and ccc should be after xxx and 2 after aaa

       lowerCamelCase_	  =					tokenizer('maɪ c'		,							do_phonemize=A_						).input_ids
       self.assertEqual(A_		,							[3, 200]						)  # mai should be <unk> (=3)

 def        a__				(       self      :						Any						)     ->			int:

       """simple docstring"""






       lowerCamelCase_	  =					self.tokenizer_class.from_pretrained('facebook/wav2vec2-lv-60-espeak-cv-ft'						)

       lowerCamelCase_	  =					'Hello how are you'
       lowerCamelCase_	  =					tokenizer.phonemize(A_		,							phonemizer_lang='en-us'						)
       self.assertEqual(A_		,							'h ə l oʊ h aʊ ɑːɹ j uː'						)

 def        a__				(       self      :						List[Any]						)     ->			Dict:

       """simple docstring"""






       lowerCamelCase_	  =					self.tokenizer_class.from_pretrained('facebook/wav2vec2-lv-60-espeak-cv-ft'						)

       lowerCamelCase_	  =					'Hello how are you'
       lowerCamelCase_	  =					tokenizer.phonemize(A_		,							phonemizer_lang='en-us'						)
       self.assertEqual(tokenizer(A_						).input_ids		,							tokenizer(A_		,							do_phonemize=A_						).input_ids						)

 def        a__				(       self      :						Optional[Any]						)     ->			Dict:

       """simple docstring"""






       lowerCamelCase_	  =					self.tokenizer_class.from_pretrained('facebook/wav2vec2-lv-60-espeak-cv-ft'						)
       lowerCamelCase_	  =					'Hello how are you'
       lowerCamelCase_	  =					tokenizer.phonemize(A_		,							phonemizer_lang='en-us'						)

       lowerCamelCase_	  =					tokenizer.decode(tokenizer(A_						).input_ids						)

       self.assertEqual(A_		,							A_						)

 def        a__				(       self      :						Union[str, Any]						)     ->			List[Any]:

       """simple docstring"""






       lowerCamelCase_	  =					self.tokenizer_class.from_pretrained('facebook/wav2vec2-lv-60-espeak-cv-ft'						)

       lowerCamelCase_	  =					[
           [11, 5, 15, tokenizer.pad_token_id, 15, 8, 98],
           [24, 22, 5, 24, 22, 5, 77],
       ]
       lowerCamelCase_	  =					tokenizer.decode(sample_ids[0]						)
       lowerCamelCase_	  =					tokenizer.batch_decode(A_						)
       self.assertEqual(A_		,							batch_tokens[0]						)
       self.assertEqual(A_		,							['k s ɾ ɾ l ɭʲ', 'j ð s j ð s oːɹ']						)

 def        a__				(       self      :						Tuple						)     ->			int:

       """simple docstring"""






       lowerCamelCase_	  =					self.tokenizer_class.from_pretrained(
           'facebook/wav2vec2-lv-60-espeak-cv-ft'		,							word_delimiter_token='|'						)
       tokenizer.add_tokens('|'						)

       lowerCamelCase_	  =					'Hello how are you'
       lowerCamelCase_	  =					tokenizer.phonemize(A_		,							phonemizer_lang='en-us'						)
       self.assertEqual(A_		,							'h ə l oʊ | h aʊ | ɑːɹ | j uː |'						)

 def        a__				(       self      :						str						)     ->			str:

       """simple docstring"""






       lowerCamelCase_	  =					self.tokenizer_class.from_pretrained(
           'facebook/wav2vec2-lv-60-espeak-cv-ft'		,							word_delimiter_token='|'						)
       tokenizer.add_tokens('|'						)

       lowerCamelCase_	  =					'Hello how are you'
       lowerCamelCase_	  =					tokenizer.phonemize(A_		,							phonemizer_lang='en-us'						)
       self.assertEqual(tokenizer(A_						).input_ids		,							tokenizer(A_		,							do_phonemize=A_						).input_ids						)

 def        a__				(       self      :						Optional[Any]						)     ->			int:

       """simple docstring"""






       lowerCamelCase_	  =					self.tokenizer_class.from_pretrained(
           'facebook/wav2vec2-lv-60-espeak-cv-ft'		,							word_delimiter_token='|'						)
       tokenizer.add_tokens('|'						)

       # fmt: off
       lowerCamelCase_	  =					[
           [11, 5, 15, tokenizer.pad_token_id, tokenizer.word_delimiter_token_id, 15, 8, tokenizer.word_delimiter_token_id, 98],
           [tokenizer.word_delimiter_token_id, 24, 22, tokenizer.word_delimiter_token_id, 5, 24, 22, 5, 77],
       ]
       # fmt: on

       # decode with word_del_token filter
       lowerCamelCase_	  =					tokenizer.decode(sample_ids[0]						)
       lowerCamelCase_	  =					tokenizer.batch_decode(A_						)
       self.assertEqual(A_		,							batch_tokens[0]						)
       self.assertEqual(A_		,							['k s ɾ ɾ l ɭʲ', 'j ð s j ð s oːɹ']						)

       # decode with no word_del_token filter
       lowerCamelCase_	  =					tokenizer.decode(sample_ids[0]		,							filter_word_delimiter_token=A_						)
       lowerCamelCase_	  =					tokenizer.batch_decode(A_		,							filter_word_delimiter_token=A_						)
       self.assertEqual(A_		,							batch_tokens[0]						)
       self.assertEqual(A_		,							['k s ɾ | ɾ l | ɭʲ', '| j ð | s j ð s oːɹ']						)

 def        a__				(       self      :						Tuple						)     ->			Union[str, Any]:

       """simple docstring"""






       lowerCamelCase_	  =					self.tokenizer_class.from_pretrained(
           'facebook/wav2vec2-lv-60-espeak-cv-ft'		,							word_delimiter_token='|'						)
       tokenizer.add_tokens('|'						)

       lowerCamelCase_	  =					'Hello how are you'
       lowerCamelCase_	  =					tokenizer.phonemize(A_		,							phonemizer_lang='en-us'						)

       lowerCamelCase_	  =					tokenizer.decode(tokenizer(A_						).input_ids		,							filter_word_delimiter_token=A_						)

       self.assertEqual(A_		,							A_						)

 def        a__				(       self      :						Tuple						)     ->			Any:

       """simple docstring"""






       lowerCamelCase_	  =					self.tokenizer_class.from_pretrained(
           'facebook/wav2vec2-lv-60-espeak-cv-ft'		,							word_delimiter_token='|'						)
       tokenizer.add_tokens('|'						)

       lowerCamelCase_	  =					'Hello how are you'
       lowerCamelCase_	  =					tokenizer.phonemize(A_		,							phonemizer_lang='en-us'						)

       lowerCamelCase_	  =					tokenizer.decode(tokenizer(A_						).input_ids		,							filter_word_delimiter_token=A_						)

       self.assertEqual(' '.join([p.strip() for p in phonemes.split(' |'						)]						).strip()		,							A_						)

 def        a__				(       self      :						Union[str, Any]						)     ->			List[Any]:

       """simple docstring"""






       lowerCamelCase_	  =					self.tokenizer_class.from_pretrained(
           'facebook/wav2vec2-lv-60-espeak-cv-ft'		,							word_delimiter_token=A_						)
       lowerCamelCase_	  =					'Hello how are you'

       lowerCamelCase_	  =					tokenizer(A_		,							phonemizer_lang='en-us'						).input_ids
       lowerCamelCase_	  =					tokenizer(A_		,							phonemizer_lang='fr-fr'						).input_ids

       self.assertNotEqual(A_		,							A_						)

       lowerCamelCase_	  =					tokenizer.decode(A_						)
       lowerCamelCase_	  =					tokenizer.decode(A_						)

       self.assertEqual(A_		,							'h ə l oʊ h aʊ ɑːɹ j uː'						)
       self.assertEqual(A_		,							'ɛ l o h aʊ a ʁ j u'						)

 def        a__				(       self      :						List[str]						)     ->			Dict:

       """simple docstring"""






       lowerCamelCase_	  =					self.tokenizer_class.from_pretrained('facebook/wav2vec2-lv-60-espeak-cv-ft'						)
       lowerCamelCase_	  =					'Hello how Are you'
       lowerCamelCase_	  =					'hello how are you'

       lowerCamelCase_	  =					tokenizer(A_						).input_ids
       lowerCamelCase_	  =					tokenizer(A_						).input_ids

       self.assertEqual(A_		,							A_						)

 def        a__				(       self      :						Union[str, Any]						)     ->			List[Any]:

       """simple docstring"""






       lowerCamelCase_	  =					self.tokenizer_class.from_pretrained('facebook/wav2vec2-lv-60-espeak-cv-ft'						)
       tokenizer.add_tokens(['!', '?']						)
       tokenizer.add_special_tokens({'cls_token': '$$$'}						)

       # fmt: off
       lowerCamelCase_	  =					[
           [11, 5, 15, tokenizer.pad_token_id, 15, 8, 98, 392, 392, 393, 392, 392, 393, 394, 394],
           [24, 22, 5, 24, 22, 5, 77, tokenizer.pad_token_id, 394, 394],
       ]
       # fmt: on

       lowerCamelCase_	  =					tokenizer.batch_decode(A_						)
       self.assertEqual(A_		,							['k s ɾ ɾ l ɭʲ!?!? $$$', 'j ð s j ð s oːɹ $$$']						)

 @staticmethod
 def        a__				(       A_      :						str		,							A_      :						Optional[Any]						)     ->			List[str]:

       """simple docstring"""






       lowerCamelCase_	  =					[d[key] for d in offsets]
       return retrieved_list

 def        a__				(       self      :						List[Any]						)     ->			Optional[int]:

       """simple docstring"""






       lowerCamelCase_	  =					self.get_tokenizer(word_delimiter_token='|'						)
       tokenizer.add_tokens('|'						)

       # fmt: off
       # ksssɾɾ|ɾɾ<pad>ɾɾ|<pad>ɾlll|ɭʲ -> k s ɾ ɾ | ɾ l | ɭʲ"
       lowerCamelCase_	  =					[11, 5, 5, 5, 15, 15, tokenizer.pad_token_id, 15, 15, tokenizer.word_delimiter_token_id, tokenizer.pad_token_id, 15, 8, 8, 8, tokenizer.word_delimiter_token_id, 98]
       # fmt: on

       lowerCamelCase_	  =					tokenizer.decode(A_		,							output_char_offsets=A_		,							filter_word_delimiter_token=A_						)
       # check Wav2Vec2CTCTokenizerOutput keys for char
       self.assertEqual(len(outputs.keys()						)		,							2						)
       self.assertTrue('text' in outputs						)
       self.assertTrue('char_offsets' in outputs						)
       self.assertTrue(isinstance(A_		,							A_						)						)

       # check that order of chars is correct and identical for both outputs
       self.assertEqual(' '.join(self.get_from_offsets(outputs['char_offsets']		,							'char'						)						)		,							outputs.text						)
       self.assertListEqual(
           self.get_from_offsets(outputs['char_offsets']		,							'char'						)		,							['k', 's', 'ɾ', 'ɾ', '|', 'ɾ', 'l', '|', 'ɭʲ']						)

       # check that offsets are actually correct for char
       # 0-1 is 11, 1-4 is 5, 4-6 is first 15, 6-7 is <pad> (thus not shown), 7-9 is second 15, 9-10 is word_delimiter_token,
       # 10-11 is <pad> (thus not shown), 11-12 is third 15, 12-15 is 8, 15-16 is word_delimiter_token, 16-17 is 98
       self.assertListEqual(
           self.get_from_offsets(outputs['char_offsets']		,							'start_offset'						)		,							[0, 1, 4, 7, 9, 11, 12, 15, 16]						)
       self.assertListEqual(
           self.get_from_offsets(outputs['char_offsets']		,							'end_offset'						)		,							[1, 4, 6, 9, 10, 12, 15, 16, 17]						)

 def        a__				(       self      :						Tuple						)     ->			Optional[Any]:

       """simple docstring"""






       lowerCamelCase_	  =					self.get_tokenizer(word_delimiter_token='|'						)

       def check_list_tuples_equal(A_      :						Optional[int]		,							A_      :						int						):
             self.assertTrue(isinstance(A_		,							A_						)						)
             self.assertTrue(isinstance(outputs_list[0]		,							A_						)						)

             # transform list to ModelOutput
             lowerCamelCase_	  =					WavaVecaPhonemeCTCTokenizerOutput(
                 {k: [d[k] for d in outputs_list] for k in outputs_list[0]}						)

             self.assertListEqual(outputs_batch['text']		,							outputs_batch_a['text']						)

             def recursive_check(A_      :						List[str]		,							A_      :						Optional[int]						):
                   if isinstance(A_		,							A_						):
                         [recursive_check(A_		,							A_						) for la, la in zip(A_		,							A_						)]
                   self.assertEqual(A_		,							A_						)

             if "char_offsets" in outputs_batch:
                   recursive_check(outputs_batch['char_offsets']		,							outputs_batch_a['char_offsets']						)

        # fmt: off
       lowerCamelCase_	  =					[
           [11, 5, 15, tokenizer.pad_token_id, 15, 4, 8, 98, 32, 32, 32, 32, 4, 33, tokenizer.word_delimiter_token_id, 32, 32, 33, 34, 34],
           [24, 22, 5, tokenizer.word_delimiter_token_id, tokenizer.word_delimiter_token_id, 24, 22, 22, 22, 4, 5, 77, tokenizer.pad_token_id, 22, 22, 4, 34, 34, 34, 34],
       ]
       # fmt: on

       # We assume that `decode` works as expected. All we will check now is
       # the output type is correct and the output is identical to `decode`

       # char
       lowerCamelCase_	  =					tokenizer.batch_decode(A_		,							output_char_offsets=A_						)
       lowerCamelCase_	  =					[tokenizer.decode(A_		,							output_char_offsets=A_						) for ids in sample_ids]
       check_list_tuples_equal(A_		,							A_						)

 @unittest.skip('Wav2Vec2PhonemeTokenizer always lower cases letters to correctly map to phonemes'						)
 def        a__				(       self      :						Tuple						)     ->			Any:

       """simple docstring"""






       pass

 @unittest.skip('Wav2Vec2PhonemeTokenizer always puts spaces between phonemes'						)
 def        a__				(       self      :						List[Any]						)     ->			Tuple:

       """simple docstring"""






       pass

 @unittest.skip('encodes to text to ids, but decodes ids to phonemes -> not possible to have internal consistency'						)
 def        a__				(       self      :						Any						)     ->			List[str]:

       """simple docstring"""






       pass

 @unittest.skip('Wav2Vec2PhonemeModel has no max model length => no testing'						)
 def        a__				(       self      :						Optional[Any]						)     ->			Tuple:

       """simple docstring"""






       pass

 def        a__				(       self      :						str						)     ->			Union[str, Any]:

       """simple docstring"""






       lowerCamelCase_	  =					self.get_tokenizers(do_lower_case=A_						)
       for tokenizer in tokenizers:
             with self.subTest(f"""{tokenizer.__class__.__name__}"""						):
                   lowerCamelCase_	  =					tokenizer.vocab_size
                   lowerCamelCase_	  =					len(A_						)

                   self.assertNotEqual(A_		,							0						)

                   # We usually have added tokens from the start in tests because our vocab fixtures are
                   # smaller than the original vocabs - let's not assert this
                   # self.assertEqual(vocab_size, all_size)

                   lowerCamelCase_	  =					['aaaaa bbbbbb', 'cccccccccdddddddd']
                   lowerCamelCase_	  =					tokenizer.add_tokens(A_						)
                   lowerCamelCase_	  =					tokenizer.vocab_size
                   lowerCamelCase_	  =					len(A_						)

                   self.assertNotEqual(A_		,							0						)
                   self.assertEqual(A_		,							A_						)
                   self.assertEqual(A_		,							len(A_						)						)
                   self.assertEqual(A_		,							all_size + len(A_						)						)

                   lowerCamelCase_	  =					tokenizer.encode('aaaaa bbbbbb low cccccccccdddddddd l'		,							add_special_tokens=A_						)

                   self.assertGreaterEqual(len(A_						)		,							4						)
                   self.assertGreater(tokens[0]		,							tokenizer.vocab_size - 1						)
                   self.assertGreater(tokens[-3]		,							tokenizer.vocab_size - 1						)

                   lowerCamelCase_	  =					{'eos_token': '>>>>|||<||<<|<<', 'pad_token': '<<<<<|||>|>>>>|>'}
                   lowerCamelCase_	  =					tokenizer.add_special_tokens(A_						)
                   lowerCamelCase_	  =					tokenizer.vocab_size
                   lowerCamelCase_	  =					len(A_						)

                   self.assertNotEqual(A_		,							0						)
                   self.assertEqual(A_		,							A_						)
                   self.assertEqual(A_		,							len(A_						)						)
                   self.assertEqual(A_		,							all_size_a + len(A_						)						)

                   lowerCamelCase_	  =					tokenizer.encode(
                       '>>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l'		,							add_special_tokens=A_						)

                   self.assertGreaterEqual(len(A_						)		,							6						)
                   self.assertGreater(tokens[0]		,							tokenizer.vocab_size - 1						)
                   self.assertGreater(tokens[0]		,							tokens[1]						)
                   self.assertGreater(tokens[-3]		,							tokenizer.vocab_size - 1						)
                   self.assertGreater(tokens[-3]		,							tokens[-4]						)
                   self.assertEqual(tokens[0]		,							tokenizer.eos_token_id						)
                   self.assertEqual(tokens[-3]		,							tokenizer.pad_token_id						)

 @unittest.skip('The tokenizer shouldn\'t be used to encode input IDs (except for labels), only to decode.'						)
 def        a__				(       self      :						List[str]						)     ->			Any:

       """simple docstring"""






       pass

 @unittest.skip('The tokenizer shouldn\'t be used to encode input IDs (except for labels), only to decode.'						)
 def        a__				(       self      :						str						)     ->			Any:

       """simple docstring"""






       pass





 def        a__				(       self      :						Optional[int]						)     ->			Tuple:

       """simple docstring"""






       lowerCamelCase_	  =					self.get_tokenizers(fast=A_		,							do_lower_case=A_						)
       for tokenizer in tokenizers:
             with self.subTest(f"""{tokenizer.__class__.__name__}"""						):
                   lowerCamelCase_	  =					['ð', 'ɪ', 's', 'ɪ', 'z', 'ɐ', 't', 'ɛ', 'k', 's', 't']
                   lowerCamelCase_	  =					tokenizer.convert_tokens_to_string(A_						)

                   self.assertIsInstance(output['text']		,							A_						)









from __future__ import annotations

import copy
import tempfile
import unittest

from transformers import CONFIG_MAPPING, AutoConfig, BertConfig, GPTaConfig, TaConfig, TapasConfig, is_tf_available
from transformers.testing_utils import (
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tensorflow_probability,
    require_tf,
    slow,
)

from ..bert.test_modeling_bert import BertModelTester


if is_tf_available():
 from transformers import (
     TFAutoModel,
     TFAutoModelForCausalLM,
     TFAutoModelForMaskedLM,
     TFAutoModelForPreTraining,
     TFAutoModelForQuestionAnswering,
     TFAutoModelForSeqaSeqLM,
     TFAutoModelForSequenceClassification,
     TFAutoModelForTableQuestionAnswering,
     TFAutoModelForTokenClassification,
     TFAutoModelWithLMHead,
     TFBertForMaskedLM,
     TFBertForPreTraining,
     TFBertForQuestionAnswering,
     TFBertForSequenceClassification,
     TFBertModel,
     TFFunnelBaseModel,
     TFFunnelModel,
     TFGPTaLMHeadModel,
     TFRobertaForMaskedLM,
     TFTaForConditionalGeneration,
     TFTapasForQuestionAnswering,
 )
 from transformers.models.auto.modeling_tf_auto import (
     TF_MODEL_FOR_CAUSAL_LM_MAPPING,
     TF_MODEL_FOR_MASKED_LM_MAPPING,
     TF_MODEL_FOR_PRETRAINING_MAPPING,
     TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
     TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
     TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
     TF_MODEL_MAPPING,
 )
 from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
 from transformers.models.gpta.modeling_tf_gpta import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
 from transformers.models.ta.modeling_tf_ta import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST
 from transformers.models.tapas.modeling_tf_tapas import TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST







class 						A(		UpperCamelCase     ):



 '''simple docstring'''


 UpperCamelCase		     =		'''new-model'''


if is_tf_available():







 class 						A(		UpperCamelCase     ):



  '''simple docstring'''


  UpperCamelCase		     =		NewModelConfig







@require_tf
class 						A(		unittest.TestCase     ):



 '''simple docstring'''


 @slow
 def        a__				(       self      :						str						)     ->			Union[str, Any]:

       """simple docstring"""






       lowerCamelCase_	  =					'bert-base-cased'
       lowerCamelCase_	  =					AutoConfig.from_pretrained(A_						)
       self.assertIsNotNone(A_						)
       self.assertIsInstance(A_		,							A_						)

       lowerCamelCase_	  =					TFAutoModel.from_pretrained(A_						)
       self.assertIsNotNone(A_						)
       self.assertIsInstance(A_		,							A_						)

 @slow
 def        a__				(       self      :						List[Any]						)     ->			Dict:

       """simple docstring"""






       lowerCamelCase_	  =					'bert-base-cased'
       lowerCamelCase_	  =					AutoConfig.from_pretrained(A_						)
       self.assertIsNotNone(A_						)
       self.assertIsInstance(A_		,							A_						)

       lowerCamelCase_	  =					TFAutoModelForPreTraining.from_pretrained(A_						)
       self.assertIsNotNone(A_						)
       self.assertIsInstance(A_		,							A_						)

 @slow
 def        a__				(       self      :						Union[str, Any]						)     ->			str:

       """simple docstring"""






       for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             lowerCamelCase_	  =					AutoConfig.from_pretrained(A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

             lowerCamelCase_	  =					TFAutoModelForCausalLM.from_pretrained(A_						)
             lowerCamelCase_  ,       lowerCamelCase_	  =					TFAutoModelForCausalLM.from_pretrained(A_		,							output_loading_info=A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

 @slow
 def        a__				(       self      :						List[Any]						)     ->			Dict:

       """simple docstring"""






       for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             lowerCamelCase_	  =					AutoConfig.from_pretrained(A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

             lowerCamelCase_	  =					TFAutoModelWithLMHead.from_pretrained(A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

 @slow
 def        a__				(       self      :						int						)     ->			str:

       """simple docstring"""






       for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             lowerCamelCase_	  =					AutoConfig.from_pretrained(A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

             lowerCamelCase_	  =					TFAutoModelForMaskedLM.from_pretrained(A_						)
             lowerCamelCase_  ,       lowerCamelCase_	  =					TFAutoModelForMaskedLM.from_pretrained(A_		,							output_loading_info=A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

 @slow
 def        a__				(       self      :						Any						)     ->			List[Any]:

       """simple docstring"""






       for model_name in TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             lowerCamelCase_	  =					AutoConfig.from_pretrained(A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

             lowerCamelCase_	  =					TFAutoModelForSeqaSeqLM.from_pretrained(A_						)
             lowerCamelCase_  ,       lowerCamelCase_	  =					TFAutoModelForSeqaSeqLM.from_pretrained(A_		,							output_loading_info=A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

 @slow
 def        a__				(       self      :						Tuple						)     ->			str:

       """simple docstring"""






       for model_name in ["bert-base-uncased"]:
             lowerCamelCase_	  =					AutoConfig.from_pretrained(A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

             lowerCamelCase_	  =					TFAutoModelForSequenceClassification.from_pretrained(A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

 @slow
 def        a__				(       self      :						List[Any]						)     ->			Any:

       """simple docstring"""






       for model_name in ["bert-base-uncased"]:
             lowerCamelCase_	  =					AutoConfig.from_pretrained(A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

             lowerCamelCase_	  =					TFAutoModelForQuestionAnswering.from_pretrained(A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

 @slow
 @require_tensorflow_probability
 def        a__				(       self      :						int						)     ->			Union[str, Any]:

       """simple docstring"""






       for model_name in TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST[5:6]:
             lowerCamelCase_	  =					AutoConfig.from_pretrained(A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

             lowerCamelCase_	  =					TFAutoModelForTableQuestionAnswering.from_pretrained(A_						)
             lowerCamelCase_  ,       lowerCamelCase_	  =					TFAutoModelForTableQuestionAnswering.from_pretrained(
                 A_		,							output_loading_info=A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

 def        a__				(       self      :						int						)     ->			Union[str, Any]:

       """simple docstring"""






       lowerCamelCase_	  =					TFAutoModelWithLMHead.from_pretrained(A_						)
       self.assertIsInstance(A_		,							A_						)
       self.assertEqual(model.num_parameters()		,							14410						)
       self.assertEqual(model.num_parameters(only_trainable=A_						)		,							14410						)

 def        a__				(       self      :						Optional[int]						)     ->			List[Any]:

       """simple docstring"""






       lowerCamelCase_	  =					TFAutoModelWithLMHead.from_pretrained(A_						)
       self.assertIsInstance(A_		,							A_						)
       self.assertEqual(model.num_parameters()		,							14410						)
       self.assertEqual(model.num_parameters(only_trainable=A_						)		,							14410						)

 def        a__				(       self      :						Tuple						)     ->			Optional[Any]:

       """simple docstring"""






       lowerCamelCase_	  =					TFAutoModel.from_pretrained('sgugger/funnel-random-tiny'						)
       self.assertIsInstance(A_		,							A_						)

       lowerCamelCase_	  =					copy.deepcopy(model.config						)
       lowerCamelCase_	  =					['FunnelBaseModel']
       lowerCamelCase_	  =					TFAutoModel.from_config(A_						)
       self.assertIsInstance(A_		,							A_						)

       with tempfile.TemporaryDirectory() as tmp_dir:
             model.save_pretrained(A_						)
             lowerCamelCase_	  =					TFAutoModel.from_pretrained(A_						)
             self.assertIsInstance(A_		,							A_						)

 def        a__				(       self      :						Any						)     ->			Tuple:

       """simple docstring"""






       try:
             AutoConfig.register('new-model'		,							A_						)

             lowerCamelCase_	  =					[
                 TFAutoModel,
                 TFAutoModelForCausalLM,
                 TFAutoModelForMaskedLM,
                 TFAutoModelForPreTraining,
                 TFAutoModelForQuestionAnswering,
                 TFAutoModelForSequenceClassification,
                 TFAutoModelForTokenClassification,
             ]

             for auto_class in auto_classes:
                   with self.subTest(auto_class.__name__						):
                         # Wrong config class will raise an error
                         with self.assertRaises(A_						):
                               auto_class.register(A_		,							A_						)
                         auto_class.register(A_		,							A_						)
                         # Trying to register something existing in the Transformers library will raise an error
                         with self.assertRaises(A_						):
                               auto_class.register(A_		,							A_						)

                         # Now that the config is registered, it can be used as any other config with the auto-API
                         lowerCamelCase_	  =					BertModelTester(self						).get_config()
                         lowerCamelCase_	  =					NewModelConfig(**tiny_config.to_dict()						)
                         lowerCamelCase_	  =					auto_class.from_config(A_						)
                         self.assertIsInstance(A_		,							A_						)

                         with tempfile.TemporaryDirectory() as tmp_dir:
                               model.save_pretrained(A_						)
                               lowerCamelCase_	  =					auto_class.from_pretrained(A_						)
                               self.assertIsInstance(A_		,							A_						)

       finally:
             if "new-model" in CONFIG_MAPPING._extra_content:
                   del CONFIG_MAPPING._extra_content["new-model"]
             for mapping in (
                 TF_MODEL_MAPPING,
                 TF_MODEL_FOR_PRETRAINING_MAPPING,
                 TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
                 TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
                 TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
                 TF_MODEL_FOR_CAUSAL_LM_MAPPING,
                 TF_MODEL_FOR_MASKED_LM_MAPPING,
             ):
                   if NewModelConfig in mapping._extra_content:
                         del mapping._extra_content[NewModelConfig]

 def        a__				(       self      :						int						)     ->			int:

       """simple docstring"""






       with self.assertRaisesRegex(
           A_		,							'bert-base is not a local folder and is not a valid model identifier'						):
             lowerCamelCase_	  =					TFAutoModel.from_pretrained('bert-base'						)

 def        a__				(       self      :						Any						)     ->			Dict:

       """simple docstring"""






       with self.assertRaisesRegex(
           A_		,							r'aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'						):
             lowerCamelCase_	  =					TFAutoModel.from_pretrained(A_		,							revision='aaaaaa'						)

 def        a__				(       self      :						str						)     ->			int:

       """simple docstring"""






       with self.assertRaisesRegex(
           A_		,							'hf-internal-testing/config-no-model does not appear to have a file named pytorch_model.bin'		,							):
             lowerCamelCase_	  =					TFAutoModel.from_pretrained('hf-internal-testing/config-no-model'						)

 def        a__				(       self      :						Any						)     ->			List[Any]:

       """simple docstring"""






       with self.assertRaisesRegex(A_		,							'Use `from_pt=True` to load this model'						):
             lowerCamelCase_	  =					TFAutoModel.from_pretrained('hf-internal-testing/tiny-bert-pt-only'						)





 def        a__				(       self      :						Tuple						)     ->			Union[str, Any]:

       """simple docstring"""






       lowerCamelCase_	  =					TFAutoModel.from_pretrained('hf-internal-testing/tiny-random-bert'						)
       with RequestCounter() as counter:
             lowerCamelCase_	  =					TFAutoModel.from_pretrained('hf-internal-testing/tiny-random-bert'						)
             self.assertEqual(counter.get_request_count		,							0						)
             self.assertEqual(counter.head_request_count		,							1						)
             self.assertEqual(counter.other_request_count		,							0						)

       # With a sharded checkpoint
       lowerCamelCase_	  =					TFAutoModel.from_pretrained('ArthurZ/tiny-random-bert-sharded'						)
       with RequestCounter() as counter:
             lowerCamelCase_	  =					TFAutoModel.from_pretrained('ArthurZ/tiny-random-bert-sharded'						)
             self.assertEqual(counter.get_request_count		,							0						)
             self.assertEqual(counter.head_request_count		,							1						)
             self.assertEqual(counter.other_request_count		,							0						)



