


"""simple docstring"""





import pickle
import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, XLMRobertaTokenizer, XLMRobertaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


snake_case__		:     Dict    				=				get_tests_dir('''fixtures/test_sentencepiece.model''')



@require_sentencepiece
@require_tokenizers
class 		snake_case_(       a__							,					unittest.TestCase			):
     __UpperCamelCase  =					XLMRobertaTokenizer
     __UpperCamelCase  =					XLMRobertaTokenizerFast
     __UpperCamelCase  =					True
     __UpperCamelCase  =					True






     def      lowerCamelCase__     (	self  :					Dict				):
           super().setUp()

           # We have a SentencePiece fixture for testing
           lowerCAmelCase					:	Dict       =   XLMRobertaTokenizer(UpperCamelCase_					,				keep_accents=UpperCamelCase_				)
           tokenizer.save_pretrained(self.tmpdirname				)






     def      lowerCamelCase__     (	self  :					Any				):
           lowerCAmelCase					:	Any       =   '''<pad>'''
           lowerCAmelCase					:	List[Any]       =   1

           self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase_				)					,				UpperCamelCase_				)
           self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase_				)					,				UpperCamelCase_				)






     def      lowerCamelCase__     (	self  :					Optional[int]				):
           lowerCAmelCase					:	Tuple       =   list(self.get_tokenizer().get_vocab().keys()				)

           self.assertEqual(vocab_keys[0]					,				'''<s>'''				)
           self.assertEqual(vocab_keys[1]					,				'''<pad>'''				)
           self.assertEqual(vocab_keys[-1]					,				'''<mask>'''				)
           self.assertEqual(len(UpperCamelCase_				)					,				1_0_0_2				)






     def      lowerCamelCase__     (	self  :					Tuple				):
           self.assertEqual(self.get_tokenizer().vocab_size					,				1_0_0_2				)






     def      lowerCamelCase__     (	self  :					Union[str, Any]				):
           lowerCAmelCase					:	Tuple       =   XLMRobertaTokenizer(UpperCamelCase_					,				keep_accents=UpperCamelCase_				)

           lowerCAmelCase					:	Optional[int]       =   tokenizer.tokenize('''This is a test'''				)
           self.assertListEqual(UpperCamelCase_					,				['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']				)

           self.assertListEqual(
               tokenizer.convert_tokens_to_ids(UpperCamelCase_				)					,				[value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]					,				)

           lowerCAmelCase					:	Tuple       =   tokenizer.tokenize('''I was born in 92000, and this is falsé.'''				)
           self.assertListEqual(
               UpperCamelCase_					,				[
                   SPIECE_UNDERLINE + '''I''',
                   SPIECE_UNDERLINE + '''was''',
                   SPIECE_UNDERLINE + '''b''',
                   '''or''',
                   '''n''',
                   SPIECE_UNDERLINE + '''in''',
                   SPIECE_UNDERLINE + '''''',
                   '''9''',
                   '''2''',
                   '''0''',
                   '''0''',
                   '''0''',
                   ''',''',
                   SPIECE_UNDERLINE + '''and''',
                   SPIECE_UNDERLINE + '''this''',
                   SPIECE_UNDERLINE + '''is''',
                   SPIECE_UNDERLINE + '''f''',
                   '''al''',
                   '''s''',
                   '''é''',
                   '''.''',
               ]					,				)
           lowerCAmelCase					:	Any       =   tokenizer.convert_tokens_to_ids(UpperCamelCase_				)
           self.assertListEqual(
               UpperCamelCase_					,				[
                   value + tokenizer.fairseq_offset
                   for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
                   #                                       ^ unk: 2 + 1 = 3                  unk: 2 + 1 = 3 ^
               ]					,				)

           lowerCAmelCase					:	List[Any]       =   tokenizer.convert_ids_to_tokens(UpperCamelCase_				)
           self.assertListEqual(
               UpperCamelCase_					,				[
                   SPIECE_UNDERLINE + '''I''',
                   SPIECE_UNDERLINE + '''was''',
                   SPIECE_UNDERLINE + '''b''',
                   '''or''',
                   '''n''',
                   SPIECE_UNDERLINE + '''in''',
                   SPIECE_UNDERLINE + '''''',
                   '''<unk>''',
                   '''2''',
                   '''0''',
                   '''0''',
                   '''0''',
                   ''',''',
                   SPIECE_UNDERLINE + '''and''',
                   SPIECE_UNDERLINE + '''this''',
                   SPIECE_UNDERLINE + '''is''',
                   SPIECE_UNDERLINE + '''f''',
                   '''al''',
                   '''s''',
                   '''<unk>''',
                   '''.''',
               ]					,				)






     def      lowerCamelCase__     (	self  :					List[str]				):
           if not self.test_slow_tokenizer:
                 # as we don't have a slow version, we can't compare the outputs between slow and fast versions
                 return

           lowerCAmelCase					:	List[str]       =   (self.rust_tokenizer_class, '''hf-internal-testing/tiny-xlm-roberta''', {})
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''				):
                       lowerCAmelCase					:	List[str]       =   self.rust_tokenizer_class.from_pretrained(UpperCamelCase_					,				**UpperCamelCase_				)
                       lowerCAmelCase					:	List[Any]       =   self.tokenizer_class.from_pretrained(UpperCamelCase_					,				**UpperCamelCase_				)

                       lowerCAmelCase					:	Optional[int]       =   tempfile.mkdtemp()

                       lowerCAmelCase					:	Union[str, Any]       =   tokenizer_r.save_pretrained(UpperCamelCase_				)
                       lowerCAmelCase					:	Union[str, Any]       =   tokenizer_p.save_pretrained(UpperCamelCase_				)

                       # Checks it save with the same files + the tokenizer.json file for the fast one
                       self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files				)				)
                       lowerCAmelCase					:	Tuple       =   tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f				)
                       self.assertSequenceEqual(UpperCamelCase_					,				UpperCamelCase_				)

                       # Checks everything loads correctly in the same way
                       lowerCAmelCase					:	Any       =   tokenizer_r.from_pretrained(UpperCamelCase_				)
                       lowerCAmelCase					:	Optional[int]       =   tokenizer_p.from_pretrained(UpperCamelCase_				)

                       # Check special tokens are set accordingly on Rust and Python
                       for key in tokenizer_pp.special_tokens_map:
                             self.assertTrue(hasattr(UpperCamelCase_					,				UpperCamelCase_				)				)
                             # self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
                             # self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

                       shutil.rmtree(UpperCamelCase_				)

                       # Save tokenizer rust, legacy_format=True
                       lowerCAmelCase					:	Union[str, Any]       =   tempfile.mkdtemp()

                       lowerCAmelCase					:	Optional[Any]       =   tokenizer_r.save_pretrained(UpperCamelCase_					,				legacy_format=UpperCamelCase_				)
                       lowerCAmelCase					:	str       =   tokenizer_p.save_pretrained(UpperCamelCase_				)

                       # Checks it save with the same files
                       self.assertSequenceEqual(UpperCamelCase_					,				UpperCamelCase_				)

                       # Checks everything loads correctly in the same way
                       lowerCAmelCase					:	int       =   tokenizer_r.from_pretrained(UpperCamelCase_				)
                       lowerCAmelCase					:	List[Any]       =   tokenizer_p.from_pretrained(UpperCamelCase_				)

                       # Check special tokens are set accordingly on Rust and Python
                       for key in tokenizer_pp.special_tokens_map:
                             self.assertTrue(hasattr(UpperCamelCase_					,				UpperCamelCase_				)				)

                       shutil.rmtree(UpperCamelCase_				)

                       # Save tokenizer rust, legacy_format=False
                       lowerCAmelCase					:	Dict       =   tempfile.mkdtemp()

                       lowerCAmelCase					:	Any       =   tokenizer_r.save_pretrained(UpperCamelCase_					,				legacy_format=UpperCamelCase_				)
                       lowerCAmelCase					:	List[Any]       =   tokenizer_p.save_pretrained(UpperCamelCase_				)

                       # Checks it saved the tokenizer.json file
                       self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files				)				)

                       # Checks everything loads correctly in the same way
                       lowerCAmelCase					:	str       =   tokenizer_r.from_pretrained(UpperCamelCase_				)
                       lowerCAmelCase					:	Union[str, Any]       =   tokenizer_p.from_pretrained(UpperCamelCase_				)

                       # Check special tokens are set accordingly on Rust and Python
                       for key in tokenizer_pp.special_tokens_map:
                             self.assertTrue(hasattr(UpperCamelCase_					,				UpperCamelCase_				)				)

                       shutil.rmtree(UpperCamelCase_				)






     @cached_property
     def      lowerCamelCase__     (	self  :					List[str]				):
           return XLMRobertaTokenizer.from_pretrained('''xlm-roberta-base'''				)






     def      lowerCamelCase__     (	self  :					Optional[Any]				):
           with tempfile.NamedTemporaryFile() as f:
                 shutil.copyfile(UpperCamelCase_					,				f.name				)
                 lowerCAmelCase					:	List[str]       =   XLMRobertaTokenizer(f.name					,				keep_accents=UpperCamelCase_				)
                 lowerCAmelCase					:	Dict       =   pickle.dumps(UpperCamelCase_				)
           pickle.loads(UpperCamelCase_				)






     def      lowerCamelCase__     (	self  :					int				):
           if not self.test_rust_tokenizer:
                 return

           lowerCAmelCase					:	Dict       =   self.get_tokenizer()
           lowerCAmelCase					:	Union[str, Any]       =   self.get_rust_tokenizer()

           lowerCAmelCase					:	Optional[Any]       =   '''I was born in 92000, and this is falsé.'''

           lowerCAmelCase					:	Dict       =   tokenizer.tokenize(UpperCamelCase_				)
           lowerCAmelCase					:	List[str]       =   rust_tokenizer.tokenize(UpperCamelCase_				)
           self.assertListEqual(UpperCamelCase_					,				UpperCamelCase_				)

           lowerCAmelCase					:	List[Any]       =   tokenizer.encode(UpperCamelCase_					,				add_special_tokens=UpperCamelCase_				)
           lowerCAmelCase					:	Dict       =   rust_tokenizer.encode(UpperCamelCase_					,				add_special_tokens=UpperCamelCase_				)
           self.assertListEqual(UpperCamelCase_					,				UpperCamelCase_				)

           lowerCAmelCase					:	Tuple       =   self.get_rust_tokenizer()
           lowerCAmelCase					:	Dict       =   tokenizer.encode(UpperCamelCase_				)
           lowerCAmelCase					:	Optional[Any]       =   rust_tokenizer.encode(UpperCamelCase_				)
           self.assertListEqual(UpperCamelCase_					,				UpperCamelCase_				)






     @slow
     def      lowerCamelCase__     (	self  :					str				):
           lowerCAmelCase					:	Dict       =   '''Hello World!'''
           lowerCAmelCase					:	Tuple       =   [0, 3_5_3_7_8, 6_6_6_1, 3_8, 2]
           # xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
           # xlmr.eval()
           # xlmr.encode(symbols)

           self.assertListEqual(UpperCamelCase_					,				self.big_tokenizer.encode(UpperCamelCase_				)				)






     @slow
     def      lowerCamelCase__     (	self  :					Optional[int]				):
           lowerCAmelCase					:	List[Any]       =   (
               '''This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'''
               ''' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'''
           )
           lowerCAmelCase					:	Tuple       =   [
               0,
               3_2_9_3,
               8_3,
               1_0,
               4_5_5_2,
               4_9_8_9,
               7_9_8_6,
               6_7_8,
               1_0,
               5_9_1_5,
               1_1_1,
               1_7_9_4_5_9,
               1_2_4_8_5_0,
               4,
               6_0_4_4,
               2_3_7,
               1_2,
               6,
               5,
               6,
               4,
               6_7_8_0,
               7_0_5,
               1_5,
               1_3_8_8,
               4_4,
               3_7_8,
               1_0_1_1_4,
               7_1_1,
               1_5_2,
               2_0,
               6,
               5,
               2_2_3_7_6,
               6_4_2,
               1_2_2_1,
               1_5_1_9_0,
               3_4_1_5_3,
               4_5_0,
               5_6_0_8,
               9_5_9,
               1_1_1_9,
               5_7_7_0_2,
               1_3_6,
               1_8_6,
               4_7,
               1_0_9_8,
               2_9_3_6_7,
               4_7,
               # 4426, # What fairseq tokenizes from "<unk>": "_<"
               # 3678, # What fairseq tokenizes from "<unk>": "unk"
               # 2740, # What fairseq tokenizes from "<unk>": ">"
               3,  # What we tokenize from "<unk>": "<unk>"
               6,  # Residue from the tokenization: an extra sentencepiece underline
               4,
               6_0_4_4,
               2_3_7,
               6_2_8_4,
               5_0_9_0_1,
               5_2_8,
               3_1,
               9_0,
               3_4,
               9_2_7,
               2,
           ]
           # xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
           # xlmr.eval()
           # xlmr.encode(symbols)

           self.assertListEqual(UpperCamelCase_					,				self.big_tokenizer.encode(UpperCamelCase_				)				)






     @slow
     def      lowerCamelCase__     (	self  :					Any				):
           # fmt: off
           lowerCAmelCase					:	List[str]       =   {'''input_ids''': [[0, 1_1_0_6_2, 8_2_7_7_2, 7, 1_5, 8_2_7_7_2, 5_3_8, 5_1_5_2_9, 2_3_7, 1_7_1_9_8, 1_2_9_0, 2_0_6, 9, 2_1_5_1_7_5, 1_3_1_4, 1_3_6, 1_7_1_9_8, 1_2_9_0, 2_0_6, 9, 5_6_3_5_9, 4_2, 1_2_2_0_0_9, 9, 1_6_4_6_6, 1_6, 8_7_3_4_4, 4_5_3_7, 9, 4_7_1_7, 7_8_3_8_1, 6, 1_5_9_9_5_8, 7, 1_5, 2_4_4_8_0, 6_1_8, 4, 5_2_7, 2_2_6_9_3, 5_4_2_8, 4, 2_7_7_7, 2_4_4_8_0, 9_8_7_4, 4, 4_3_5_2_3, 5_9_4, 4, 8_0_3, 1_8_3_9_2, 3_3_1_8_9, 1_8, 4, 4_3_5_2_3, 2_4_4_4_7, 1_2_3_9_9, 1_0_0, 2_4_9_5_5, 8_3_6_5_8, 9_6_2_6, 1_4_4_0_5_7, 1_5, 8_3_9, 2_2_3_3_5, 1_6, 1_3_6, 2_4_9_5_5, 8_3_6_5_8, 8_3_4_7_9, 1_5, 3_9_1_0_2, 7_2_4, 1_6, 6_7_8, 6_4_5, 2_7_8_9, 1_3_2_8, 4_5_8_9, 4_2, 1_2_2_0_0_9, 1_1_5_7_7_4, 2_3, 8_0_5, 1_3_2_8, 4_6_8_7_6, 7, 1_3_6, 5_3_8_9_4, 1_9_4_0, 4_2_2_2_7, 4_1_1_5_9, 1_7_7_2_1, 8_2_3, 4_2_5, 4, 2_7_5_1_2, 9_8_7_2_2, 2_0_6, 1_3_6, 5_5_3_1, 4_9_7_0, 9_1_9, 1_7_3_3_6, 5, 2], [0, 2_0_0_8_0, 6_1_8, 8_3, 8_2_7_7_5, 4_7, 4_7_9, 9, 1_5_1_7, 7_3, 5_3_8_9_4, 3_3_3, 8_0_5_8_1, 1_1_0_1_1_7, 1_8_8_1_1, 5_2_5_6, 1_2_9_5, 5_1, 1_5_2_5_2_6, 2_9_7, 7_9_8_6, 3_9_0, 1_2_4_4_1_6, 5_3_8, 3_5_4_3_1, 2_1_4, 9_8, 1_5_0_4_4, 2_5_7_3_7, 1_3_6, 7_1_0_8, 4_3_7_0_1, 2_3, 7_5_6, 1_3_5_3_5_5, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 5_8_1, 6_3_7_7_3, 1_1_9_4_5_5, 6, 1_4_7_7_9_7, 8_8_2_0_3, 7, 6_4_5, 7_0, 2_1, 3_2_8_5, 1_0_2_6_9, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
           # fmt: on

           self.tokenizer_integration_test_util(
               expected_encoding=UpperCamelCase_					,				model_name='''xlm-roberta-base'''					,				revision='''d9d8a8ea5eb94b1c6654ae9249df7793cd2933d3'''					,				)






"""simple docstring"""





import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / '''utils'''))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
 from test_module.custom_tokenization_fast import CustomTokenizerFast



class 		snake_case_(       unittest.TestCase			):






     def      lowerCamelCase__     (	self  :					Optional[int]				):
           lowerCAmelCase					:	Optional[Any]       =   0






     @slow
     def      lowerCamelCase__     (	self  :					Dict				):
           for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
                 lowerCAmelCase					:	Optional[Any]       =   AutoTokenizer.from_pretrained(UpperCamelCase_				)
                 self.assertIsNotNone(UpperCamelCase_				)
                 self.assertIsInstance(UpperCamelCase_					,				(BertTokenizer, BertTokenizerFast)				)
                 self.assertGreater(len(UpperCamelCase_				)					,				0				)

           for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
                 lowerCAmelCase					:	Tuple       =   AutoTokenizer.from_pretrained(UpperCamelCase_				)
                 self.assertIsNotNone(UpperCamelCase_				)
                 self.assertIsInstance(UpperCamelCase_					,				(GPTaTokenizer, GPTaTokenizerFast)				)
                 self.assertGreater(len(UpperCamelCase_				)					,				0				)






     def      lowerCamelCase__     (	self  :					Union[str, Any]				):
           lowerCAmelCase					:	Optional[Any]       =   AutoTokenizer.from_pretrained(UpperCamelCase_				)
           self.assertIsInstance(UpperCamelCase_					,				(BertTokenizer, BertTokenizerFast)				)
           self.assertEqual(tokenizer.vocab_size					,				1_2				)






     def      lowerCamelCase__     (	self  :					Dict				):
           lowerCAmelCase					:	Tuple       =   AutoTokenizer.from_pretrained(UpperCamelCase_				)
           self.assertIsInstance(UpperCamelCase_					,				(RobertaTokenizer, RobertaTokenizerFast)				)
           self.assertEqual(tokenizer.vocab_size					,				2_0				)






     def      lowerCamelCase__     (	self  :					Dict				):
           lowerCAmelCase					:	int       =   AutoConfig.from_pretrained(UpperCamelCase_				)
           self.assertIsInstance(UpperCamelCase_					,				UpperCamelCase_				)
           # Check that tokenizer_type ≠ model_type
           lowerCAmelCase					:	List[Any]       =   AutoTokenizer.from_pretrained(UpperCamelCase_					,				config=UpperCamelCase_				)
           self.assertIsInstance(UpperCamelCase_					,				(BertTokenizer, BertTokenizerFast)				)
           self.assertEqual(tokenizer.vocab_size					,				1_2				)






     def      lowerCamelCase__     (	self  :					Any				):
           with tempfile.TemporaryDirectory() as tmp_dir:
                 shutil.copy('''./tests/fixtures/vocab.txt'''					,				os.path.join(UpperCamelCase_					,				'''vocab.txt'''				)				)

                 lowerCAmelCase					:	Any       =   AutoTokenizer.from_pretrained(UpperCamelCase_					,				tokenizer_type='''bert'''					,				use_fast=UpperCamelCase_				)
                 self.assertIsInstance(UpperCamelCase_					,				UpperCamelCase_				)

           with tempfile.TemporaryDirectory() as tmp_dir:
                 shutil.copy('''./tests/fixtures/vocab.json'''					,				os.path.join(UpperCamelCase_					,				'''vocab.json'''				)				)
                 shutil.copy('''./tests/fixtures/merges.txt'''					,				os.path.join(UpperCamelCase_					,				'''merges.txt'''				)				)

                 lowerCAmelCase					:	List[Any]       =   AutoTokenizer.from_pretrained(UpperCamelCase_					,				tokenizer_type='''gpt2'''					,				use_fast=UpperCamelCase_				)
                 self.assertIsInstance(UpperCamelCase_					,				UpperCamelCase_				)






     @require_tokenizers
     def      lowerCamelCase__     (	self  :					Union[str, Any]				):
           with tempfile.TemporaryDirectory() as tmp_dir:
                 shutil.copy('''./tests/fixtures/vocab.txt'''					,				os.path.join(UpperCamelCase_					,				'''vocab.txt'''				)				)

                 lowerCAmelCase					:	Optional[Any]       =   AutoTokenizer.from_pretrained(UpperCamelCase_					,				tokenizer_type='''bert'''				)
                 self.assertIsInstance(UpperCamelCase_					,				UpperCamelCase_				)

           with tempfile.TemporaryDirectory() as tmp_dir:
                 shutil.copy('''./tests/fixtures/vocab.json'''					,				os.path.join(UpperCamelCase_					,				'''vocab.json'''				)				)
                 shutil.copy('''./tests/fixtures/merges.txt'''					,				os.path.join(UpperCamelCase_					,				'''merges.txt'''				)				)

                 lowerCAmelCase					:	int       =   AutoTokenizer.from_pretrained(UpperCamelCase_					,				tokenizer_type='''gpt2'''				)
                 self.assertIsInstance(UpperCamelCase_					,				UpperCamelCase_				)






     def      lowerCamelCase__     (	self  :					Dict				):
           with pytest.raises(UpperCamelCase_				):
                 AutoTokenizer.from_pretrained('''./'''					,				tokenizer_type='''xxx'''				)






     @require_tokenizers
     def      lowerCamelCase__     (	self  :					str				):
           for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
                 lowerCAmelCase					:	Dict       =   tokenizer_class.from_pretrained('''wietsedv/bert-base-dutch-cased'''				)
                 self.assertIsInstance(UpperCamelCase_					,				(BertTokenizer, BertTokenizerFast)				)

                 if isinstance(UpperCamelCase_					,				UpperCamelCase_				):
                       self.assertEqual(tokenizer.basic_tokenizer.do_lower_case					,				UpperCamelCase_				)
                 else:
                       self.assertEqual(tokenizer.do_lower_case					,				UpperCamelCase_				)

                 self.assertEqual(tokenizer.model_max_length					,				5_1_2				)






     @require_tokenizers
     def      lowerCamelCase__     (	self  :					Optional[int]				):
           for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
                 with self.assertRaisesRegex(
                     UpperCamelCase_					,				'''julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier'''					,				):
                       lowerCAmelCase					:	Any       =   tokenizer_class.from_pretrained('''julien-c/herlolip-not-exists'''				)






     def      lowerCamelCase__     (	self  :					Tuple				):
           # tests: https://github.com/huggingface/transformers/pull/13251
           # 1. models with `-`, e.g. xlm-roberta -> xlm_roberta
           # 2. models that don't remap 1-1 from model-name to model file, e.g., openai-gpt -> openai
           lowerCAmelCase					:	Optional[Any]       =   TOKENIZER_MAPPING.values()
           lowerCAmelCase					:	Optional[Any]       =   []

           for slow_tok, fast_tok in tokenizers:
                 if slow_tok is not None:
                       tokenizer_names.append(slow_tok.__name__				)

                 if fast_tok is not None:
                       tokenizer_names.append(fast_tok.__name__				)

           for tokenizer_name in tokenizer_names:
                 # must find the right class
                 tokenizer_class_from_name(UpperCamelCase_				)






     @require_tokenizers
     def      lowerCamelCase__     (	self  :					Any				):
           self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased'''					,				use_fast=UpperCamelCase_				)					,				UpperCamelCase_				)
           self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased'''				)					,				UpperCamelCase_				)






     @require_tokenizers
     def      lowerCamelCase__     (	self  :					Dict				):
           lowerCAmelCase					:	List[Any]       =   AutoTokenizer.from_pretrained('''distilbert-base-uncased'''					,				do_lower_case=UpperCamelCase_				)
           lowerCAmelCase					:	Union[str, Any]       =   '''Hello, world. How are you?'''
           lowerCAmelCase					:	Optional[Any]       =   tokenizer.tokenize(UpperCamelCase_				)
           self.assertEqual('''[UNK]'''					,				tokens[0]				)

           lowerCAmelCase					:	List[str]       =   AutoTokenizer.from_pretrained('''microsoft/mpnet-base'''					,				do_lower_case=UpperCamelCase_				)
           lowerCAmelCase					:	Optional[int]       =   tokenizer.tokenize(UpperCamelCase_				)
           self.assertEqual('''[UNK]'''					,				tokens[0]				)






     @require_tokenizers
     def      lowerCamelCase__     (	self  :					int				):
           lowerCAmelCase					:	Union[str, Any]       =   AutoTokenizer.from_pretrained('''robot-test/dummy-tokenizer-fast-with-model-config'''				)
           self.assertEqual(type(UpperCamelCase_				)					,				UpperCamelCase_				)
           self.assertEqual(tokenizer.model_max_length					,				5_1_2				)
           self.assertEqual(tokenizer.vocab_size					,				3_0_0_0_0				)
           self.assertEqual(tokenizer.unk_token					,				'''[UNK]'''				)
           self.assertEqual(tokenizer.padding_side					,				'''right'''				)
           self.assertEqual(tokenizer.truncation_side					,				'''right'''				)






     def      lowerCamelCase__     (	self  :					List[Any]				):
           lowerCAmelCase					:	int       =   AutoTokenizer.from_pretrained(UpperCamelCase_				)
           self.assertIsInstance(UpperCamelCase_					,				(BertTokenizer, BertTokenizerFast)				)
           with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(UpperCamelCase_				)
                 lowerCAmelCase					:	List[Any]       =   AutoTokenizer.from_pretrained(UpperCamelCase_				)

           self.assertIsInstance(UpperCamelCase_					,				tokenizer.__class__				)
           self.assertEqual(tokenizera.vocab_size					,				1_2				)






     def      lowerCamelCase__     (	self  :					List[str]				):
           lowerCAmelCase					:	List[Any]       =   AutoTokenizer.from_pretrained('''ctrl'''				)
           # There is no fast CTRL so this always gives us a slow tokenizer.
           self.assertIsInstance(UpperCamelCase_					,				UpperCamelCase_				)






     def      lowerCamelCase__     (	self  :					Dict				):
           # Check we can load the tokenizer config of an online model.
           lowerCAmelCase					:	Any       =   get_tokenizer_config('''bert-base-cased'''				)
           lowerCAmelCase					:	Optional[int]       =   config.pop('''_commit_hash'''					,				UpperCamelCase_				)
           # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
           self.assertEqual(UpperCamelCase_					,				{'''do_lower_case''': False}				)

           # This model does not have a tokenizer_config so we get back an empty dict.
           lowerCAmelCase					:	Union[str, Any]       =   get_tokenizer_config(UpperCamelCase_				)
           self.assertDictEqual(UpperCamelCase_					,				{}				)

           # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
           lowerCAmelCase					:	List[Any]       =   AutoTokenizer.from_pretrained(UpperCamelCase_				)
           with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(UpperCamelCase_				)
                 lowerCAmelCase					:	Dict       =   get_tokenizer_config(UpperCamelCase_				)

           # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
           self.assertEqual(config['''tokenizer_class''']					,				'''BertTokenizer'''				)






     def      lowerCamelCase__     (	self  :					Optional[int]				):
           try:
                 AutoConfig.register('''custom'''					,				UpperCamelCase_				)

                 AutoTokenizer.register(UpperCamelCase_					,				slow_tokenizer_class=UpperCamelCase_				)
                 # Trying to register something existing in the Transformers library will raise an error
                 with self.assertRaises(UpperCamelCase_				):
                       AutoTokenizer.register(UpperCamelCase_					,				slow_tokenizer_class=UpperCamelCase_				)

                 lowerCAmelCase					:	Union[str, Any]       =   CustomTokenizer.from_pretrained(UpperCamelCase_				)
                 with tempfile.TemporaryDirectory() as tmp_dir:
                       tokenizer.save_pretrained(UpperCamelCase_				)

                       lowerCAmelCase					:	Tuple       =   AutoTokenizer.from_pretrained(UpperCamelCase_				)
                       self.assertIsInstance(UpperCamelCase_					,				UpperCamelCase_				)

           finally:
                 if "custom" in CONFIG_MAPPING._extra_content:
                       del CONFIG_MAPPING._extra_content["custom"]
                 if CustomConfig in TOKENIZER_MAPPING._extra_content:
                       del TOKENIZER_MAPPING._extra_content[CustomConfig]






     @require_tokenizers
     def      lowerCamelCase__     (	self  :					str				):
           try:
                 AutoConfig.register('''custom'''					,				UpperCamelCase_				)

                 # Can register in two steps
                 AutoTokenizer.register(UpperCamelCase_					,				slow_tokenizer_class=UpperCamelCase_				)
                 self.assertEqual(TOKENIZER_MAPPING[CustomConfig]					,				(CustomTokenizer, None)				)
                 AutoTokenizer.register(UpperCamelCase_					,				fast_tokenizer_class=UpperCamelCase_				)
                 self.assertEqual(TOKENIZER_MAPPING[CustomConfig]					,				(CustomTokenizer, CustomTokenizerFast)				)

                 del TOKENIZER_MAPPING._extra_content[CustomConfig]
                 # Can register in one step
                 AutoTokenizer.register(
                     UpperCamelCase_					,				slow_tokenizer_class=UpperCamelCase_					,				fast_tokenizer_class=UpperCamelCase_				)
                 self.assertEqual(TOKENIZER_MAPPING[CustomConfig]					,				(CustomTokenizer, CustomTokenizerFast)				)

                 # Trying to register something existing in the Transformers library will raise an error
                 with self.assertRaises(UpperCamelCase_				):
                       AutoTokenizer.register(UpperCamelCase_					,				fast_tokenizer_class=UpperCamelCase_				)

                 # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
                 # and that model does not have a tokenizer.json
                 with tempfile.TemporaryDirectory() as tmp_dir:
                       lowerCAmelCase					:	Dict       =   BertTokenizerFast.from_pretrained(UpperCamelCase_				)
                       bert_tokenizer.save_pretrained(UpperCamelCase_				)
                       lowerCAmelCase					:	int       =   CustomTokenizerFast.from_pretrained(UpperCamelCase_				)

                 with tempfile.TemporaryDirectory() as tmp_dir:
                       tokenizer.save_pretrained(UpperCamelCase_				)

                       lowerCAmelCase					:	Optional[int]       =   AutoTokenizer.from_pretrained(UpperCamelCase_				)
                       self.assertIsInstance(UpperCamelCase_					,				UpperCamelCase_				)

                       lowerCAmelCase					:	List[str]       =   AutoTokenizer.from_pretrained(UpperCamelCase_					,				use_fast=UpperCamelCase_				)
                       self.assertIsInstance(UpperCamelCase_					,				UpperCamelCase_				)

           finally:
                 if "custom" in CONFIG_MAPPING._extra_content:
                       del CONFIG_MAPPING._extra_content["custom"]
                 if CustomConfig in TOKENIZER_MAPPING._extra_content:
                       del TOKENIZER_MAPPING._extra_content[CustomConfig]






     def      lowerCamelCase__     (	self  :					Optional[int]				):
           # If remote code is not set, we will time out when asking whether to load the model.
           with self.assertRaises(UpperCamelCase_				):
                 lowerCAmelCase					:	int       =   AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''				)
           # If remote code is disabled, we can't load this config.
           with self.assertRaises(UpperCamelCase_				):
                 lowerCAmelCase					:	str       =   AutoTokenizer.from_pretrained(
                     '''hf-internal-testing/test_dynamic_tokenizer'''					,				trust_remote_code=UpperCamelCase_				)

           lowerCAmelCase					:	List[str]       =   AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''					,				trust_remote_code=UpperCamelCase_				)
           self.assertTrue(tokenizer.special_attribute_present				)
           # Test tokenizer can be reloaded.
           with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(UpperCamelCase_				)
                 lowerCAmelCase					:	Optional[Any]       =   AutoTokenizer.from_pretrained(UpperCamelCase_					,				trust_remote_code=UpperCamelCase_				)
           self.assertTrue(reloaded_tokenizer.special_attribute_present				)

           if is_tokenizers_available():
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizerFast'''				)
                 self.assertEqual(reloaded_tokenizer.__class__.__name__					,				'''NewTokenizerFast'''				)

                 # Test we can also load the slow version
                 lowerCAmelCase					:	Union[str, Any]       =   AutoTokenizer.from_pretrained(
                     '''hf-internal-testing/test_dynamic_tokenizer'''					,				trust_remote_code=UpperCamelCase_					,				use_fast=UpperCamelCase_				)
                 self.assertTrue(tokenizer.special_attribute_present				)
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizer'''				)
                 # Test tokenizer can be reloaded.
                 with tempfile.TemporaryDirectory() as tmp_dir:
                       tokenizer.save_pretrained(UpperCamelCase_				)
                       lowerCAmelCase					:	List[str]       =   AutoTokenizer.from_pretrained(UpperCamelCase_					,				trust_remote_code=UpperCamelCase_					,				use_fast=UpperCamelCase_				)
                 self.assertEqual(reloaded_tokenizer.__class__.__name__					,				'''NewTokenizer'''				)
                 self.assertTrue(reloaded_tokenizer.special_attribute_present				)
           else:
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizer'''				)
                 self.assertEqual(reloaded_tokenizer.__class__.__name__					,				'''NewTokenizer'''				)






     @require_tokenizers
     def      lowerCamelCase__     (	self  :					Optional[int]				):



           class 		snake_case_(       a__			):
                __UpperCamelCase  =					False



           class 		snake_case_(       a__			):
                __UpperCamelCase  =					NewTokenizer
                __UpperCamelCase  =					False

           try:
                 AutoConfig.register('''custom'''					,				UpperCamelCase_				)
                 AutoTokenizer.register(UpperCamelCase_					,				slow_tokenizer_class=UpperCamelCase_				)
                 AutoTokenizer.register(UpperCamelCase_					,				fast_tokenizer_class=UpperCamelCase_				)
                 # If remote code is not set, the default is to use local
                 lowerCAmelCase					:	Optional[int]       =   AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''				)
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizerFast'''				)
                 self.assertFalse(tokenizer.special_attribute_present				)
                 lowerCAmelCase					:	str       =   AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''					,				use_fast=UpperCamelCase_				)
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizer'''				)
                 self.assertFalse(tokenizer.special_attribute_present				)

                 # If remote code is disabled, we load the local one.
                 lowerCAmelCase					:	Union[str, Any]       =   AutoTokenizer.from_pretrained(
                     '''hf-internal-testing/test_dynamic_tokenizer'''					,				trust_remote_code=UpperCamelCase_				)
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizerFast'''				)
                 self.assertFalse(tokenizer.special_attribute_present				)
                 lowerCAmelCase					:	Dict       =   AutoTokenizer.from_pretrained(
                     '''hf-internal-testing/test_dynamic_tokenizer'''					,				trust_remote_code=UpperCamelCase_					,				use_fast=UpperCamelCase_				)
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizer'''				)
                 self.assertFalse(tokenizer.special_attribute_present				)

                 # If remote is enabled, we load from the Hub
                 lowerCAmelCase					:	int       =   AutoTokenizer.from_pretrained(
                     '''hf-internal-testing/test_dynamic_tokenizer'''					,				trust_remote_code=UpperCamelCase_				)
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizerFast'''				)
                 self.assertTrue(tokenizer.special_attribute_present				)
                 lowerCAmelCase					:	int       =   AutoTokenizer.from_pretrained(
                     '''hf-internal-testing/test_dynamic_tokenizer'''					,				trust_remote_code=UpperCamelCase_					,				use_fast=UpperCamelCase_				)
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizer'''				)
                 self.assertTrue(tokenizer.special_attribute_present				)

           finally:
                 if "custom" in CONFIG_MAPPING._extra_content:
                       del CONFIG_MAPPING._extra_content["custom"]
                 if CustomConfig in TOKENIZER_MAPPING._extra_content:
                       del TOKENIZER_MAPPING._extra_content[CustomConfig]






     def      lowerCamelCase__     (	self  :					Tuple				):
           lowerCAmelCase					:	str       =   AutoTokenizer.from_pretrained(
               '''hf-internal-testing/test_dynamic_tokenizer_legacy'''					,				trust_remote_code=UpperCamelCase_				)
           self.assertTrue(tokenizer.special_attribute_present				)
           if is_tokenizers_available():
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizerFast'''				)

                 # Test we can also load the slow version
                 lowerCAmelCase					:	List[str]       =   AutoTokenizer.from_pretrained(
                     '''hf-internal-testing/test_dynamic_tokenizer_legacy'''					,				trust_remote_code=UpperCamelCase_					,				use_fast=UpperCamelCase_				)
                 self.assertTrue(tokenizer.special_attribute_present				)
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizer'''				)
           else:
                 self.assertEqual(tokenizer.__class__.__name__					,				'''NewTokenizer'''				)






     def      lowerCamelCase__     (	self  :					str				):
           with self.assertRaisesRegex(
               UpperCamelCase_					,				'''bert-base is not a local folder and is not a valid model identifier'''				):
                 lowerCAmelCase					:	List[str]       =   AutoTokenizer.from_pretrained('''bert-base'''				)






     def      lowerCamelCase__     (	self  :					int				):
           with self.assertRaisesRegex(
               UpperCamelCase_					,				r'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'''				):
                 lowerCAmelCase					:	List[Any]       =   AutoTokenizer.from_pretrained(UpperCamelCase_					,				revision='''aaaaaa'''				)






     def      lowerCamelCase__     (	self  :					Optional[int]				):
           # Make sure we have cached the tokenizer.
           lowerCAmelCase					:	List[str]       =   AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''				)
           with RequestCounter() as counter:
                 lowerCAmelCase					:	int       =   AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''				)
                 self.assertEqual(counter.get_request_count					,				0				)
                 self.assertEqual(counter.head_request_count					,				1				)
                 self.assertEqual(counter.other_request_count					,				0				)


