





import argparse
import os
import re


_lowerCamelCase			       =      'src/diffusers'

# Pattern that looks at the indentation in a line.
_lowerCamelCase			       =      re.compile(r'^(\s*)\S')
# Pattern that matches `"key":" and puts `key` in group 0.
_lowerCamelCase			       =      re.compile(r'^\s*"([^"]+)":')
# Pattern that matches `_import_structure["key"]` and puts `key` in group 0.
_lowerCamelCase			       =      re.compile(r'^\s*_import_structure\["([^"]+)"\]')
# Pattern that matches `"key",` and puts `key` in group 0.
_lowerCamelCase			       =      re.compile(r'^\s*"([^"]+)",\s*$')
# Pattern that matches any `[stuff]` and puts `stuff` in group 0.
_lowerCamelCase			       =      re.compile(r'\[([^\]]+)\]')




def    __UpperCAmelCase(      lowercase_      ):
       _lowerCamelCase							:     int	=		_re_indent.search(lowercase_      )
       return "" if search is None else search.groups()[0]




def    __UpperCAmelCase(      lowercase_ ,    lowercase_="" ,    lowercase_=None ,    lowercase_=None      ):
       _lowerCamelCase							:     Any	=		0
       _lowerCamelCase							:     Any	=		code.split('''\n'''      )
       if start_prompt is not None:
              while not lines[index].startswith(lowercase_      ):
                     index += 1
              _lowerCamelCase							:     Tuple	=		['''\n'''.join(lines[:index]      )]
       else:
              _lowerCamelCase							:     Union[str, Any]	=		[]

       # We split into blocks until we get to the `end_prompt` (or the end of the block).
       _lowerCamelCase							:     Dict	=		[lines[index]]
       index += 1
       while index < len(lowercase_      ) and (end_prompt is None or not lines[index].startswith(lowercase_      )):
              if len(lines[index]      ) > 0 and get_indent(lines[index]      ) == indent_level:
                     if len(lowercase_      ) > 0 and get_indent(current_block[-1]      ).startswith(indent_level + ''' '''      ):
                            current_block.append(lines[index]      )
                            blocks.append('''\n'''.join(lowercase_      )      )
                            if index < len(lowercase_      ) - 1:
                                   _lowerCamelCase							:     Any	=		[lines[index + 1]]
                                   index += 1
                            else:
                                   _lowerCamelCase							:     Optional[Any]	=		[]
                     else:
                            blocks.append('''\n'''.join(lowercase_      )      )
                            _lowerCamelCase							:     Optional[Any]	=		[lines[index]]
              else:
                     current_block.append(lines[index]      )
              index += 1

       # Adds current block if it's nonempty.
       if len(lowercase_      ) > 0:
              blocks.append('''\n'''.join(lowercase_      )      )

       # Add final block after end_prompt if provided.
       if end_prompt is not None and index < len(lowercase_      ):
              blocks.append('''\n'''.join(lines[index:]      )      )

       return blocks




def    __UpperCAmelCase(      lowercase_      ):

       def _inner(lowercase_      ):
              return key(lowercase_      ).lower().replace('''_''' ,    ''''''      )

       return _inner




def    __UpperCAmelCase(      lowercase_ ,    lowercase_=None      ):

       # If no key is provided, we use a noop.
       def noop(lowercase_      ):
              return x

       if key is None:
              _lowerCamelCase							:     Tuple	=		noop
       # Constants are all uppercase, they go first.
       _lowerCamelCase							:     Any	=		[obj for obj in objects if key(lowercase_      ).isupper()]
       # Classes are not all uppercase but start with a capital, they go second.
       _lowerCamelCase							:     Optional[Any]	=		[obj for obj in objects if key(lowercase_      )[0].isupper() and not key(lowercase_      ).isupper()]
       # Functions begin with a lowercase, they go last.
       _lowerCamelCase							:     int	=		[obj for obj in objects if not key(lowercase_      )[0].isupper()]

       _lowerCamelCase							:     List[str]	=		ignore_underscore(lowercase_      )
       return sorted(lowercase_ ,    key=lowercase_      ) + sorted(lowercase_ ,    key=lowercase_      ) + sorted(lowercase_ ,    key=lowercase_      )




def    __UpperCAmelCase(      lowercase_      ):

       # This inner function sort imports between [ ].
       def _replace(lowercase_      ):
              _lowerCamelCase							:     Optional[int]	=		match.groups()[0]
              if "," not in imports:
                     return F"""[{imports}]"""
              _lowerCamelCase							:     Optional[Any]	=		[part.strip().replace('''"''' ,    ''''''      ) for part in imports.split(''','''      )]
              # We will have a final empty element if the line finished with a comma.
              if len(keys[-1]      ) == 0:
                     _lowerCamelCase							:     str	=		keys[:-1]
              return "[" + ", ".join([F"""\"{k}\"""" for k in sort_objects(lowercase_      )]      ) + "]"

       _lowerCamelCase							:     Tuple	=		import_statement.split('''\n'''      )
       if len(lowercase_      ) > 3:
              # Here we have to sort internal imports that are on several lines (one per name):
              # key: [
              #     "object1",
              #     "object2",
              #     ...
              # ]

              # We may have to ignore one or two lines on each side.
              _lowerCamelCase							:     Optional[int]	=		2 if lines[1].strip() == '''[''' else 1
              _lowerCamelCase							:     Optional[Any]	=		[(i, _re_strip_line.search(lowercase_      ).groups()[0]) for i, line in enumerate(lines[idx:-idx]      )]
              _lowerCamelCase							:     Union[str, Any]	=		sort_objects(lowercase_ ,    key=lambda lowercase_      : x[1]      )
              _lowerCamelCase							:     int	=		[lines[x[0] + idx] for x in sorted_indices]
              return "\n".join(lines[:idx] + sorted_lines + lines[-idx:]      )
       elif len(lowercase_      ) == 3:
              # Here we have to sort internal imports that are on one separate line:
              # key: [
              #     "object1", "object2", ...
              # ]
              if _re_bracket_content.search(lines[1]      ) is not None:
                     _lowerCamelCase							:     Dict	=		_re_bracket_content.sub(_replace ,    lines[1]      )
              else:
                     _lowerCamelCase							:     Dict	=		[part.strip().replace('''"''' ,    ''''''      ) for part in lines[1].split(''','''      )]
                     # We will have a final empty element if the line finished with a comma.
                     if len(keys[-1]      ) == 0:
                            _lowerCamelCase							:     Union[str, Any]	=		keys[:-1]
                     _lowerCamelCase							:     Optional[Any]	=		get_indent(lines[1]      ) + ''', '''.join([F"""\"{k}\"""" for k in sort_objects(lowercase_      )]      )
              return "\n".join(lowercase_      )
       else:
              # Finally we have to deal with imports fitting on one line
              _lowerCamelCase							:     str	=		_re_bracket_content.sub(_replace ,    lowercase_      )
              return import_statement




def    __UpperCAmelCase(      lowercase_ ,    lowercase_=True      ):
       with open(lowercase_ ,    '''r'''      ) as f:
              _lowerCamelCase							:     Union[str, Any]	=		f.read()

       if "_import_structure" not in code:
              return

       # Blocks of indent level 0
       _lowerCamelCase							:     Any	=		split_code_in_indented_blocks(
           lowercase_ ,    start_prompt='''_import_structure = {''' ,    end_prompt='''if TYPE_CHECKING:'''      )

       # We ignore block 0 (everything until start_prompt) and the last block (everything after end_prompt).
       for block_idx in range(1 ,    len(lowercase_      ) - 1      ):
              # Check if the block contains some `_import_structure`s thingy to sort.
              _lowerCamelCase							:     Optional[int]	=		main_blocks[block_idx]
              _lowerCamelCase							:     Optional[int]	=		block.split('''\n'''      )

              # Get to the start of the imports.
              _lowerCamelCase							:     Union[str, Any]	=		0
              while line_idx < len(lowercase_      ) and "_import_structure" not in block_lines[line_idx]:
                     # Skip dummy import blocks
                     if "import dummy" in block_lines[line_idx]:
                            _lowerCamelCase							:     Optional[int]	=		len(lowercase_      )
                     else:
                            line_idx += 1
              if line_idx >= len(lowercase_      ):
                     continue

              # Ignore beginning and last line: they don't contain anything.
              _lowerCamelCase							:     Optional[int]	=		'''\n'''.join(block_lines[line_idx:-1]      )
              _lowerCamelCase							:     Dict	=		get_indent(block_lines[1]      )
              # Slit the internal block into blocks of indent level 1.
              _lowerCamelCase							:     Any	=		split_code_in_indented_blocks(lowercase_ ,    indent_level=lowercase_      )
              # We have two categories of import key: list or _import_structure[key].append/extend
              _lowerCamelCase							:     List[str]	=		_re_direct_key if '''_import_structure''' in block_lines[0] else _re_indirect_key
              # Grab the keys, but there is a trap: some lines are empty or just comments.
              _lowerCamelCase							:     Optional[Any]	=		[(pattern.search(lowercase_      ).groups()[0] if pattern.search(lowercase_      ) is not None else None) for b in internal_blocks]
              # We only sort the lines with a key.
              _lowerCamelCase							:     Optional[Any]	=		[(i, key) for i, key in enumerate(lowercase_      ) if key is not None]
              _lowerCamelCase							:     int	=		[x[0] for x in sorted(lowercase_ ,    key=lambda lowercase_      : x[1]      )]

              # We reorder the blocks by leaving empty lines/comments as they were and reorder the rest.
              _lowerCamelCase							:     List[str]	=		0
              _lowerCamelCase							:     Dict	=		[]
              for i in range(len(lowercase_      )      ):
                     if keys[i] is None:
                            reordered_blocks.append(internal_blocks[i]      )
                     else:
                            _lowerCamelCase							:     Optional[int]	=		sort_objects_in_import(internal_blocks[sorted_indices[count]]      )
                            reordered_blocks.append(lowercase_      )
                            count += 1

        # And we put our main block back together with its first and last line.
              _lowerCamelCase							:     List[str]	=		'''\n'''.join(block_lines[:line_idx] + reordered_blocks + [block_lines[-1]]      )

       if code != "\n".join(lowercase_      ):
              if check_only:
                     return True
              else:
                     print(F"""Overwriting {file}."""      )
                     with open(lowercase_ ,    '''w'''      ) as f:
                            f.write('''\n'''.join(lowercase_      )      )




def    __UpperCAmelCase(      lowercase_=True      ):
       _lowerCamelCase							:     Any	=		[]
       for root, _, files in os.walk(lowercase_      ):
              if "__init__.py" in files:
                     _lowerCamelCase							:     List[str]	=		sort_imports(os.path.join(lowercase_ ,    '''__init__.py'''      ) ,    check_only=lowercase_      )
                     if result:
                            _lowerCamelCase							:     Optional[int]	=		[os.path.join(lowercase_ ,    '''__init__.py'''      )]
       if len(lowercase_      ) > 0:
              raise ValueError(F"""Would overwrite {len(lowercase_      )} files, run `make style`."""      )


if __name__ == "__main__":
       _lowerCamelCase			       =      argparse.ArgumentParser()
       parser.add_argument('--check_only', action='store_true', help='Whether to only check or fix style.')
       _lowerCamelCase			       =      parser.parse_args()

       sort_imports_in_all_inits(check_only=args.check_only)








import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin







@custom_tokenizers
class 			__A       (      lowerCamelCase__ ,unittest.TestCase  ):




      """simple docstring"""







      UpperCAmelCase__						=      BertJapaneseTokenizer
      UpperCAmelCase__						=      False
      UpperCAmelCase__						=      True


      def 				__snake_case			(		self):



             """simple docstring"""





             super().setUp()

             _lowerCamelCase							:     Optional[int]	=		[
                 '''[UNK]''',
                 '''[CLS]''',
                 '''[SEP]''',
                 '''こんにちは''',
                 '''こん''',
                 '''にちは''',
                 '''ばんは''',
                 '''##こん''',
                 '''##にちは''',
                 '''##ばんは''',
                 '''世界''',
                 '''##世界''',
                 '''、''',
                 '''##、''',
                 '''。''',
                 '''##。''',
             ]

             _lowerCamelCase							:     Dict	=		os.path.join(self.tmpdirname      ,							VOCAB_FILES_NAMES['''vocab_file'''])
             with open(self.vocab_file      ,							'''w'''      ,							encoding='''utf-8''') as vocab_writer:
                    vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]))


      def 				__snake_case			(		self      ,							a__):



             """simple docstring"""





             _lowerCamelCase							:     Dict	=		'''こんにちは、世界。 \nこんばんは、世界。'''
             _lowerCamelCase							:     Union[str, Any]	=		'''こんにちは 、 世界 。 こんばんは 、 世界 。'''
             return input_text, output_text


      def 				__snake_case			(		self      ,							a__):



             """simple docstring"""





             _lowerCamelCase,  _lowerCamelCase							:     int	=		self.get_input_output_texts(a__)
             _lowerCamelCase							:     Optional[Any]	=		tokenizer.encode(a__      ,							add_special_tokens=a__)
             _lowerCamelCase							:     str	=		tokenizer.decode(a__      ,							clean_up_tokenization_spaces=a__)
             return text, ids


      def 				__snake_case			(		self):



             """simple docstring"""





             pass  # TODO add if relevant


      def 				__snake_case			(		self):



             """simple docstring"""





             pass  # TODO add if relevant


      def 				__snake_case			(		self):



             """simple docstring"""





             pass  # TODO add if relevant


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     int	=		self.tokenizer_class(self.vocab_file)

             _lowerCamelCase							:     Tuple	=		tokenizer.tokenize('''こんにちは、世界。\nこんばんは、世界。''')
             self.assertListEqual(a__      ,							['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。'''])
             self.assertListEqual(tokenizer.convert_tokens_to_ids(a__)      ,							[3, 12, 10, 14, 4, 9, 12, 10, 14])


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     str	=		self.tokenizer_class(self.vocab_file      ,							word_tokenizer_type='''mecab''')
             self.assertIsNotNone(a__)

             _lowerCamelCase							:     List[str]	=		'''こんにちは、世界。\nこんばんは、世界。'''
             _lowerCamelCase							:     int	=		tokenizer.tokenize(a__)
             self.assertListEqual(a__      ,							['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。'''])
             self.assertListEqual(tokenizer.convert_tokens_to_ids(a__)      ,							[3, 12, 10, 14, 4, 9, 12, 10, 14])

             _lowerCamelCase							:     Optional[int]	=		os.path.join(self.tmpdirname      ,							'''tokenizer.bin''')
             with open(a__      ,							'''wb''') as handle:
                    pickle.dump(a__      ,							a__)

             with open(a__      ,							'''rb''') as handle:
                    _lowerCamelCase							:     List[str]	=		pickle.load(a__)

             _lowerCamelCase							:     Dict	=		tokenizer_new.tokenize(a__)

             self.assertListEqual(a__      ,							a__)


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Union[str, Any]	=		MecabTokenizer(mecab_dic='''ipadic''')

             self.assertListEqual(
                 tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ''')      ,							['''アップルストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']      ,							)


      def 				__snake_case			(		self):



             """simple docstring"""





             try:
                    _lowerCamelCase							:     str	=		MecabTokenizer(mecab_dic='''unidic_lite''')
             except ModuleNotFoundError:
                    return

             self.assertListEqual(
                 tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ''')      ,							['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']      ,							)


      def 				__snake_case			(		self):



             """simple docstring"""





             try:
                    _lowerCamelCase							:     int	=		MecabTokenizer(mecab_dic='''unidic''')
             except ModuleNotFoundError:
                    return

             self.assertListEqual(
                 tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ''')      ,							['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']      ,							)


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     List[Any]	=		MecabTokenizer(do_lower_case=a__      ,							mecab_dic='''ipadic''')

             self.assertListEqual(
                 tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ''')      ,							['''アップルストア''', '''で''', '''iphone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']      ,							)


      def 				__snake_case			(		self):



             """simple docstring"""





             try:
                    _lowerCamelCase							:     Any	=		MecabTokenizer(
                        do_lower_case=a__      ,							normalize_text=a__      ,							mecab_option='''-d /usr/local/lib/mecab/dic/jumandic''')
             except RuntimeError:
                    # if dict doesn't exist in the system, previous code raises this error.
                    return

             self.assertListEqual(
                 tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ''')      ,							['''ｱｯﾌﾟﾙストア''', '''で''', '''iPhone''', '''８''', '''が''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']      ,							)


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Optional[int]	=		MecabTokenizer(normalize_text=a__      ,							mecab_dic='''ipadic''')

             self.assertListEqual(
                 tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ''')      ,							['''ｱｯﾌﾟﾙストア''', '''で''', '''iPhone''', '''８''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''　''', '''。''']      ,							)


      @require_sudachi
      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Optional[int]	=		self.tokenizer_class(self.vocab_file      ,							word_tokenizer_type='''sudachi''')
             self.assertIsNotNone(a__)

             _lowerCamelCase							:     Tuple	=		'''こんにちは、世界。\nこんばんは、世界。'''
             _lowerCamelCase							:     Tuple	=		tokenizer.tokenize(a__)
             self.assertListEqual(a__      ,							['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。'''])
             self.assertListEqual(tokenizer.convert_tokens_to_ids(a__)      ,							[3, 12, 10, 14, 4, 9, 12, 10, 14])

             _lowerCamelCase							:     Tuple	=		os.path.join(self.tmpdirname      ,							'''tokenizer.bin''')
             with open(a__      ,							'''wb''') as handle:
                    pickle.dump(a__      ,							a__)

             with open(a__      ,							'''rb''') as handle:
                    _lowerCamelCase							:     str	=		pickle.load(a__)

             _lowerCamelCase							:     Any	=		tokenizer_new.tokenize(a__)

             self.assertListEqual(a__      ,							a__)


      @require_sudachi
      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Optional[int]	=		SudachiTokenizer(sudachi_dict_type='''core''')

             self.assertListEqual(
                 tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ''')      ,							[''' ''', '''\t''', '''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', ''' ''', '''。''', ''' ''', ''' ''']      ,							)


      @require_sudachi
      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Dict	=		SudachiTokenizer(sudachi_dict_type='''core'''      ,							sudachi_split_mode='''A''')

             self.assertListEqual(tokenizer.tokenize('''外国人参政権''')      ,							['''外国''', '''人''', '''参政''', '''権'''])


      @require_sudachi
      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Union[str, Any]	=		SudachiTokenizer(sudachi_dict_type='''core'''      ,							sudachi_split_mode='''B''')

             self.assertListEqual(tokenizer.tokenize('''外国人参政権''')      ,							['''外国人''', '''参政権'''])


      @require_sudachi
      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Dict	=		SudachiTokenizer(sudachi_dict_type='''core'''      ,							sudachi_split_mode='''C''')

             self.assertListEqual(tokenizer.tokenize('''外国人参政権''')      ,							['''外国人参政権'''])


      @require_sudachi
      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Tuple	=		SudachiTokenizer(do_lower_case=a__      ,							sudachi_dict_type='''core''')

             self.assertListEqual(
                 tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ''')      ,							[''' ''', '''\t''', '''アップル''', '''ストア''', '''で''', '''iphone''', '''8''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', ''' ''', '''。''', ''' ''', ''' ''']      ,							)


      @require_sudachi
      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Union[str, Any]	=		SudachiTokenizer(normalize_text=a__      ,							sudachi_dict_type='''core''')

             self.assertListEqual(
                 tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ''')      ,							[''' ''', '''\t''', '''ｱｯﾌﾟﾙ''', '''ストア''', '''で''', '''iPhone''', '''８''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', '''\u3000''', '''。''', ''' ''', ''' ''']      ,							)


      @require_sudachi
      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     List[Any]	=		SudachiTokenizer(trim_whitespace=a__      ,							sudachi_dict_type='''core''')

             self.assertListEqual(
                 tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ''')      ,							['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']      ,							)


      @require_jumanpp
      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Any	=		self.tokenizer_class(self.vocab_file      ,							word_tokenizer_type='''jumanpp''')
             self.assertIsNotNone(a__)

             _lowerCamelCase							:     List[str]	=		'''こんにちは、世界。\nこんばんは、世界。'''
             _lowerCamelCase							:     List[str]	=		tokenizer.tokenize(a__)
             self.assertListEqual(a__      ,							['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。'''])
             self.assertListEqual(tokenizer.convert_tokens_to_ids(a__)      ,							[3, 12, 10, 14, 4, 9, 12, 10, 14])

             _lowerCamelCase							:     List[str]	=		os.path.join(self.tmpdirname      ,							'''tokenizer.bin''')
             with open(a__      ,							'''wb''') as handle:
                    pickle.dump(a__      ,							a__)

             with open(a__      ,							'''rb''') as handle:
                    _lowerCamelCase							:     Optional[Any]	=		pickle.load(a__)

             _lowerCamelCase							:     Dict	=		tokenizer_new.tokenize(a__)

             self.assertListEqual(a__      ,							a__)


      @require_jumanpp
      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Optional[Any]	=		JumanppTokenizer()

             self.assertListEqual(
                 tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ''')      ,							['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']      ,							)


      @require_jumanpp
      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Optional[Any]	=		JumanppTokenizer(do_lower_case=a__)

             self.assertListEqual(
                 tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ''')      ,							['''アップル''', '''ストア''', '''で''', '''iphone''', '''8''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']      ,							)


      @require_jumanpp
      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Dict	=		JumanppTokenizer(normalize_text=a__)

             self.assertListEqual(
                 tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ''')      ,							['''ｱ''', '''ｯ''', '''ﾌ''', '''ﾟ''', '''ﾙ''', '''ストア''', '''で''', '''iPhone''', '''８''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']      ,							)


      @require_jumanpp
      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     int	=		JumanppTokenizer(trim_whitespace=a__)

             self.assertListEqual(
                 tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ''')      ,							['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れた''', '''。''']      ,							)


      @require_jumanpp
      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Any	=		JumanppTokenizer()

             self.assertListEqual(
                 tokenizer.tokenize('''ありがとうございますm(_ _)ｍ見つけるのが大変です。''')      ,							['''ありがとう''', '''ございます''', '''m(_ _)m''', '''見つける''', '''の''', '''が''', '''大変です''', '''。''']      ,							)


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     str	=		['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こんにちは''', '''こん''', '''にちは''', '''ばんは''', '''##こん''', '''##にちは''', '''##ばんは''']

             _lowerCamelCase							:     List[str]	=		{}
             for i, token in enumerate(a__):
                    _lowerCamelCase							:     Optional[Any]	=		i
             _lowerCamelCase							:     Union[str, Any]	=		WordpieceTokenizer(vocab=a__      ,							unk_token='''[UNK]''')

             self.assertListEqual(tokenizer.tokenize('''''')      ,							[])

             self.assertListEqual(tokenizer.tokenize('''こんにちは''')      ,							['''こんにちは'''])

             self.assertListEqual(tokenizer.tokenize('''こんばんは''')      ,							['''こん''', '''##ばんは'''])

             self.assertListEqual(tokenizer.tokenize('''こんばんは こんばんにちは こんにちは''')      ,							['''こん''', '''##ばんは''', '''[UNK]''', '''こんにちは'''])


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Tuple	=		BertJapaneseTokenizer.from_pretrained('''nlp-waseda/roberta-base-japanese-with-auto-jumanpp''')
             _lowerCamelCase							:     str	=		tokenizer.subword_tokenizer

             _lowerCamelCase							:     Any	=		subword_tokenizer.tokenize('''国境 の 長い トンネル を 抜ける と 雪国 であった 。''')
             self.assertListEqual(a__      ,							['''▁国境''', '''▁の''', '''▁長い''', '''▁トンネル''', '''▁を''', '''▁抜ける''', '''▁と''', '''▁雪''', '''国''', '''▁であった''', '''▁。'''])

             _lowerCamelCase							:     str	=		subword_tokenizer.tokenize('''こんばんは こんばん にち は こんにちは''')
             self.assertListEqual(a__      ,							['''▁こん''', '''ばん''', '''は''', '''▁こん''', '''ばん''', '''▁に''', '''ち''', '''▁は''', '''▁こんにちは'''])



      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Optional[Any]	=		self.tokenizer_class.from_pretrained('''cl-tohoku/bert-base-japanese''')

             _lowerCamelCase							:     int	=		tokenizer.encode('''ありがとう。'''      ,							add_special_tokens=a__)
             _lowerCamelCase							:     Union[str, Any]	=		tokenizer.encode('''どういたしまして。'''      ,							add_special_tokens=a__)

             _lowerCamelCase							:     int	=		tokenizer.build_inputs_with_special_tokens(a__)
             _lowerCamelCase							:     List[str]	=		tokenizer.build_inputs_with_special_tokens(a__      ,							a__)

             # 2 is for "[CLS]", 3 is for "[SEP]"
             assert encoded_sentence == [2] + text + [3]
             assert encoded_pair == [2] + text + [3] + text_a + [3]







@custom_tokenizers
class 			__A       (      lowerCamelCase__ ,unittest.TestCase  ):




      """simple docstring"""







      UpperCAmelCase__						=      BertJapaneseTokenizer
      UpperCAmelCase__						=      False


      def 				__snake_case			(		self):



             """simple docstring"""





             super().setUp()

             _lowerCamelCase							:     Dict	=		['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

             _lowerCamelCase							:     Optional[int]	=		os.path.join(self.tmpdirname      ,							VOCAB_FILES_NAMES['''vocab_file'''])
             with open(self.vocab_file      ,							'''w'''      ,							encoding='''utf-8''') as vocab_writer:
                    vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]))


      def 				__snake_case			(		self      ,							**a__):



             """simple docstring"""





             return BertJapaneseTokenizer.from_pretrained(self.tmpdirname      ,							subword_tokenizer_type='''character'''      ,							**a__)


      def 				__snake_case			(		self      ,							a__):



             """simple docstring"""





             _lowerCamelCase							:     Any	=		'''こんにちは、世界。 \nこんばんは、世界。'''
             _lowerCamelCase							:     int	=		'''こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'''
             return input_text, output_text


      def 				__snake_case			(		self):



             """simple docstring"""





             pass  # TODO add if relevant


      def 				__snake_case			(		self):



             """simple docstring"""





             pass  # TODO add if relevant


      def 				__snake_case			(		self):



             """simple docstring"""





             pass  # TODO add if relevant


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     int	=		self.tokenizer_class(self.vocab_file      ,							subword_tokenizer_type='''character''')

             _lowerCamelCase							:     Optional[int]	=		tokenizer.tokenize('''こんにちは、世界。 \nこんばんは、世界。''')
             self.assertListEqual(
                 a__      ,							['''こ''', '''ん''', '''に''', '''ち''', '''は''', '''、''', '''世''', '''界''', '''。''', '''こ''', '''ん''', '''ば''', '''ん''', '''は''', '''、''', '''世''', '''界''', '''。'''])
             self.assertListEqual(
                 tokenizer.convert_tokens_to_ids(a__)      ,							[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12])


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     List[Any]	=		['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

             _lowerCamelCase							:     List[str]	=		{}
             for i, token in enumerate(a__):
                    _lowerCamelCase							:     List[Any]	=		i
             _lowerCamelCase							:     Union[str, Any]	=		CharacterTokenizer(vocab=a__      ,							unk_token='''[UNK]''')

             self.assertListEqual(tokenizer.tokenize('''''')      ,							[])

             self.assertListEqual(tokenizer.tokenize('''こんにちは''')      ,							['''こ''', '''ん''', '''に''', '''ち''', '''は'''])

             self.assertListEqual(tokenizer.tokenize('''こんにちほ''')      ,							['''こ''', '''ん''', '''に''', '''ち''', '''[UNK]'''])



      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Dict	=		self.tokenizer_class.from_pretrained('''cl-tohoku/bert-base-japanese-char''')

             _lowerCamelCase							:     List[str]	=		tokenizer.encode('''ありがとう。'''      ,							add_special_tokens=a__)
             _lowerCamelCase							:     Optional[int]	=		tokenizer.encode('''どういたしまして。'''      ,							add_special_tokens=a__)

             _lowerCamelCase							:     Union[str, Any]	=		tokenizer.build_inputs_with_special_tokens(a__)
             _lowerCamelCase							:     int	=		tokenizer.build_inputs_with_special_tokens(a__      ,							a__)

             # 2 is for "[CLS]", 3 is for "[SEP]"
             assert encoded_sentence == [2] + text + [3]
             assert encoded_pair == [2] + text + [3] + text_a + [3]







@custom_tokenizers
class 			__A       (      unittest.TestCase  ):




      """simple docstring"""







      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     List[Any]	=		'''cl-tohoku/bert-base-japanese'''
             _lowerCamelCase							:     Optional[int]	=		AutoTokenizer.from_pretrained(a__)
             self.assertIsInstance(a__      ,							a__)







class 			__A       (      unittest.TestCase  ):




      """simple docstring"""







      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Union[str, Any]	=		'''cl-tohoku/bert-base-japanese'''
             with self.assertLogs('''transformers'''      ,							level='''WARNING''') as cm:
                    BertTokenizer.from_pretrained(a__)
                    self.assertTrue(
                        cm.records[0].message.startswith(
                            '''The tokenizer class you load from this checkpoint is not the same type as the class this function'''
                            ''' is called from.'''))
             _lowerCamelCase							:     List[Any]	=		'''bert-base-cased'''
             with self.assertLogs('''transformers'''      ,							level='''WARNING''') as cm:
                    BertJapaneseTokenizer.from_pretrained(a__)
                    self.assertTrue(
                        cm.records[0].message.startswith(
                            '''The tokenizer class you load from this checkpoint is not the same type as the class this function'''
                            ''' is called from.'''))

