


import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin






@custom_tokenizers
class   __lowercase				(  __snake_case ,					unittest.TestCase  ):
      UpperCamelCase								=       BertJapaneseTokenizer
      UpperCamelCase								=       False
      UpperCamelCase								=       True





      def        _lowercase				(		self						:					Tuple       )    ->   Any:






         """simple docstring"""







         super().setUp()

         UpperCAmelCase							=     [
             """[UNK]""",
             """[CLS]""",
             """[SEP]""",
             """こんにちは""",
             """こん""",
             """にちは""",
             """ばんは""",
             """##こん""",
             """##にちは""",
             """##ばんは""",
             """世界""",
             """##世界""",
             """、""",
             """##、""",
             """。""",
             """##。""",
         ]

         UpperCAmelCase							=     os.path.join(self.tmpdirname					,       VOCAB_FILES_NAMES["""vocab_file"""]       )
         with open(self.vocab_file					,       """w"""					,       encoding="""utf-8"""       ) as vocab_writer:
            vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]       )       )





      def        _lowercase				(		self						:					List[str]					,       __lowerCamelCase						:					int       )    ->   Dict:






         """simple docstring"""







         UpperCAmelCase							=     """こんにちは、世界。 \nこんばんは、世界。"""
         UpperCAmelCase							=     """こんにちは 、 世界 。 こんばんは 、 世界 。"""
         return input_text, output_text





      def        _lowercase				(		self						:					Union[str, Any]					,       __lowerCamelCase						:					Dict       )    ->   int:






         """simple docstring"""







         UpperCAmelCase				,  UpperCAmelCase							=     self.get_input_output_texts(__lowerCamelCase       )
         UpperCAmelCase							=     tokenizer.encode(__lowerCamelCase					,       add_special_tokens=__lowerCamelCase       )
         UpperCAmelCase							=     tokenizer.decode(__lowerCamelCase					,       clean_up_tokenization_spaces=__lowerCamelCase       )
         return text, ids





      def        _lowercase				(		self						:					Optional[int]       )    ->   Tuple:






         """simple docstring"""







         pass  # TODO add if relevant





      def        _lowercase				(		self						:					str       )    ->   Optional[Any]:






         """simple docstring"""







         pass  # TODO add if relevant





      def        _lowercase				(		self						:					Dict       )    ->   Dict:






         """simple docstring"""







         pass  # TODO add if relevant





      def        _lowercase				(		self						:					str       )    ->   Tuple:






         """simple docstring"""







         UpperCAmelCase							=     self.tokenizer_class(self.vocab_file       )

         UpperCAmelCase							=     tokenizer.tokenize("""こんにちは、世界。\nこんばんは、世界。"""       )
         self.assertListEqual(__lowerCamelCase					,       ["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]       )
         self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase       )					,       [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]       )





      def        _lowercase				(		self						:					Optional[Any]       )    ->   List[Any]:






         """simple docstring"""







         UpperCAmelCase							=     self.tokenizer_class(self.vocab_file					,       word_tokenizer_type="""mecab"""       )
         self.assertIsNotNone(__lowerCamelCase       )

         UpperCAmelCase							=     """こんにちは、世界。\nこんばんは、世界。"""
         UpperCAmelCase							=     tokenizer.tokenize(__lowerCamelCase       )
         self.assertListEqual(__lowerCamelCase					,       ["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]       )
         self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase       )					,       [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]       )

         UpperCAmelCase							=     os.path.join(self.tmpdirname					,       """tokenizer.bin"""       )
         with open(__lowerCamelCase					,       """wb"""       ) as handle:
            pickle.dump(__lowerCamelCase					,       __lowerCamelCase       )

         with open(__lowerCamelCase					,       """rb"""       ) as handle:
            UpperCAmelCase							=     pickle.load(__lowerCamelCase       )

         UpperCAmelCase							=     tokenizer_new.tokenize(__lowerCamelCase       )

         self.assertListEqual(__lowerCamelCase					,       __lowerCamelCase       )





      def        _lowercase				(		self						:					Tuple       )    ->   int:






         """simple docstring"""







         UpperCAmelCase							=     MecabTokenizer(mecab_dic="""ipadic"""       )

         self.assertListEqual(
             tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )					,       ["""アップルストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]					,       )





      def        _lowercase				(		self						:					Any       )    ->   Any:






         """simple docstring"""







         try:
            UpperCAmelCase							=     MecabTokenizer(mecab_dic="""unidic_lite"""       )
         except ModuleNotFoundError:
            return

         self.assertListEqual(
             tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )					,       ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]					,       )





      def        _lowercase				(		self						:					Union[str, Any]       )    ->   Optional[int]:






         """simple docstring"""







         try:
            UpperCAmelCase							=     MecabTokenizer(mecab_dic="""unidic"""       )
         except ModuleNotFoundError:
            return

         self.assertListEqual(
             tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )					,       ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]					,       )





      def        _lowercase				(		self						:					str       )    ->   Any:






         """simple docstring"""







         UpperCAmelCase							=     MecabTokenizer(do_lower_case=__lowerCamelCase					,       mecab_dic="""ipadic"""       )

         self.assertListEqual(
             tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )					,       ["""アップルストア""", """で""", """iphone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]					,       )





      def        _lowercase				(		self						:					Optional[Any]       )    ->   Optional[Any]:






         """simple docstring"""







         try:
            UpperCAmelCase							=     MecabTokenizer(
                do_lower_case=__lowerCamelCase					,       normalize_text=__lowerCamelCase					,       mecab_option="""-d /usr/local/lib/mecab/dic/jumandic"""       )
         except RuntimeError:
            # if dict doesn't exist in the system, previous code raises this error.
            return

         self.assertListEqual(
             tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )					,       ["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れた""", """\u3000""", """。"""]					,       )





      def        _lowercase				(		self						:					Tuple       )    ->   Any:






         """simple docstring"""







         UpperCAmelCase							=     MecabTokenizer(normalize_text=__lowerCamelCase					,       mecab_dic="""ipadic"""       )

         self.assertListEqual(
             tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )					,       ["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れ""", """た""", """　""", """。"""]					,       )





      @require_sudachi
      def        _lowercase				(		self						:					Dict       )    ->   Optional[Any]:






         """simple docstring"""







         UpperCAmelCase							=     self.tokenizer_class(self.vocab_file					,       word_tokenizer_type="""sudachi"""       )
         self.assertIsNotNone(__lowerCamelCase       )

         UpperCAmelCase							=     """こんにちは、世界。\nこんばんは、世界。"""
         UpperCAmelCase							=     tokenizer.tokenize(__lowerCamelCase       )
         self.assertListEqual(__lowerCamelCase					,       ["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]       )
         self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase       )					,       [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]       )

         UpperCAmelCase							=     os.path.join(self.tmpdirname					,       """tokenizer.bin"""       )
         with open(__lowerCamelCase					,       """wb"""       ) as handle:
            pickle.dump(__lowerCamelCase					,       __lowerCamelCase       )

         with open(__lowerCamelCase					,       """rb"""       ) as handle:
            UpperCAmelCase							=     pickle.load(__lowerCamelCase       )

         UpperCAmelCase							=     tokenizer_new.tokenize(__lowerCamelCase       )

         self.assertListEqual(__lowerCamelCase					,       __lowerCamelCase       )





      @require_sudachi
      def        _lowercase				(		self						:					Tuple       )    ->   Optional[int]:






         """simple docstring"""







         UpperCAmelCase							=     SudachiTokenizer(sudachi_dict_type="""core"""       )

         self.assertListEqual(
             tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )					,       [""" """, """\t""", """アップル""", """ストア""", """で""", """iPhone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]					,       )





      @require_sudachi
      def        _lowercase				(		self						:					Any       )    ->   str:






         """simple docstring"""







         UpperCAmelCase							=     SudachiTokenizer(sudachi_dict_type="""core"""					,       sudachi_split_mode="""A"""       )

         self.assertListEqual(tokenizer.tokenize("""外国人参政権"""       )					,       ["""外国""", """人""", """参政""", """権"""]       )





      @require_sudachi
      def        _lowercase				(		self						:					List[Any]       )    ->   List[str]:






         """simple docstring"""







         UpperCAmelCase							=     SudachiTokenizer(sudachi_dict_type="""core"""					,       sudachi_split_mode="""B"""       )

         self.assertListEqual(tokenizer.tokenize("""外国人参政権"""       )					,       ["""外国人""", """参政権"""]       )





      @require_sudachi
      def        _lowercase				(		self						:					str       )    ->   List[str]:






         """simple docstring"""







         UpperCAmelCase							=     SudachiTokenizer(sudachi_dict_type="""core"""					,       sudachi_split_mode="""C"""       )

         self.assertListEqual(tokenizer.tokenize("""外国人参政権"""       )					,       ["""外国人参政権"""]       )





      @require_sudachi
      def        _lowercase				(		self						:					str       )    ->   Optional[int]:






         """simple docstring"""







         UpperCAmelCase							=     SudachiTokenizer(do_lower_case=__lowerCamelCase					,       sudachi_dict_type="""core"""       )

         self.assertListEqual(
             tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )					,       [""" """, """\t""", """アップル""", """ストア""", """で""", """iphone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]					,       )





      @require_sudachi
      def        _lowercase				(		self						:					List[Any]       )    ->   List[Any]:






         """simple docstring"""







         UpperCAmelCase							=     SudachiTokenizer(normalize_text=__lowerCamelCase					,       sudachi_dict_type="""core"""       )

         self.assertListEqual(
             tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )					,       [""" """, """\t""", """ｱｯﾌﾟﾙ""", """ストア""", """で""", """iPhone""", """８""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """\u3000""", """。""", """ """, """ """]					,       )





      @require_sudachi
      def        _lowercase				(		self						:					str       )    ->   Any:






         """simple docstring"""







         UpperCAmelCase							=     SudachiTokenizer(trim_whitespace=__lowerCamelCase					,       sudachi_dict_type="""core"""       )

         self.assertListEqual(
             tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )					,       ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]					,       )





      @require_jumanpp
      def        _lowercase				(		self						:					Optional[Any]       )    ->   Dict:






         """simple docstring"""







         UpperCAmelCase							=     self.tokenizer_class(self.vocab_file					,       word_tokenizer_type="""jumanpp"""       )
         self.assertIsNotNone(__lowerCamelCase       )

         UpperCAmelCase							=     """こんにちは、世界。\nこんばんは、世界。"""
         UpperCAmelCase							=     tokenizer.tokenize(__lowerCamelCase       )
         self.assertListEqual(__lowerCamelCase					,       ["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]       )
         self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase       )					,       [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]       )

         UpperCAmelCase							=     os.path.join(self.tmpdirname					,       """tokenizer.bin"""       )
         with open(__lowerCamelCase					,       """wb"""       ) as handle:
            pickle.dump(__lowerCamelCase					,       __lowerCamelCase       )

         with open(__lowerCamelCase					,       """rb"""       ) as handle:
            UpperCAmelCase							=     pickle.load(__lowerCamelCase       )

         UpperCAmelCase							=     tokenizer_new.tokenize(__lowerCamelCase       )

         self.assertListEqual(__lowerCamelCase					,       __lowerCamelCase       )





      @require_jumanpp
      def        _lowercase				(		self						:					int       )    ->   Tuple:






         """simple docstring"""







         UpperCAmelCase							=     JumanppTokenizer()

         self.assertListEqual(
             tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )					,       ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]					,       )





      @require_jumanpp
      def        _lowercase				(		self						:					List[Any]       )    ->   str:






         """simple docstring"""







         UpperCAmelCase							=     JumanppTokenizer(do_lower_case=__lowerCamelCase       )

         self.assertListEqual(
             tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )					,       ["""アップル""", """ストア""", """で""", """iphone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]					,       )





      @require_jumanpp
      def        _lowercase				(		self						:					List[str]       )    ->   str:






         """simple docstring"""







         UpperCAmelCase							=     JumanppTokenizer(normalize_text=__lowerCamelCase       )

         self.assertListEqual(
             tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )					,       ["""ｱ""", """ｯ""", """ﾌ""", """ﾟ""", """ﾙ""", """ストア""", """で""", """iPhone""", """８""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]					,       )





      @require_jumanpp
      def        _lowercase				(		self						:					Union[str, Any]       )    ->   str:






         """simple docstring"""







         UpperCAmelCase							=     JumanppTokenizer(trim_whitespace=__lowerCamelCase       )

         self.assertListEqual(
             tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )					,       ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れた""", """。"""]					,       )





      @require_jumanpp
      def        _lowercase				(		self						:					List[str]       )    ->   int:






         """simple docstring"""







         UpperCAmelCase							=     JumanppTokenizer()

         self.assertListEqual(
             tokenizer.tokenize("""ありがとうございますm(_ _)ｍ見つけるのが大変です。"""       )					,       ["""ありがとう""", """ございます""", """m(_ _)m""", """見つける""", """の""", """が""", """大変です""", """。"""]					,       )





      def        _lowercase				(		self						:					Dict       )    ->   Optional[int]:






         """simple docstring"""







         UpperCAmelCase							=     ["""[UNK]""", """[CLS]""", """[SEP]""", """こんにちは""", """こん""", """にちは""", """ばんは""", """##こん""", """##にちは""", """##ばんは"""]

         UpperCAmelCase							=     {}
         for i, token in enumerate(__lowerCamelCase       ):
            UpperCAmelCase							=     i
         UpperCAmelCase							=     WordpieceTokenizer(vocab=__lowerCamelCase					,       unk_token="""[UNK]"""       )

         self.assertListEqual(tokenizer.tokenize(""""""       )					,       []       )

         self.assertListEqual(tokenizer.tokenize("""こんにちは"""       )					,       ["""こんにちは"""]       )

         self.assertListEqual(tokenizer.tokenize("""こんばんは"""       )					,       ["""こん""", """##ばんは"""]       )

         self.assertListEqual(tokenizer.tokenize("""こんばんは こんばんにちは こんにちは"""       )					,       ["""こん""", """##ばんは""", """[UNK]""", """こんにちは"""]       )





      def        _lowercase				(		self						:					str       )    ->   Dict:






         """simple docstring"""







         UpperCAmelCase							=     BertJapaneseTokenizer.from_pretrained("""nlp-waseda/roberta-base-japanese-with-auto-jumanpp"""       )
         UpperCAmelCase							=     tokenizer.subword_tokenizer

         UpperCAmelCase							=     subword_tokenizer.tokenize("""国境 の 長い トンネル を 抜ける と 雪国 であった 。"""       )
         self.assertListEqual(__lowerCamelCase					,       ["""▁国境""", """▁の""", """▁長い""", """▁トンネル""", """▁を""", """▁抜ける""", """▁と""", """▁雪""", """国""", """▁であった""", """▁。"""]       )

         UpperCAmelCase							=     subword_tokenizer.tokenize("""こんばんは こんばん にち は こんにちは"""       )
         self.assertListEqual(__lowerCamelCase					,       ["""▁こん""", """ばん""", """は""", """▁こん""", """ばん""", """▁に""", """ち""", """▁は""", """▁こんにちは"""]       )





      def        _lowercase				(		self						:					List[str]       )    ->   Any:






         """simple docstring"""







         UpperCAmelCase							=     self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese"""       )

         UpperCAmelCase							=     tokenizer.encode("""ありがとう。"""					,       add_special_tokens=__lowerCamelCase       )
         UpperCAmelCase							=     tokenizer.encode("""どういたしまして。"""					,       add_special_tokens=__lowerCamelCase       )

         UpperCAmelCase							=     tokenizer.build_inputs_with_special_tokens(__lowerCamelCase       )
         UpperCAmelCase							=     tokenizer.build_inputs_with_special_tokens(__lowerCamelCase					,       __lowerCamelCase       )

         # 2 is for "[CLS]", 3 is for "[SEP]"
         assert encoded_sentence == [2] + text + [3]
         assert encoded_pair == [2] + text + [3] + text_a + [3]






@custom_tokenizers
class   __lowercase				(  __snake_case ,					unittest.TestCase  ):
      UpperCamelCase								=       BertJapaneseTokenizer
      UpperCamelCase								=       False





      def        _lowercase				(		self						:					Union[str, Any]       )    ->   Tuple:






         """simple docstring"""







         super().setUp()

         UpperCAmelCase							=     ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

         UpperCAmelCase							=     os.path.join(self.tmpdirname					,       VOCAB_FILES_NAMES["""vocab_file"""]       )
         with open(self.vocab_file					,       """w"""					,       encoding="""utf-8"""       ) as vocab_writer:
            vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]       )       )





      def        _lowercase				(		self						:					Dict					,       **__lowerCamelCase						:					Dict       )    ->   List[Any]:






         """simple docstring"""







         return BertJapaneseTokenizer.from_pretrained(self.tmpdirname					,       subword_tokenizer_type="""character"""					,       **__lowerCamelCase       )





      def        _lowercase				(		self						:					List[str]					,       __lowerCamelCase						:					List[str]       )    ->   Union[str, Any]:






         """simple docstring"""







         UpperCAmelCase							=     """こんにちは、世界。 \nこんばんは、世界。"""
         UpperCAmelCase							=     """こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"""
         return input_text, output_text





      def        _lowercase				(		self						:					Union[str, Any]       )    ->   List[str]:






         """simple docstring"""







         pass  # TODO add if relevant





      def        _lowercase				(		self						:					str       )    ->   Tuple:






         """simple docstring"""







         pass  # TODO add if relevant





      def        _lowercase				(		self						:					Union[str, Any]       )    ->   Tuple:






         """simple docstring"""







         pass  # TODO add if relevant





      def        _lowercase				(		self						:					int       )    ->   int:






         """simple docstring"""







         UpperCAmelCase							=     self.tokenizer_class(self.vocab_file					,       subword_tokenizer_type="""character"""       )

         UpperCAmelCase							=     tokenizer.tokenize("""こんにちは、世界。 \nこんばんは、世界。"""       )
         self.assertListEqual(
             __lowerCamelCase					,       ["""こ""", """ん""", """に""", """ち""", """は""", """、""", """世""", """界""", """。""", """こ""", """ん""", """ば""", """ん""", """は""", """、""", """世""", """界""", """。"""]       )
         self.assertListEqual(
             tokenizer.convert_tokens_to_ids(__lowerCamelCase       )					,       [3, 4, 5, 6, 7, 1_1, 9, 1_0, 1_2, 3, 4, 8, 4, 7, 1_1, 9, 1_0, 1_2]       )





      def        _lowercase				(		self						:					Optional[int]       )    ->   Tuple:






         """simple docstring"""







         UpperCAmelCase							=     ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

         UpperCAmelCase							=     {}
         for i, token in enumerate(__lowerCamelCase       ):
            UpperCAmelCase							=     i
         UpperCAmelCase							=     CharacterTokenizer(vocab=__lowerCamelCase					,       unk_token="""[UNK]"""       )

         self.assertListEqual(tokenizer.tokenize(""""""       )					,       []       )

         self.assertListEqual(tokenizer.tokenize("""こんにちは"""       )					,       ["""こ""", """ん""", """に""", """ち""", """は"""]       )

         self.assertListEqual(tokenizer.tokenize("""こんにちほ"""       )					,       ["""こ""", """ん""", """に""", """ち""", """[UNK]"""]       )





      def        _lowercase				(		self						:					Dict       )    ->   Dict:






         """simple docstring"""







         UpperCAmelCase							=     self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese-char"""       )

         UpperCAmelCase							=     tokenizer.encode("""ありがとう。"""					,       add_special_tokens=__lowerCamelCase       )
         UpperCAmelCase							=     tokenizer.encode("""どういたしまして。"""					,       add_special_tokens=__lowerCamelCase       )

         UpperCAmelCase							=     tokenizer.build_inputs_with_special_tokens(__lowerCamelCase       )
         UpperCAmelCase							=     tokenizer.build_inputs_with_special_tokens(__lowerCamelCase					,       __lowerCamelCase       )

         # 2 is for "[CLS]", 3 is for "[SEP]"
         assert encoded_sentence == [2] + text + [3]
         assert encoded_pair == [2] + text + [3] + text_a + [3]






@custom_tokenizers
class   __lowercase				(  unittest.TestCase  ):





      def        _lowercase				(		self						:					Optional[Any]       )    ->   Union[str, Any]:






         """simple docstring"""







         UpperCAmelCase							=     """cl-tohoku/bert-base-japanese"""
         UpperCAmelCase							=     AutoTokenizer.from_pretrained(__lowerCamelCase       )
         self.assertIsInstance(__lowerCamelCase					,       __lowerCamelCase       )






class   __lowercase				(  unittest.TestCase  ):





      def        _lowercase				(		self						:					Optional[int]       )    ->   Dict:






         """simple docstring"""







         UpperCAmelCase							=     """cl-tohoku/bert-base-japanese"""
         with self.assertLogs("""transformers"""					,       level="""WARNING"""       ) as cm:
            BertTokenizer.from_pretrained(__lowerCamelCase       )
            self.assertTrue(
                cm.records[0].message.startswith(
                    """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
                    """ is called from."""       )       )
         UpperCAmelCase							=     """bert-base-cased"""
         with self.assertLogs("""transformers"""					,       level="""WARNING"""       ) as cm:
            BertJapaneseTokenizer.from_pretrained(__lowerCamelCase       )
            self.assertTrue(
                cm.records[0].message.startswith(
                    """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
                    """ is called from."""       )       )








from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


__a     =			{"""configuration_ibert""": ["""IBERT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """IBertConfig""", """IBertOnnxConfig"""]}

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __a     =			[
     """IBERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
     """IBertForMaskedLM""",
     """IBertForMultipleChoice""",
     """IBertForQuestionAnswering""",
     """IBertForSequenceClassification""",
     """IBertForTokenClassification""",
     """IBertModel""",
     """IBertPreTrainedModel""",
 ]

if TYPE_CHECKING:
 from .configuration_ibert import IBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, IBertConfig, IBertOnnxConfig

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_ibert import (
      IBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
      IBertForMaskedLM,
      IBertForMultipleChoice,
      IBertForQuestionAnswering,
      IBertForSequenceClassification,
      IBertForTokenClassification,
      IBertModel,
      IBertPreTrainedModel,
  )

else:
 import sys

 __a     =			_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)



