'''simple docstring'''
import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin


@custom_tokenizers
class        _UpperCAmelCase						(		SCREAMING_SNAKE_CASE_				,					unittest.TestCase			):
    __lowerCamelCase:   int		       =    BertJapaneseTokenizer
    __lowerCamelCase:   Tuple		       =    False
    __lowerCamelCase:   str		       =    True




    def 				lowerCAmelCase__				(			self  :       Tuple			):






       '''simple docstring'''






       super().setUp()

       lowercase_	:    Dict           =     [
           '''[UNK]''',
           '''[CLS]''',
           '''[SEP]''',
           '''こんにちは''',
           '''こん''',
           '''にちは''',
           '''ばんは''',
           '''##こん''',
           '''##にちは''',
           '''##ばんは''',
           '''世界''',
           '''##世界''',
           '''、''',
           '''##、''',
           '''。''',
           '''##。''',
       ]

       lowercase_	:    List[str]           =     os.path.join(self.tmpdirname  ,     VOCAB_FILES_NAMES["vocab_file"]			)
       with open(self.vocab_file  ,     "w"  ,     encoding="utf-8"			) as vocab_writer:
          vocab_writer.write("".join([x + "\n" for x in vocab_tokens]			)			)




    def 				lowerCAmelCase__				(			self  :       Union[str, Any]  ,     a  :       Any			):






       '''simple docstring'''






       lowercase_	:    Any           =     '''こんにちは、世界。 \nこんばんは、世界。'''
       lowercase_	:    Dict           =     '''こんにちは 、 世界 。 こんばんは 、 世界 。'''
       return input_text, output_text




    def 				lowerCAmelCase__				(			self  :       str  ,     a  :       Optional[Any]			):






       '''simple docstring'''






       lowercase_	:    Dict           =     self.get_input_output_texts(UpperCamelCase__			)
       lowercase_	:    List[str]           =     tokenizer.encode(UpperCamelCase__  ,     add_special_tokens=UpperCamelCase__			)
       lowercase_	:    Dict           =     tokenizer.decode(UpperCamelCase__  ,     clean_up_tokenization_spaces=UpperCamelCase__			)
       return text, ids




    def 				lowerCAmelCase__				(			self  :       Dict			):






       '''simple docstring'''






       pass  # TODO add if relevant




    def 				lowerCAmelCase__				(			self  :       int			):






       '''simple docstring'''






       pass  # TODO add if relevant




    def 				lowerCAmelCase__				(			self  :       Union[str, Any]			):






       '''simple docstring'''






       pass  # TODO add if relevant




    def 				lowerCAmelCase__				(			self  :       List[str]			):






       '''simple docstring'''






       lowercase_	:    int           =     self.tokenizer_class(self.vocab_file			)

       lowercase_	:    Any           =     tokenizer.tokenize("こんにちは、世界。\nこんばんは、世界。"			)
       self.assertListEqual(UpperCamelCase__  ,     ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]			)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__			)  ,     [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]			)




    def 				lowerCAmelCase__				(			self  :       Union[str, Any]			):






       '''simple docstring'''






       lowercase_	:    Any           =     self.tokenizer_class(self.vocab_file  ,     word_tokenizer_type="mecab"			)
       self.assertIsNotNone(UpperCamelCase__			)

       lowercase_	:    Dict           =     '''こんにちは、世界。\nこんばんは、世界。'''
       lowercase_	:    Dict           =     tokenizer.tokenize(UpperCamelCase__			)
       self.assertListEqual(UpperCamelCase__  ,     ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]			)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__			)  ,     [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]			)

       lowercase_	:    Optional[int]           =     os.path.join(self.tmpdirname  ,     "tokenizer.bin"			)
       with open(UpperCamelCase__  ,     "wb"			) as handle:
          pickle.dump(UpperCamelCase__  ,     UpperCamelCase__			)

       with open(UpperCamelCase__  ,     "rb"			) as handle:
          lowercase_	:    Tuple           =     pickle.load(UpperCamelCase__			)

       lowercase_	:    str           =     tokenizer_new.tokenize(UpperCamelCase__			)

       self.assertListEqual(UpperCamelCase__  ,     UpperCamelCase__			)




    def 				lowerCAmelCase__				(			self  :       Dict			):






       '''simple docstring'''






       lowercase_	:    Dict           =     MecabTokenizer(mecab_dic="ipadic"			)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)  ,     ["アップルストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]  ,     )




    def 				lowerCAmelCase__				(			self  :       Any			):






       '''simple docstring'''






       try:
          lowercase_	:    Tuple           =     MecabTokenizer(mecab_dic="unidic_lite"			)
       except ModuleNotFoundError:
          return

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)  ,     ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]  ,     )




    def 				lowerCAmelCase__				(			self  :       Any			):






       '''simple docstring'''






       try:
          lowercase_	:    Optional[int]           =     MecabTokenizer(mecab_dic="unidic"			)
       except ModuleNotFoundError:
          return

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)  ,     ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]  ,     )




    def 				lowerCAmelCase__				(			self  :       Optional[int]			):






       '''simple docstring'''






       lowercase_	:    Tuple           =     MecabTokenizer(do_lower_case=UpperCamelCase__  ,     mecab_dic="ipadic"			)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)  ,     ["アップルストア", "で", "iphone", "8", "が", "発売", "さ", "れ", "た", "。"]  ,     )




    def 				lowerCAmelCase__				(			self  :       List[str]			):






       '''simple docstring'''






       try:
          lowercase_	:    Dict           =     MecabTokenizer(
              do_lower_case=UpperCamelCase__  ,     normalize_text=UpperCamelCase__  ,     mecab_option="-d /usr/local/lib/mecab/dic/jumandic"			)
       except RuntimeError:
          # if dict doesn't exist in the system, previous code raises this error.
          return

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)  ,     ["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れた", "\u3000", "。"]  ,     )




    def 				lowerCAmelCase__				(			self  :       Optional[Any]			):






       '''simple docstring'''






       lowercase_	:    Optional[Any]           =     MecabTokenizer(normalize_text=UpperCamelCase__  ,     mecab_dic="ipadic"			)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)  ,     ["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れ", "た", "　", "。"]  ,     )




    @require_sudachi
    def 				lowerCAmelCase__				(			self  :       Any			):






       '''simple docstring'''






       lowercase_	:    Any           =     self.tokenizer_class(self.vocab_file  ,     word_tokenizer_type="sudachi"			)
       self.assertIsNotNone(UpperCamelCase__			)

       lowercase_	:    int           =     '''こんにちは、世界。\nこんばんは、世界。'''
       lowercase_	:    Any           =     tokenizer.tokenize(UpperCamelCase__			)
       self.assertListEqual(UpperCamelCase__  ,     ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]			)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__			)  ,     [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]			)

       lowercase_	:    str           =     os.path.join(self.tmpdirname  ,     "tokenizer.bin"			)
       with open(UpperCamelCase__  ,     "wb"			) as handle:
          pickle.dump(UpperCamelCase__  ,     UpperCamelCase__			)

       with open(UpperCamelCase__  ,     "rb"			) as handle:
          lowercase_	:    Optional[int]           =     pickle.load(UpperCamelCase__			)

       lowercase_	:    Any           =     tokenizer_new.tokenize(UpperCamelCase__			)

       self.assertListEqual(UpperCamelCase__  ,     UpperCamelCase__			)




    @require_sudachi
    def 				lowerCAmelCase__				(			self  :       int			):






       '''simple docstring'''






       lowercase_	:    str           =     SudachiTokenizer(sudachi_dict_type="core"			)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)  ,     [" ", "\t", "アップル", "ストア", "で", "iPhone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]  ,     )




    @require_sudachi
    def 				lowerCAmelCase__				(			self  :       Dict			):






       '''simple docstring'''






       lowercase_	:    Optional[Any]           =     SudachiTokenizer(sudachi_dict_type="core"  ,     sudachi_split_mode="A"			)

       self.assertListEqual(tokenizer.tokenize("外国人参政権"			)  ,     ["外国", "人", "参政", "権"]			)




    @require_sudachi
    def 				lowerCAmelCase__				(			self  :       int			):






       '''simple docstring'''






       lowercase_	:    Tuple           =     SudachiTokenizer(sudachi_dict_type="core"  ,     sudachi_split_mode="B"			)

       self.assertListEqual(tokenizer.tokenize("外国人参政権"			)  ,     ["外国人", "参政権"]			)




    @require_sudachi
    def 				lowerCAmelCase__				(			self  :       Dict			):






       '''simple docstring'''






       lowercase_	:    str           =     SudachiTokenizer(sudachi_dict_type="core"  ,     sudachi_split_mode="C"			)

       self.assertListEqual(tokenizer.tokenize("外国人参政権"			)  ,     ["外国人参政権"]			)




    @require_sudachi
    def 				lowerCAmelCase__				(			self  :       int			):






       '''simple docstring'''






       lowercase_	:    Optional[int]           =     SudachiTokenizer(do_lower_case=UpperCamelCase__  ,     sudachi_dict_type="core"			)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)  ,     [" ", "\t", "アップル", "ストア", "で", "iphone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]  ,     )




    @require_sudachi
    def 				lowerCAmelCase__				(			self  :       Any			):






       '''simple docstring'''






       lowercase_	:    Optional[int]           =     SudachiTokenizer(normalize_text=UpperCamelCase__  ,     sudachi_dict_type="core"			)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)  ,     [" ", "\t", "ｱｯﾌﾟﾙ", "ストア", "で", "iPhone", "８", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", "\u3000", "。", " ", " "]  ,     )




    @require_sudachi
    def 				lowerCAmelCase__				(			self  :       Tuple			):






       '''simple docstring'''






       lowercase_	:    Optional[int]           =     SudachiTokenizer(trim_whitespace=UpperCamelCase__  ,     sudachi_dict_type="core"			)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)  ,     ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]  ,     )




    @require_jumanpp
    def 				lowerCAmelCase__				(			self  :       List[Any]			):






       '''simple docstring'''






       lowercase_	:    Tuple           =     self.tokenizer_class(self.vocab_file  ,     word_tokenizer_type="jumanpp"			)
       self.assertIsNotNone(UpperCamelCase__			)

       lowercase_	:    Dict           =     '''こんにちは、世界。\nこんばんは、世界。'''
       lowercase_	:    List[str]           =     tokenizer.tokenize(UpperCamelCase__			)
       self.assertListEqual(UpperCamelCase__  ,     ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]			)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__			)  ,     [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]			)

       lowercase_	:    Dict           =     os.path.join(self.tmpdirname  ,     "tokenizer.bin"			)
       with open(UpperCamelCase__  ,     "wb"			) as handle:
          pickle.dump(UpperCamelCase__  ,     UpperCamelCase__			)

       with open(UpperCamelCase__  ,     "rb"			) as handle:
          lowercase_	:    Dict           =     pickle.load(UpperCamelCase__			)

       lowercase_	:    Any           =     tokenizer_new.tokenize(UpperCamelCase__			)

       self.assertListEqual(UpperCamelCase__  ,     UpperCamelCase__			)




    @require_jumanpp
    def 				lowerCAmelCase__				(			self  :       Tuple			):






       '''simple docstring'''






       lowercase_	:    Any           =     JumanppTokenizer()

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)  ,     ["アップル", "ストア", "で", "iPhone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]  ,     )




    @require_jumanpp
    def 				lowerCAmelCase__				(			self  :       int			):






       '''simple docstring'''






       lowercase_	:    Union[str, Any]           =     JumanppTokenizer(do_lower_case=UpperCamelCase__			)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)  ,     ["アップル", "ストア", "で", "iphone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]  ,     )




    @require_jumanpp
    def 				lowerCAmelCase__				(			self  :       str			):






       '''simple docstring'''






       lowercase_	:    int           =     JumanppTokenizer(normalize_text=UpperCamelCase__			)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)  ,     ["ｱ", "ｯ", "ﾌ", "ﾟ", "ﾙ", "ストア", "で", "iPhone", "８", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]  ,     )




    @require_jumanpp
    def 				lowerCAmelCase__				(			self  :       List[str]			):






       '''simple docstring'''






       lowercase_	:    Dict           =     JumanppTokenizer(trim_whitespace=UpperCamelCase__			)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)  ,     ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れた", "。"]  ,     )




    @require_jumanpp
    def 				lowerCAmelCase__				(			self  :       List[str]			):






       '''simple docstring'''






       lowercase_	:    str           =     JumanppTokenizer()

       self.assertListEqual(
           tokenizer.tokenize("ありがとうございますm(_ _)ｍ見つけるのが大変です。"			)  ,     ["ありがとう", "ございます", "m(_ _)m", "見つける", "の", "が", "大変です", "。"]  ,     )




    def 				lowerCAmelCase__				(			self  :       str			):






       '''simple docstring'''






       lowercase_	:    Tuple           =     ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こんにちは''', '''こん''', '''にちは''', '''ばんは''', '''##こん''', '''##にちは''', '''##ばんは''']

       lowercase_	:    str           =     {}
       for i, token in enumerate(UpperCamelCase__			):
          lowercase_	:    str           =     i
       lowercase_	:    Union[str, Any]           =     WordpieceTokenizer(vocab=UpperCamelCase__  ,     unk_token="[UNK]"			)

       self.assertListEqual(tokenizer.tokenize(""			)  ,     []			)

       self.assertListEqual(tokenizer.tokenize("こんにちは"			)  ,     ["こんにちは"]			)

       self.assertListEqual(tokenizer.tokenize("こんばんは"			)  ,     ["こん", "##ばんは"]			)

       self.assertListEqual(tokenizer.tokenize("こんばんは こんばんにちは こんにちは"			)  ,     ["こん", "##ばんは", "[UNK]", "こんにちは"]			)




    def 				lowerCAmelCase__				(			self  :       Optional[int]			):






       '''simple docstring'''






       lowercase_	:    Optional[Any]           =     BertJapaneseTokenizer.from_pretrained("nlp-waseda/roberta-base-japanese-with-auto-jumanpp"			)
       lowercase_	:    str           =     tokenizer.subword_tokenizer

       lowercase_	:    Tuple           =     subword_tokenizer.tokenize("国境 の 長い トンネル を 抜ける と 雪国 であった 。"			)
       self.assertListEqual(UpperCamelCase__  ,     ["▁国境", "▁の", "▁長い", "▁トンネル", "▁を", "▁抜ける", "▁と", "▁雪", "国", "▁であった", "▁。"]			)

       lowercase_	:    Tuple           =     subword_tokenizer.tokenize("こんばんは こんばん にち は こんにちは"			)
       self.assertListEqual(UpperCamelCase__  ,     ["▁こん", "ばん", "は", "▁こん", "ばん", "▁に", "ち", "▁は", "▁こんにちは"]			)





    def 				lowerCAmelCase__				(			self  :       Tuple			):






       '''simple docstring'''






       lowercase_	:    Dict           =     self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese"			)

       lowercase_	:    Optional[int]           =     tokenizer.encode("ありがとう。"  ,     add_special_tokens=UpperCamelCase__			)
       lowercase_	:    Any           =     tokenizer.encode("どういたしまして。"  ,     add_special_tokens=UpperCamelCase__			)

       lowercase_	:    List[str]           =     tokenizer.build_inputs_with_special_tokens(UpperCamelCase__			)
       lowercase_	:    List[str]           =     tokenizer.build_inputs_with_special_tokens(UpperCamelCase__  ,     UpperCamelCase__			)

       # 2 is for "[CLS]", 3 is for "[SEP]"
       assert encoded_sentence == [2] + text + [3]
       assert encoded_pair == [2] + text + [3] + text_a + [3]


@custom_tokenizers
class        _UpperCAmelCase						(		SCREAMING_SNAKE_CASE_				,					unittest.TestCase			):
    __lowerCamelCase:   Optional[int]		       =    BertJapaneseTokenizer
    __lowerCamelCase:   Union[str, Any]		       =    False




    def 				lowerCAmelCase__				(			self  :       List[Any]			):






       '''simple docstring'''






       super().setUp()

       lowercase_	:    Optional[Any]           =     ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

       lowercase_	:    Optional[Any]           =     os.path.join(self.tmpdirname  ,     VOCAB_FILES_NAMES["vocab_file"]			)
       with open(self.vocab_file  ,     "w"  ,     encoding="utf-8"			) as vocab_writer:
          vocab_writer.write("".join([x + "\n" for x in vocab_tokens]			)			)




    def 				lowerCAmelCase__				(			self  :       Optional[int]  ,     **a  :       Dict			):






       '''simple docstring'''






       return BertJapaneseTokenizer.from_pretrained(self.tmpdirname  ,     subword_tokenizer_type="character"  ,     **UpperCamelCase__			)




    def 				lowerCAmelCase__				(			self  :       Optional[Any]  ,     a  :       str			):






       '''simple docstring'''






       lowercase_	:    int           =     '''こんにちは、世界。 \nこんばんは、世界。'''
       lowercase_	:    Dict           =     '''こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'''
       return input_text, output_text




    def 				lowerCAmelCase__				(			self  :       int			):






       '''simple docstring'''






       pass  # TODO add if relevant




    def 				lowerCAmelCase__				(			self  :       Optional[Any]			):






       '''simple docstring'''






       pass  # TODO add if relevant




    def 				lowerCAmelCase__				(			self  :       str			):






       '''simple docstring'''






       pass  # TODO add if relevant




    def 				lowerCAmelCase__				(			self  :       List[Any]			):






       '''simple docstring'''






       lowercase_	:    Tuple           =     self.tokenizer_class(self.vocab_file  ,     subword_tokenizer_type="character"			)

       lowercase_	:    str           =     tokenizer.tokenize("こんにちは、世界。 \nこんばんは、世界。"			)
       self.assertListEqual(
           UpperCamelCase__  ,     ["こ", "ん", "に", "ち", "は", "、", "世", "界", "。", "こ", "ん", "ば", "ん", "は", "、", "世", "界", "。"]			)
       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(UpperCamelCase__			)  ,     [3, 4, 5, 6, 7, 1_1, 9, 1_0, 1_2, 3, 4, 8, 4, 7, 1_1, 9, 1_0, 1_2]			)




    def 				lowerCAmelCase__				(			self  :       int			):






       '''simple docstring'''






       lowercase_	:    List[str]           =     ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

       lowercase_	:    Optional[int]           =     {}
       for i, token in enumerate(UpperCamelCase__			):
          lowercase_	:    Optional[Any]           =     i
       lowercase_	:    Dict           =     CharacterTokenizer(vocab=UpperCamelCase__  ,     unk_token="[UNK]"			)

       self.assertListEqual(tokenizer.tokenize(""			)  ,     []			)

       self.assertListEqual(tokenizer.tokenize("こんにちは"			)  ,     ["こ", "ん", "に", "ち", "は"]			)

       self.assertListEqual(tokenizer.tokenize("こんにちほ"			)  ,     ["こ", "ん", "に", "ち", "[UNK]"]			)





    def 				lowerCAmelCase__				(			self  :       int			):






       '''simple docstring'''






       lowercase_	:    List[Any]           =     self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese-char"			)

       lowercase_	:    Any           =     tokenizer.encode("ありがとう。"  ,     add_special_tokens=UpperCamelCase__			)
       lowercase_	:    str           =     tokenizer.encode("どういたしまして。"  ,     add_special_tokens=UpperCamelCase__			)

       lowercase_	:    List[Any]           =     tokenizer.build_inputs_with_special_tokens(UpperCamelCase__			)
       lowercase_	:    int           =     tokenizer.build_inputs_with_special_tokens(UpperCamelCase__  ,     UpperCamelCase__			)

       # 2 is for "[CLS]", 3 is for "[SEP]"
       assert encoded_sentence == [2] + text + [3]
       assert encoded_pair == [2] + text + [3] + text_a + [3]


@custom_tokenizers
class        _UpperCAmelCase						(		unittest.TestCase			):




    def 				lowerCAmelCase__				(			self  :       List[str]			):






       '''simple docstring'''






       lowercase_	:    Tuple           =     '''cl-tohoku/bert-base-japanese'''
       lowercase_	:    str           =     AutoTokenizer.from_pretrained(UpperCamelCase__			)
       self.assertIsInstance(UpperCamelCase__  ,     UpperCamelCase__			)


class        _UpperCAmelCase						(		unittest.TestCase			):




    def 				lowerCAmelCase__				(			self  :       Optional[int]			):






       '''simple docstring'''






       lowercase_	:    str           =     '''cl-tohoku/bert-base-japanese'''
       with self.assertLogs("transformers"  ,     level="WARNING"			) as cm:
          BertTokenizer.from_pretrained(UpperCamelCase__			)
          self.assertTrue(
              cm.records[0].message.startswith(
                  "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                  " is called from."			)			)
       lowercase_	:    Any           =     '''bert-base-cased'''
       with self.assertLogs("transformers"  ,     level="WARNING"			) as cm:
          BertJapaneseTokenizer.from_pretrained(UpperCamelCase__			)
          self.assertTrue(
              cm.records[0].message.startswith(
                  "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                  " is called from."			)			)












'''simple docstring'''
from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase__ =			logging.get_logger(__name__)

UpperCamelCase__ =			{
    'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/config.json',
}


class        _UpperCAmelCase						(		snake_case			):
    __lowerCamelCase:   List[Any]		       =    'gpt_neox_japanese'




    def __init__(			self  :       List[str]  ,     a  :       List[Any]=3_2_0_0_0  ,     a  :       Union[str, Any]=2_5_6_0  ,     a  :       Optional[Any]=3_2  ,     a  :       Any=3_2  ,     a  :       str=4  ,     a  :       Optional[int]="gelu"  ,     a  :       Optional[Any]=1.00  ,     a  :       Dict=1_0_0_0_0  ,     a  :       List[Any]=2_0_4_8  ,     a  :       Dict=0.02  ,     a  :       int=1e-5  ,     a  :       Optional[int]=True  ,     a  :       Union[str, Any]=3_1_9_9_6  ,     a  :       List[Any]=3_1_9_9_9  ,     a  :       List[str]=0.1  ,     a  :       Dict=0.0  ,     **a  :       Union[str, Any]  ,     ):






       '''simple docstring'''






       super().__init__(bos_token_id=a  ,     eos_token_id=a  ,     **a			)
       lowercase_	:    int           =     vocab_size
       lowercase_	:    int           =     max_position_embeddings
       lowercase_	:    List[str]           =     hidden_size
       lowercase_	:    Any           =     num_hidden_layers
       lowercase_	:    Any           =     num_attention_heads
       lowercase_	:    List[Any]           =     intermediate_multiple_size
       lowercase_	:    List[str]           =     hidden_act
       lowercase_	:    Optional[int]           =     rotary_pct
       lowercase_	:    Tuple           =     rotary_emb_base
       lowercase_	:    Optional[Any]           =     initializer_range
       lowercase_	:    List[str]           =     layer_norm_eps
       lowercase_	:    List[str]           =     use_cache
       lowercase_	:    Any           =     attention_dropout
       lowercase_	:    List[Any]           =     hidden_dropout




