






'''simple docstring'''







import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin

@custom_tokenizers
class _lowercase			(       _lowercase       ,   unittest.TestCase	):
							a          = BertJapaneseTokenizer
							a          = False
							a          = True




							def 	lowerCamelCase_       (							self:       int  ):
														super().setUp()

														lowerCamelCase__      :			str 						= [
														    """[UNK]""",
														    """[CLS]""",
														    """[SEP]""",
														    """こんにちは""",
														    """こん""",
														    """にちは""",
														    """ばんは""",
														    """##こん""",
														    """##にちは""",
														    """##ばんは""",
														    """世界""",
														    """##世界""",
														    """、""",
														    """##、""",
														    """。""",
														    """##。""",
														]

														lowerCamelCase__      :			Union[str, Any] 						= os.path.join(self.tmpdirname    ,							VOCAB_FILES_NAMES["""vocab_file"""]  )
														with open(self.vocab_file    ,							"""w"""    ,							encoding="""utf-8"""  ) as vocab_writer:
																					vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]  )  )




							def 	lowerCamelCase_       (							self:       Dict    ,							UpperCamelCase__:       str  ):
														lowerCamelCase__      :			int 						= """こんにちは、世界。 \nこんばんは、世界。"""
														lowerCamelCase__      :			Dict 						= """こんにちは 、 世界 。 こんばんは 、 世界 。"""
														return input_text, output_text




							def 	lowerCamelCase_       (							self:       List[Any]    ,							UpperCamelCase__:       str  ):
														lowerCamelCase__      :			List[Any] 						= self.get_input_output_texts(UpperCamelCase__  )
														lowerCamelCase__      :			Optional[int] 						= tokenizer.encode(UpperCamelCase__    ,							add_special_tokens=UpperCamelCase__  )
														lowerCamelCase__      :			Union[str, Any] 						= tokenizer.decode(UpperCamelCase__    ,							clean_up_tokenization_spaces=UpperCamelCase__  )
														return text, ids




							def 	lowerCamelCase_       (							self:       Tuple  ):
														pass  # TODO add if relevant




							def 	lowerCamelCase_       (							self:       List[str]  ):
														pass  # TODO add if relevant




							def 	lowerCamelCase_       (							self:       str  ):
														pass  # TODO add if relevant




							def 	lowerCamelCase_       (							self:       Optional[Any]  ):
														lowerCamelCase__      :			int 						= self.tokenizer_class(self.vocab_file  )

														lowerCamelCase__      :			str 						= tokenizer.tokenize("""こんにちは、世界。\nこんばんは、世界。"""  )
														self.assertListEqual(UpperCamelCase__    ,							["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]  )
														self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__  )    ,							[3, 12, 10, 14, 4, 9, 12, 10, 14]  )




							def 	lowerCamelCase_       (							self:       Any  ):
														lowerCamelCase__      :			Dict 						= self.tokenizer_class(self.vocab_file    ,							word_tokenizer_type="""mecab"""  )
														self.assertIsNotNone(UpperCamelCase__  )

														lowerCamelCase__      :			str 						= """こんにちは、世界。\nこんばんは、世界。"""
														lowerCamelCase__      :			Optional[Any] 						= tokenizer.tokenize(UpperCamelCase__  )
														self.assertListEqual(UpperCamelCase__    ,							["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]  )
														self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__  )    ,							[3, 12, 10, 14, 4, 9, 12, 10, 14]  )

														lowerCamelCase__      :			Dict 						= os.path.join(self.tmpdirname    ,							"""tokenizer.bin"""  )
														with open(UpperCamelCase__    ,							"""wb"""  ) as handle:
																					pickle.dump(UpperCamelCase__    ,							UpperCamelCase__  )

														with open(UpperCamelCase__    ,							"""rb"""  ) as handle:
																					lowerCamelCase__      :			Dict 						= pickle.load(UpperCamelCase__  )

														lowerCamelCase__      :			int 						= tokenizer_new.tokenize(UpperCamelCase__  )

														self.assertListEqual(UpperCamelCase__    ,							UpperCamelCase__  )




							def 	lowerCamelCase_       (							self:       Any  ):
														lowerCamelCase__      :			Any 						= MecabTokenizer(mecab_dic="""ipadic"""  )

														self.assertListEqual(
														    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """  )    ,							["""アップルストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]    ,							)




							def 	lowerCamelCase_       (							self:       Any  ):
														try:
																					lowerCamelCase__      :			Tuple 						= MecabTokenizer(mecab_dic="""unidic_lite"""  )
														except ModuleNotFoundError:
																					return

														self.assertListEqual(
														    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """  )    ,							["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]    ,							)




							def 	lowerCamelCase_       (							self:       List[Any]  ):
														try:
																					lowerCamelCase__      :			Optional[int] 						= MecabTokenizer(mecab_dic="""unidic"""  )
														except ModuleNotFoundError:
																					return

														self.assertListEqual(
														    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """  )    ,							["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]    ,							)




							def 	lowerCamelCase_       (							self:       Optional[Any]  ):
														lowerCamelCase__      :			Any 						= MecabTokenizer(do_lower_case=UpperCamelCase__    ,							mecab_dic="""ipadic"""  )

														self.assertListEqual(
														    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """  )    ,							["""アップルストア""", """で""", """iphone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]    ,							)




							def 	lowerCamelCase_       (							self:       Any  ):
														try:
																					lowerCamelCase__      :			Optional[int] 						= MecabTokenizer(
																					    do_lower_case=UpperCamelCase__    ,							normalize_text=UpperCamelCase__    ,							mecab_option="""-d /usr/local/lib/mecab/dic/jumandic"""  )
														except RuntimeError:
																					# if dict doesn't exist in the system, previous code raises this error.
																					return

														self.assertListEqual(
														    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """  )    ,							["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れた""", """\u3000""", """。"""]    ,							)




							def 	lowerCamelCase_       (							self:       int  ):
														lowerCamelCase__      :			int 						= MecabTokenizer(normalize_text=UpperCamelCase__    ,							mecab_dic="""ipadic"""  )

														self.assertListEqual(
														    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """  )    ,							["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れ""", """た""", """　""", """。"""]    ,							)




							@require_sudachi
							def 	lowerCamelCase_       (							self:       Optional[Any]  ):
														lowerCamelCase__      :			List[Any] 						= self.tokenizer_class(self.vocab_file    ,							word_tokenizer_type="""sudachi"""  )
														self.assertIsNotNone(UpperCamelCase__  )

														lowerCamelCase__      :			int 						= """こんにちは、世界。\nこんばんは、世界。"""
														lowerCamelCase__      :			Optional[Any] 						= tokenizer.tokenize(UpperCamelCase__  )
														self.assertListEqual(UpperCamelCase__    ,							["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]  )
														self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__  )    ,							[3, 12, 10, 14, 4, 9, 12, 10, 14]  )

														lowerCamelCase__      :			Union[str, Any] 						= os.path.join(self.tmpdirname    ,							"""tokenizer.bin"""  )
														with open(UpperCamelCase__    ,							"""wb"""  ) as handle:
																					pickle.dump(UpperCamelCase__    ,							UpperCamelCase__  )

														with open(UpperCamelCase__    ,							"""rb"""  ) as handle:
																					lowerCamelCase__      :			List[Any] 						= pickle.load(UpperCamelCase__  )

														lowerCamelCase__      :			Tuple 						= tokenizer_new.tokenize(UpperCamelCase__  )

														self.assertListEqual(UpperCamelCase__    ,							UpperCamelCase__  )




							@require_sudachi
							def 	lowerCamelCase_       (							self:       List[str]  ):
														lowerCamelCase__      :			Dict 						= SudachiTokenizer(sudachi_dict_type="""core"""  )

														self.assertListEqual(
														    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """  )    ,							[""" """, """\t""", """アップル""", """ストア""", """で""", """iPhone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]    ,							)




							@require_sudachi
							def 	lowerCamelCase_       (							self:       Union[str, Any]  ):
														lowerCamelCase__      :			Dict 						= SudachiTokenizer(sudachi_dict_type="""core"""    ,							sudachi_split_mode="""A"""  )

														self.assertListEqual(tokenizer.tokenize("""外国人参政権"""  )    ,							["""外国""", """人""", """参政""", """権"""]  )




							@require_sudachi
							def 	lowerCamelCase_       (							self:       int  ):
														lowerCamelCase__      :			Union[str, Any] 						= SudachiTokenizer(sudachi_dict_type="""core"""    ,							sudachi_split_mode="""B"""  )

														self.assertListEqual(tokenizer.tokenize("""外国人参政権"""  )    ,							["""外国人""", """参政権"""]  )




							@require_sudachi
							def 	lowerCamelCase_       (							self:       Tuple  ):
														lowerCamelCase__      :			int 						= SudachiTokenizer(sudachi_dict_type="""core"""    ,							sudachi_split_mode="""C"""  )

														self.assertListEqual(tokenizer.tokenize("""外国人参政権"""  )    ,							["""外国人参政権"""]  )




							@require_sudachi
							def 	lowerCamelCase_       (							self:       Any  ):
														lowerCamelCase__      :			List[Any] 						= SudachiTokenizer(do_lower_case=UpperCamelCase__    ,							sudachi_dict_type="""core"""  )

														self.assertListEqual(
														    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """  )    ,							[""" """, """\t""", """アップル""", """ストア""", """で""", """iphone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]    ,							)




							@require_sudachi
							def 	lowerCamelCase_       (							self:       Any  ):
														lowerCamelCase__      :			Dict 						= SudachiTokenizer(normalize_text=UpperCamelCase__    ,							sudachi_dict_type="""core"""  )

														self.assertListEqual(
														    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """  )    ,							[""" """, """\t""", """ｱｯﾌﾟﾙ""", """ストア""", """で""", """iPhone""", """８""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """\u3000""", """。""", """ """, """ """]    ,							)




							@require_sudachi
							def 	lowerCamelCase_       (							self:       Optional[Any]  ):
														lowerCamelCase__      :			str 						= SudachiTokenizer(trim_whitespace=UpperCamelCase__    ,							sudachi_dict_type="""core"""  )

														self.assertListEqual(
														    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """  )    ,							["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]    ,							)




							@require_jumanpp
							def 	lowerCamelCase_       (							self:       Optional[int]  ):
														lowerCamelCase__      :			Union[str, Any] 						= self.tokenizer_class(self.vocab_file    ,							word_tokenizer_type="""jumanpp"""  )
														self.assertIsNotNone(UpperCamelCase__  )

														lowerCamelCase__      :			List[Any] 						= """こんにちは、世界。\nこんばんは、世界。"""
														lowerCamelCase__      :			Any 						= tokenizer.tokenize(UpperCamelCase__  )
														self.assertListEqual(UpperCamelCase__    ,							["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]  )
														self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__  )    ,							[3, 12, 10, 14, 4, 9, 12, 10, 14]  )

														lowerCamelCase__      :			Optional[int] 						= os.path.join(self.tmpdirname    ,							"""tokenizer.bin"""  )
														with open(UpperCamelCase__    ,							"""wb"""  ) as handle:
																					pickle.dump(UpperCamelCase__    ,							UpperCamelCase__  )

														with open(UpperCamelCase__    ,							"""rb"""  ) as handle:
																					lowerCamelCase__      :			Any 						= pickle.load(UpperCamelCase__  )

														lowerCamelCase__      :			Tuple 						= tokenizer_new.tokenize(UpperCamelCase__  )

														self.assertListEqual(UpperCamelCase__    ,							UpperCamelCase__  )




							@require_jumanpp
							def 	lowerCamelCase_       (							self:       Dict  ):
														lowerCamelCase__      :			List[Any] 						= JumanppTokenizer()

														self.assertListEqual(
														    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """  )    ,							["""アップル""", """ストア""", """で""", """iPhone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]    ,							)




							@require_jumanpp
							def 	lowerCamelCase_       (							self:       Any  ):
														lowerCamelCase__      :			Optional[int] 						= JumanppTokenizer(do_lower_case=UpperCamelCase__  )

														self.assertListEqual(
														    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """  )    ,							["""アップル""", """ストア""", """で""", """iphone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]    ,							)




							@require_jumanpp
							def 	lowerCamelCase_       (							self:       Union[str, Any]  ):
														lowerCamelCase__      :			Any 						= JumanppTokenizer(normalize_text=UpperCamelCase__  )

														self.assertListEqual(
														    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """  )    ,							["""ｱ""", """ｯ""", """ﾌ""", """ﾟ""", """ﾙ""", """ストア""", """で""", """iPhone""", """８""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]    ,							)




							@require_jumanpp
							def 	lowerCamelCase_       (							self:       Optional[int]  ):
														lowerCamelCase__      :			Union[str, Any] 						= JumanppTokenizer(trim_whitespace=UpperCamelCase__  )

														self.assertListEqual(
														    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """  )    ,							["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れた""", """。"""]    ,							)




							@require_jumanpp
							def 	lowerCamelCase_       (							self:       List[str]  ):
														lowerCamelCase__      :			int 						= JumanppTokenizer()

														self.assertListEqual(
														    tokenizer.tokenize("""ありがとうございますm(_ _)ｍ見つけるのが大変です。"""  )    ,							["""ありがとう""", """ございます""", """m(_ _)m""", """見つける""", """の""", """が""", """大変です""", """。"""]    ,							)




							def 	lowerCamelCase_       (							self:       Dict  ):
														lowerCamelCase__      :			Optional[Any] 						= ["""[UNK]""", """[CLS]""", """[SEP]""", """こんにちは""", """こん""", """にちは""", """ばんは""", """##こん""", """##にちは""", """##ばんは"""]

														lowerCamelCase__      :			Optional[int] 						= {}
														for i, token in enumerate(UpperCamelCase__  ):
																					lowerCamelCase__      :			List[Any] 						= i
														lowerCamelCase__      :			Any 						= WordpieceTokenizer(vocab=UpperCamelCase__    ,							unk_token="""[UNK]"""  )

														self.assertListEqual(tokenizer.tokenize(""""""  )    ,							[]  )

														self.assertListEqual(tokenizer.tokenize("""こんにちは"""  )    ,							["""こんにちは"""]  )

														self.assertListEqual(tokenizer.tokenize("""こんばんは"""  )    ,							["""こん""", """##ばんは"""]  )

														self.assertListEqual(tokenizer.tokenize("""こんばんは こんばんにちは こんにちは"""  )    ,							["""こん""", """##ばんは""", """[UNK]""", """こんにちは"""]  )




							def 	lowerCamelCase_       (							self:       Optional[int]  ):
														lowerCamelCase__      :			str 						= BertJapaneseTokenizer.from_pretrained("""nlp-waseda/roberta-base-japanese-with-auto-jumanpp"""  )
														lowerCamelCase__      :			str 						= tokenizer.subword_tokenizer

														lowerCamelCase__      :			str 						= subword_tokenizer.tokenize("""国境 の 長い トンネル を 抜ける と 雪国 であった 。"""  )
														self.assertListEqual(UpperCamelCase__    ,							["""▁国境""", """▁の""", """▁長い""", """▁トンネル""", """▁を""", """▁抜ける""", """▁と""", """▁雪""", """国""", """▁であった""", """▁。"""]  )

														lowerCamelCase__      :			str 						= subword_tokenizer.tokenize("""こんばんは こんばん にち は こんにちは"""  )
														self.assertListEqual(UpperCamelCase__    ,							["""▁こん""", """ばん""", """は""", """▁こん""", """ばん""", """▁に""", """ち""", """▁は""", """▁こんにちは"""]  )




							def 	lowerCamelCase_       (							self:       Optional[int]  ):
														lowerCamelCase__      :			Any 						= self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese"""  )

														lowerCamelCase__      :			Union[str, Any] 						= tokenizer.encode("""ありがとう。"""    ,							add_special_tokens=UpperCamelCase__  )
														lowerCamelCase__      :			Dict 						= tokenizer.encode("""どういたしまして。"""    ,							add_special_tokens=UpperCamelCase__  )

														lowerCamelCase__      :			List[str] 						= tokenizer.build_inputs_with_special_tokens(UpperCamelCase__  )
														lowerCamelCase__      :			str 						= tokenizer.build_inputs_with_special_tokens(UpperCamelCase__    ,							UpperCamelCase__  )

														# 2 is for "[CLS]", 3 is for "[SEP]"
														assert encoded_sentence == [2] + text + [3]
														assert encoded_pair == [2] + text + [3] + text_a + [3]



@custom_tokenizers
class _lowercase			(       _lowercase       ,   unittest.TestCase	):
							a          = BertJapaneseTokenizer
							a          = False




							def 	lowerCamelCase_       (							self:       Optional[int]  ):
														super().setUp()

														lowerCamelCase__      :			Any 						= ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

														lowerCamelCase__      :			Optional[Any] 						= os.path.join(self.tmpdirname    ,							VOCAB_FILES_NAMES["""vocab_file"""]  )
														with open(self.vocab_file    ,							"""w"""    ,							encoding="""utf-8"""  ) as vocab_writer:
																					vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]  )  )




							def 	lowerCamelCase_       (							self:       List[Any]    ,							**UpperCamelCase__:       int  ):
														return BertJapaneseTokenizer.from_pretrained(self.tmpdirname    ,							subword_tokenizer_type="""character"""    ,							**UpperCamelCase__  )




							def 	lowerCamelCase_       (							self:       Optional[int]    ,							UpperCamelCase__:       str  ):
														lowerCamelCase__      :			str 						= """こんにちは、世界。 \nこんばんは、世界。"""
														lowerCamelCase__      :			List[str] 						= """こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"""
														return input_text, output_text




							def 	lowerCamelCase_       (							self:       str  ):
														pass  # TODO add if relevant




							def 	lowerCamelCase_       (							self:       int  ):
														pass  # TODO add if relevant




							def 	lowerCamelCase_       (							self:       str  ):
														pass  # TODO add if relevant




							def 	lowerCamelCase_       (							self:       List[Any]  ):
														lowerCamelCase__      :			List[str] 						= self.tokenizer_class(self.vocab_file    ,							subword_tokenizer_type="""character"""  )

														lowerCamelCase__      :			List[Any] 						= tokenizer.tokenize("""こんにちは、世界。 \nこんばんは、世界。"""  )
														self.assertListEqual(
														    UpperCamelCase__    ,							["""こ""", """ん""", """に""", """ち""", """は""", """、""", """世""", """界""", """。""", """こ""", """ん""", """ば""", """ん""", """は""", """、""", """世""", """界""", """。"""]  )
														self.assertListEqual(
														    tokenizer.convert_tokens_to_ids(UpperCamelCase__  )    ,							[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]  )




							def 	lowerCamelCase_       (							self:       Optional[int]  ):
														lowerCamelCase__      :			Any 						= ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

														lowerCamelCase__      :			Optional[Any] 						= {}
														for i, token in enumerate(UpperCamelCase__  ):
																					lowerCamelCase__      :			Union[str, Any] 						= i
														lowerCamelCase__      :			Union[str, Any] 						= CharacterTokenizer(vocab=UpperCamelCase__    ,							unk_token="""[UNK]"""  )

														self.assertListEqual(tokenizer.tokenize(""""""  )    ,							[]  )

														self.assertListEqual(tokenizer.tokenize("""こんにちは"""  )    ,							["""こ""", """ん""", """に""", """ち""", """は"""]  )

														self.assertListEqual(tokenizer.tokenize("""こんにちほ"""  )    ,							["""こ""", """ん""", """に""", """ち""", """[UNK]"""]  )




							def 	lowerCamelCase_       (							self:       int  ):
														lowerCamelCase__      :			int 						= self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese-char"""  )

														lowerCamelCase__      :			List[Any] 						= tokenizer.encode("""ありがとう。"""    ,							add_special_tokens=UpperCamelCase__  )
														lowerCamelCase__      :			int 						= tokenizer.encode("""どういたしまして。"""    ,							add_special_tokens=UpperCamelCase__  )

														lowerCamelCase__      :			List[str] 						= tokenizer.build_inputs_with_special_tokens(UpperCamelCase__  )
														lowerCamelCase__      :			Union[str, Any] 						= tokenizer.build_inputs_with_special_tokens(UpperCamelCase__    ,							UpperCamelCase__  )

														# 2 is for "[CLS]", 3 is for "[SEP]"
														assert encoded_sentence == [2] + text + [3]
														assert encoded_pair == [2] + text + [3] + text_a + [3]



@custom_tokenizers
class _lowercase			(       unittest.TestCase	):




							def 	lowerCamelCase_       (							self:       Tuple  ):
														lowerCamelCase__      :			Optional[Any] 						= """cl-tohoku/bert-base-japanese"""
														lowerCamelCase__      :			Optional[int] 						= AutoTokenizer.from_pretrained(UpperCamelCase__  )
														self.assertIsInstance(UpperCamelCase__    ,							UpperCamelCase__  )



class _lowercase			(       unittest.TestCase	):




							def 	lowerCamelCase_       (							self:       Optional[int]  ):
														lowerCamelCase__      :			Union[str, Any] 						= """cl-tohoku/bert-base-japanese"""
														with self.assertLogs("""transformers"""    ,							level="""WARNING"""  ) as cm:
																					BertTokenizer.from_pretrained(UpperCamelCase__  )
																					self.assertTrue(
																					    cm.records[0].message.startswith(
																					        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
																					        """ is called from."""  )  )
														lowerCamelCase__      :			Union[str, Any] 						= """bert-base-cased"""
														with self.assertLogs("""transformers"""    ,							level="""WARNING"""  ) as cm:
																					BertJapaneseTokenizer.from_pretrained(UpperCamelCase__  )
																					self.assertTrue(
																					    cm.records[0].message.startswith(
																					        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
																					        """ is called from."""  )  )









'''simple docstring'''







# coding=utf-8
# Copyright 2023 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# this script dumps information about the environment

import os
import platform
import sys


_A			:    Dict					  ='''3'''

print('''Python version:''', sys.version)

print('''OS platform:''', platform.platform())
print('''OS architecture:''', platform.machine())

try:
			import torch

			print('''Torch version:''', torch.__version__)
			print('''Cuda available:''', torch.cuda.is_available())
			print('''Cuda version:''', torch.version.cuda)
			print('''CuDNN version:''', torch.backends.cudnn.version())
			print('''Number of GPUs available:''', torch.cuda.device_count())
except ImportError:
			print('''Torch version:''', None)

try:
			import transformers

			print('''transformers version:''', transformers.__version__)
except ImportError:
			print('''transformers version:''', None)

