






import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin
@custom_tokenizers
class    _UpperCAmelCase      ( A__	,unittest.TestCase			):

				"""simple docstring"""




				lowercase__       							= BertJapaneseTokenizer
				lowercase__       							= False
				lowercase__       							= True




				def        lowercase__     (  self		:					str						):

								'''simple docstring'''
								super().setUp()

								lowercase__     				=    [
								    '''[UNK]''',
								    '''[CLS]''',
								    '''[SEP]''',
								    '''こんにちは''',
								    '''こん''',
								    '''にちは''',
								    '''ばんは''',
								    '''##こん''',
								    '''##にちは''',
								    '''##ばんは''',
								    '''世界''',
								    '''##世界''',
								    '''、''',
								    '''##、''',
								    '''。''',
								    '''##。''',
								]

								lowercase__     				=    os.path.join(self.tmpdirname,						VOCAB_FILES_NAMES['''vocab_file''']						)
								with open(self.vocab_file,						'''w''',						encoding='''utf-8'''						) as vocab_writer:
												vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]						)						)




				def        lowercase__     (  self		:					Any,						lowerCamelCase		:					str						):

								'''simple docstring'''
								lowercase__     				=    '''こんにちは、世界。 \nこんばんは、世界。'''
								lowercase__     				=    '''こんにちは 、 世界 。 こんばんは 、 世界 。'''
								return input_text, output_text




				def        lowercase__     (  self		:					Optional[int],						lowerCamelCase		:					List[Any]						):

								'''simple docstring'''
								lowercase__				,		lowercase__     				=    self.get_input_output_texts(lowerCamelCase						)
								lowercase__     				=    tokenizer.encode(lowerCamelCase,						add_special_tokens=lowerCamelCase						)
								lowercase__     				=    tokenizer.decode(lowerCamelCase,						clean_up_tokenization_spaces=lowerCamelCase						)
								return text, ids




				def        lowercase__     (  self		:					Any						):

								'''simple docstring'''
								pass  # TODO add if relevant




				def        lowercase__     (  self		:					Optional[int]						):

								'''simple docstring'''
								pass  # TODO add if relevant




				def        lowercase__     (  self		:					Optional[Any]						):

								'''simple docstring'''
								pass  # TODO add if relevant




				def        lowercase__     (  self		:					Tuple						):

								'''simple docstring'''
								lowercase__     				=    self.tokenizer_class(self.vocab_file						)

								lowercase__     				=    tokenizer.tokenize('''こんにちは、世界。\nこんばんは、世界。'''						)
								self.assertListEqual(lowerCamelCase,						['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']						)
								self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase						),						[3, 12, 10, 14, 4, 9, 12, 10, 14]						)




				def        lowercase__     (  self		:					Tuple						):

								'''simple docstring'''
								lowercase__     				=    self.tokenizer_class(self.vocab_file,						word_tokenizer_type='''mecab'''						)
								self.assertIsNotNone(lowerCamelCase						)

								lowercase__     				=    '''こんにちは、世界。\nこんばんは、世界。'''
								lowercase__     				=    tokenizer.tokenize(lowerCamelCase						)
								self.assertListEqual(lowerCamelCase,						['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']						)
								self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase						),						[3, 12, 10, 14, 4, 9, 12, 10, 14]						)

								lowercase__     				=    os.path.join(self.tmpdirname,						'''tokenizer.bin'''						)
								with open(lowerCamelCase,						'''wb'''						) as handle:
												pickle.dump(lowerCamelCase,						lowerCamelCase						)

								with open(lowerCamelCase,						'''rb'''						) as handle:
												lowercase__     				=    pickle.load(lowerCamelCase						)

								lowercase__     				=    tokenizer_new.tokenize(lowerCamelCase						)

								self.assertListEqual(lowerCamelCase,						lowerCamelCase						)




				def        lowercase__     (  self		:					List[Any]						):

								'''simple docstring'''
								lowercase__     				=    MecabTokenizer(mecab_dic='''ipadic'''						)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''						),						['''アップルストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。'''],						)




				def        lowercase__     (  self		:					List[str]						):

								'''simple docstring'''
								try:
												lowercase__     				=    MecabTokenizer(mecab_dic='''unidic_lite'''						)
								except ModuleNotFoundError:
												return

								self.assertListEqual(
								    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''						),						['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。'''],						)




				def        lowercase__     (  self		:					Optional[Any]						):

								'''simple docstring'''
								try:
												lowercase__     				=    MecabTokenizer(mecab_dic='''unidic'''						)
								except ModuleNotFoundError:
												return

								self.assertListEqual(
								    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''						),						['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。'''],						)




				def        lowercase__     (  self		:					Any						):

								'''simple docstring'''
								lowercase__     				=    MecabTokenizer(do_lower_case=lowerCamelCase,						mecab_dic='''ipadic'''						)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''						),						['''アップルストア''', '''で''', '''iphone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。'''],						)




				def        lowercase__     (  self		:					Any						):

								'''simple docstring'''
								try:
												lowercase__     				=    MecabTokenizer(
												    do_lower_case=lowerCamelCase,						normalize_text=lowerCamelCase,						mecab_option='''-d /usr/local/lib/mecab/dic/jumandic'''						)
								except RuntimeError:
												# if dict doesn't exist in the system, previous code raises this error.
												return

								self.assertListEqual(
								    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''						),						['''ｱｯﾌﾟﾙストア''', '''で''', '''iPhone''', '''８''', '''が''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。'''],						)




				def        lowercase__     (  self		:					Dict						):

								'''simple docstring'''
								lowercase__     				=    MecabTokenizer(normalize_text=lowerCamelCase,						mecab_dic='''ipadic'''						)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''						),						['''ｱｯﾌﾟﾙストア''', '''で''', '''iPhone''', '''８''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''　''', '''。'''],						)




				@require_sudachi
				def        lowercase__     (  self		:					List[str]						):

								'''simple docstring'''
								lowercase__     				=    self.tokenizer_class(self.vocab_file,						word_tokenizer_type='''sudachi'''						)
								self.assertIsNotNone(lowerCamelCase						)

								lowercase__     				=    '''こんにちは、世界。\nこんばんは、世界。'''
								lowercase__     				=    tokenizer.tokenize(lowerCamelCase						)
								self.assertListEqual(lowerCamelCase,						['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']						)
								self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase						),						[3, 12, 10, 14, 4, 9, 12, 10, 14]						)

								lowercase__     				=    os.path.join(self.tmpdirname,						'''tokenizer.bin'''						)
								with open(lowerCamelCase,						'''wb'''						) as handle:
												pickle.dump(lowerCamelCase,						lowerCamelCase						)

								with open(lowerCamelCase,						'''rb'''						) as handle:
												lowercase__     				=    pickle.load(lowerCamelCase						)

								lowercase__     				=    tokenizer_new.tokenize(lowerCamelCase						)

								self.assertListEqual(lowerCamelCase,						lowerCamelCase						)




				@require_sudachi
				def        lowercase__     (  self		:					Optional[Any]						):

								'''simple docstring'''
								lowercase__     				=    SudachiTokenizer(sudachi_dict_type='''core'''						)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''						),						[''' ''', '''\t''', '''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', ''' ''', '''。''', ''' ''', ''' '''],						)




				@require_sudachi
				def        lowercase__     (  self		:					Dict						):

								'''simple docstring'''
								lowercase__     				=    SudachiTokenizer(sudachi_dict_type='''core''',						sudachi_split_mode='''A'''						)

								self.assertListEqual(tokenizer.tokenize('''外国人参政権'''						),						['''外国''', '''人''', '''参政''', '''権''']						)




				@require_sudachi
				def        lowercase__     (  self		:					Optional[int]						):

								'''simple docstring'''
								lowercase__     				=    SudachiTokenizer(sudachi_dict_type='''core''',						sudachi_split_mode='''B'''						)

								self.assertListEqual(tokenizer.tokenize('''外国人参政権'''						),						['''外国人''', '''参政権''']						)




				@require_sudachi
				def        lowercase__     (  self		:					Tuple						):

								'''simple docstring'''
								lowercase__     				=    SudachiTokenizer(sudachi_dict_type='''core''',						sudachi_split_mode='''C'''						)

								self.assertListEqual(tokenizer.tokenize('''外国人参政権'''						),						['''外国人参政権''']						)




				@require_sudachi
				def        lowercase__     (  self		:					Optional[int]						):

								'''simple docstring'''
								lowercase__     				=    SudachiTokenizer(do_lower_case=lowerCamelCase,						sudachi_dict_type='''core'''						)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''						),						[''' ''', '''\t''', '''アップル''', '''ストア''', '''で''', '''iphone''', '''8''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', ''' ''', '''。''', ''' ''', ''' '''],						)




				@require_sudachi
				def        lowercase__     (  self		:					Optional[int]						):

								'''simple docstring'''
								lowercase__     				=    SudachiTokenizer(normalize_text=lowerCamelCase,						sudachi_dict_type='''core'''						)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''						),						[''' ''', '''\t''', '''ｱｯﾌﾟﾙ''', '''ストア''', '''で''', '''iPhone''', '''８''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', '''\u3000''', '''。''', ''' ''', ''' '''],						)




				@require_sudachi
				def        lowercase__     (  self		:					List[Any]						):

								'''simple docstring'''
								lowercase__     				=    SudachiTokenizer(trim_whitespace=lowerCamelCase,						sudachi_dict_type='''core'''						)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''						),						['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。'''],						)




				@require_jumanpp
				def        lowercase__     (  self		:					Union[str, Any]						):

								'''simple docstring'''
								lowercase__     				=    self.tokenizer_class(self.vocab_file,						word_tokenizer_type='''jumanpp'''						)
								self.assertIsNotNone(lowerCamelCase						)

								lowercase__     				=    '''こんにちは、世界。\nこんばんは、世界。'''
								lowercase__     				=    tokenizer.tokenize(lowerCamelCase						)
								self.assertListEqual(lowerCamelCase,						['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']						)
								self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase						),						[3, 12, 10, 14, 4, 9, 12, 10, 14]						)

								lowercase__     				=    os.path.join(self.tmpdirname,						'''tokenizer.bin'''						)
								with open(lowerCamelCase,						'''wb'''						) as handle:
												pickle.dump(lowerCamelCase,						lowerCamelCase						)

								with open(lowerCamelCase,						'''rb'''						) as handle:
												lowercase__     				=    pickle.load(lowerCamelCase						)

								lowercase__     				=    tokenizer_new.tokenize(lowerCamelCase						)

								self.assertListEqual(lowerCamelCase,						lowerCamelCase						)




				@require_jumanpp
				def        lowercase__     (  self		:					Union[str, Any]						):

								'''simple docstring'''
								lowercase__     				=    JumanppTokenizer()

								self.assertListEqual(
								    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''						),						['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。'''],						)




				@require_jumanpp
				def        lowercase__     (  self		:					List[Any]						):

								'''simple docstring'''
								lowercase__     				=    JumanppTokenizer(do_lower_case=lowerCamelCase						)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''						),						['''アップル''', '''ストア''', '''で''', '''iphone''', '''8''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。'''],						)




				@require_jumanpp
				def        lowercase__     (  self		:					List[Any]						):

								'''simple docstring'''
								lowercase__     				=    JumanppTokenizer(normalize_text=lowerCamelCase						)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''						),						['''ｱ''', '''ｯ''', '''ﾌ''', '''ﾟ''', '''ﾙ''', '''ストア''', '''で''', '''iPhone''', '''８''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。'''],						)




				@require_jumanpp
				def        lowercase__     (  self		:					Optional[int]						):

								'''simple docstring'''
								lowercase__     				=    JumanppTokenizer(trim_whitespace=lowerCamelCase						)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''						),						['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れた''', '''。'''],						)




				@require_jumanpp
				def        lowercase__     (  self		:					Union[str, Any]						):

								'''simple docstring'''
								lowercase__     				=    JumanppTokenizer()

								self.assertListEqual(
								    tokenizer.tokenize('''ありがとうございますm(_ _)ｍ見つけるのが大変です。'''						),						['''ありがとう''', '''ございます''', '''m(_ _)m''', '''見つける''', '''の''', '''が''', '''大変です''', '''。'''],						)




				def        lowercase__     (  self		:					Tuple						):

								'''simple docstring'''
								lowercase__     				=    ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こんにちは''', '''こん''', '''にちは''', '''ばんは''', '''##こん''', '''##にちは''', '''##ばんは''']

								lowercase__     				=    {}
								for i, token in enumerate(lowerCamelCase						):
												lowercase__     				=    i
								lowercase__     				=    WordpieceTokenizer(vocab=lowerCamelCase,						unk_token='''[UNK]'''						)

								self.assertListEqual(tokenizer.tokenize(''''''						),						[]						)

								self.assertListEqual(tokenizer.tokenize('''こんにちは'''						),						['''こんにちは''']						)

								self.assertListEqual(tokenizer.tokenize('''こんばんは'''						),						['''こん''', '''##ばんは''']						)

								self.assertListEqual(tokenizer.tokenize('''こんばんは こんばんにちは こんにちは'''						),						['''こん''', '''##ばんは''', '''[UNK]''', '''こんにちは''']						)




				def        lowercase__     (  self		:					int						):

								'''simple docstring'''
								lowercase__     				=    BertJapaneseTokenizer.from_pretrained('''nlp-waseda/roberta-base-japanese-with-auto-jumanpp'''						)
								lowercase__     				=    tokenizer.subword_tokenizer

								lowercase__     				=    subword_tokenizer.tokenize('''国境 の 長い トンネル を 抜ける と 雪国 であった 。'''						)
								self.assertListEqual(lowerCamelCase,						['''▁国境''', '''▁の''', '''▁長い''', '''▁トンネル''', '''▁を''', '''▁抜ける''', '''▁と''', '''▁雪''', '''国''', '''▁であった''', '''▁。''']						)

								lowercase__     				=    subword_tokenizer.tokenize('''こんばんは こんばん にち は こんにちは'''						)
								self.assertListEqual(lowerCamelCase,						['''▁こん''', '''ばん''', '''は''', '''▁こん''', '''ばん''', '''▁に''', '''ち''', '''▁は''', '''▁こんにちは''']						)




				def        lowercase__     (  self		:					Optional[int]						):

								'''simple docstring'''
								lowercase__     				=    self.tokenizer_class.from_pretrained('''cl-tohoku/bert-base-japanese'''						)

								lowercase__     				=    tokenizer.encode('''ありがとう。''',						add_special_tokens=lowerCamelCase						)
								lowercase__     				=    tokenizer.encode('''どういたしまして。''',						add_special_tokens=lowerCamelCase						)

								lowercase__     				=    tokenizer.build_inputs_with_special_tokens(lowerCamelCase						)
								lowercase__     				=    tokenizer.build_inputs_with_special_tokens(lowerCamelCase,						lowerCamelCase						)

								# 2 is for "[CLS]", 3 is for "[SEP]"
								assert encoded_sentence == [2] + text + [3]
								assert encoded_pair == [2] + text + [3] + text_a + [3]



@custom_tokenizers
class    _UpperCAmelCase      ( A__	,unittest.TestCase			):

				"""simple docstring"""




				lowercase__       							= BertJapaneseTokenizer
				lowercase__       							= False




				def        lowercase__     (  self		:					List[Any]						):

								'''simple docstring'''
								super().setUp()

								lowercase__     				=    ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

								lowercase__     				=    os.path.join(self.tmpdirname,						VOCAB_FILES_NAMES['''vocab_file''']						)
								with open(self.vocab_file,						'''w''',						encoding='''utf-8'''						) as vocab_writer:
												vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]						)						)




				def        lowercase__     (  self		:					str,						**lowerCamelCase		:					Optional[Any]						):

								'''simple docstring'''
								return BertJapaneseTokenizer.from_pretrained(self.tmpdirname,						subword_tokenizer_type='''character''',						**lowerCamelCase						)




				def        lowercase__     (  self		:					Union[str, Any],						lowerCamelCase		:					Optional[Any]						):

								'''simple docstring'''
								lowercase__     				=    '''こんにちは、世界。 \nこんばんは、世界。'''
								lowercase__     				=    '''こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'''
								return input_text, output_text




				def        lowercase__     (  self		:					Any						):

								'''simple docstring'''
								pass  # TODO add if relevant




				def        lowercase__     (  self		:					int						):

								'''simple docstring'''
								pass  # TODO add if relevant




				def        lowercase__     (  self		:					Dict						):

								'''simple docstring'''
								pass  # TODO add if relevant




				def        lowercase__     (  self		:					Tuple						):

								'''simple docstring'''
								lowercase__     				=    self.tokenizer_class(self.vocab_file,						subword_tokenizer_type='''character'''						)

								lowercase__     				=    tokenizer.tokenize('''こんにちは、世界。 \nこんばんは、世界。'''						)
								self.assertListEqual(
								    lowerCamelCase,						['''こ''', '''ん''', '''に''', '''ち''', '''は''', '''、''', '''世''', '''界''', '''。''', '''こ''', '''ん''', '''ば''', '''ん''', '''は''', '''、''', '''世''', '''界''', '''。''']						)
								self.assertListEqual(
								    tokenizer.convert_tokens_to_ids(lowerCamelCase						),						[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]						)




				def        lowercase__     (  self		:					Any						):

								'''simple docstring'''
								lowercase__     				=    ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

								lowercase__     				=    {}
								for i, token in enumerate(lowerCamelCase						):
												lowercase__     				=    i
								lowercase__     				=    CharacterTokenizer(vocab=lowerCamelCase,						unk_token='''[UNK]'''						)

								self.assertListEqual(tokenizer.tokenize(''''''						),						[]						)

								self.assertListEqual(tokenizer.tokenize('''こんにちは'''						),						['''こ''', '''ん''', '''に''', '''ち''', '''は''']						)

								self.assertListEqual(tokenizer.tokenize('''こんにちほ'''						),						['''こ''', '''ん''', '''に''', '''ち''', '''[UNK]''']						)




				def        lowercase__     (  self		:					Dict						):

								'''simple docstring'''
								lowercase__     				=    self.tokenizer_class.from_pretrained('''cl-tohoku/bert-base-japanese-char'''						)

								lowercase__     				=    tokenizer.encode('''ありがとう。''',						add_special_tokens=lowerCamelCase						)
								lowercase__     				=    tokenizer.encode('''どういたしまして。''',						add_special_tokens=lowerCamelCase						)

								lowercase__     				=    tokenizer.build_inputs_with_special_tokens(lowerCamelCase						)
								lowercase__     				=    tokenizer.build_inputs_with_special_tokens(lowerCamelCase,						lowerCamelCase						)

								# 2 is for "[CLS]", 3 is for "[SEP]"
								assert encoded_sentence == [2] + text + [3]
								assert encoded_pair == [2] + text + [3] + text_a + [3]



@custom_tokenizers
class    _UpperCAmelCase      ( unittest.TestCase			):

				"""simple docstring"""




				def        lowercase__     (  self		:					Optional[int]						):

								'''simple docstring'''
								lowercase__     				=    '''cl-tohoku/bert-base-japanese'''
								lowercase__     				=    AutoTokenizer.from_pretrained(lowerCamelCase						)
								self.assertIsInstance(lowerCamelCase,						lowerCamelCase						)



class    _UpperCAmelCase      ( unittest.TestCase			):

				"""simple docstring"""




				def        lowercase__     (  self		:					Union[str, Any]						):

								'''simple docstring'''
								lowercase__     				=    '''cl-tohoku/bert-base-japanese'''
								with self.assertLogs('''transformers''',						level='''WARNING'''						) as cm:
												BertTokenizer.from_pretrained(lowerCamelCase						)
												self.assertTrue(
												    cm.records[0].message.startswith(
												        '''The tokenizer class you load from this checkpoint is not the same type as the class this function'''
												        ''' is called from.'''						)						)
								lowercase__     				=    '''bert-base-cased'''
								with self.assertLogs('''transformers''',						level='''WARNING'''						) as cm:
												BertJapaneseTokenizer.from_pretrained(lowerCamelCase						)
												self.assertTrue(
												    cm.records[0].message.startswith(
												        '''The tokenizer class you load from this checkpoint is not the same type as the class this function'''
												        ''' is called from.'''						)						)












import argparse
import re

import requests
import torch

# git clone https://github.com/salesforce/BLIP.git
from models.blip import blip_decoder
from models.blip_itm import blip_itm
from models.blip_vqa import blip_vqa
from PIL import Image
from torchvision import transforms
from torchvision.transforms.functional import InterpolationMode

from transformers import (
    BertTokenizer,
    BlipConfig,
    BlipForConditionalGeneration,
    BlipForImageTextRetrieval,
    BlipForQuestionAnswering,
)






def 						a    (		lowerCamelCase_	,     lowerCamelCase_       ):






				'''simple docstring'''
				lowercase__     				=    '''https://storage.googleapis.com/sfr-vision-language-research/BLIP/demo.jpg'''
				lowercase__     				=    Image.open(requests.get(lowerCamelCase_	,     stream=lowerCamelCase_       ).raw       ).convert('''RGB'''       )

				lowercase__     				=    transforms.Compose(
				    [
				        transforms.Resize((image_size, image_size)	,     interpolation=InterpolationMode.BICUBIC       ),
				        transforms.ToTensor(),
				        transforms.Normalize((0.48_14_54_66, 0.4_57_82_75, 0.40_82_10_73)	,     (0.26_86_29_54, 0.26_13_02_58, 0.27_57_77_11)       ),
				    ]       )
				lowercase__     				=    transform(lowerCamelCase_       ).unsqueeze(0       ).to(lowerCamelCase_       )
				return image






def 						a    (		lowerCamelCase_       ):






				'''simple docstring'''
				if "visual_encoder" in key:
								lowercase__     				=    re.sub('''visual_encoder*'''	,     '''vision_model.encoder'''	,     lowerCamelCase_       )
				if "blocks" in key:
								lowercase__     				=    re.sub(r'''blocks'''	,     '''layers'''	,     lowerCamelCase_       )
				if "attn" in key:
								lowercase__     				=    re.sub(r'''attn'''	,     '''self_attn'''	,     lowerCamelCase_       )
				if "norm1" in key:
								lowercase__     				=    re.sub(r'''norm1'''	,     '''layer_norm1'''	,     lowerCamelCase_       )
				if "norm2" in key:
								lowercase__     				=    re.sub(r'''norm2'''	,     '''layer_norm2'''	,     lowerCamelCase_       )
				if "encoder.norm" in key:
								lowercase__     				=    re.sub(r'''encoder.norm'''	,     '''post_layernorm'''	,     lowerCamelCase_       )
				if "encoder.patch_embed.proj" in key:
								lowercase__     				=    re.sub(r'''encoder.patch_embed.proj'''	,     '''embeddings.patch_embedding'''	,     lowerCamelCase_       )

				if "encoder.pos_embed" in key:
								lowercase__     				=    re.sub(r'''encoder.pos_embed'''	,     '''embeddings.position_embedding'''	,     lowerCamelCase_       )
				if "encoder.cls_token" in key:
								lowercase__     				=    re.sub(r'''encoder.cls_token'''	,     '''embeddings.class_embedding'''	,     lowerCamelCase_       )

				if "self_attn" in key:
								lowercase__     				=    re.sub(r'''self_attn.proj'''	,     '''self_attn.projection'''	,     lowerCamelCase_       )

				return key






@torch.no_grad()
def 						a    (		lowerCamelCase_	,     lowerCamelCase_=None       ):






				'''simple docstring'''
				if config_path is not None:
								lowercase__     				=    BlipConfig.from_pretrained(lowerCamelCase_       )
				else:
								lowercase__     				=    BlipConfig(projection_dim=512	,     text_config={}	,     vision_config={}       )

				lowercase__     				=    BlipForConditionalGeneration(lowerCamelCase_       ).eval()

				lowercase__     				=    '''https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_capfilt_large.pth'''

				lowercase__     				=    blip_decoder(pretrained=lowerCamelCase_	,     image_size=384	,     vit='''base'''       )
				lowercase__     				=    pt_model.eval()

				lowercase__     				=    pt_model.state_dict()
				for key in modified_state_dict.copy():
								lowercase__     				=    modified_state_dict.pop(lowerCamelCase_       )
								lowercase__     				=    rename_key(lowerCamelCase_       )
								lowercase__     				=    value

				hf_model.load_state_dict(lowerCamelCase_       )

				lowercase__     				=    384
				lowercase__     				=    load_demo_image(image_size=lowerCamelCase_	,     device='''cpu'''       )
				lowercase__     				=    BertTokenizer.from_pretrained('''bert-base-uncased'''       )
				lowercase__     				=    tokenizer(['''a picture of''']       ).input_ids

				lowercase__     				=    hf_model.generate(lowerCamelCase_	,     lowerCamelCase_       )

				assert out[0].tolist() == [3_0522, 1037, 3861, 1997, 1037, 2450, 3564, 2006, 1996, 3509, 2007, 2014, 3899, 102]

				lowercase__     				=    hf_model.generate(lowerCamelCase_       )

				assert out[0].tolist() == [3_0522, 1037, 2450, 3564, 2006, 1996, 3509, 2007, 2014, 3899, 102]

				if pytorch_dump_folder_path is not None:
								hf_model.save_pretrained(lowerCamelCase_       )

				# model_url = 'https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_vqa.pth'
				lowercase__     				=    (
				    '''https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_vqa_capfilt_large.pth'''
				)

				lowercase__     				=    blip_vqa(pretrained=lowerCamelCase_	,     image_size=lowerCamelCase_	,     vit='''base'''       )
				vqa_model.eval()

				lowercase__     				=    vqa_model.state_dict()
				for key in modified_state_dict.copy():
								lowercase__     				=    modified_state_dict.pop(lowerCamelCase_       )
								lowercase__     				=    rename_key(lowerCamelCase_       )
								lowercase__     				=    value

				lowercase__     				=    BlipForQuestionAnswering(lowerCamelCase_       )

				hf_vqa_model.load_state_dict(lowerCamelCase_       )

				lowercase__     				=    ['''How many dogs are in this image?''']
				lowercase__     				=    tokenizer(lowerCamelCase_	,     return_tensors='''pt'''       ).input_ids

				lowercase__     				=    hf_vqa_model.generate(lowerCamelCase_	,     lowerCamelCase_       )
				print(tokenizer.decode(answer[0]       )       )

				assert tokenizer.decode(answer[0]       ) == "[UNK] 1 [SEP]"
				if pytorch_dump_folder_path is not None:
								hf_vqa_model.save_pretrained(pytorch_dump_folder_path + '''_vqa'''       )

				lowercase__     				=    '''https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_retrieval_coco.pth'''

				lowercase__     				=    blip_itm(pretrained=lowerCamelCase_	,     image_size=lowerCamelCase_	,     vit='''base'''       )
				itm_model.eval()

				lowercase__     				=    itm_model.state_dict()
				for key in modified_state_dict.copy():
								lowercase__     				=    modified_state_dict.pop(lowerCamelCase_       )
								lowercase__     				=    rename_key(lowerCamelCase_       )
								lowercase__     				=    value

				lowercase__     				=    BlipForImageTextRetrieval(lowerCamelCase_       )

				lowercase__     				=    ['''A picture of a woman with a dog sitting in a beach''']
				lowercase__     				=    tokenizer(
				    lowerCamelCase_	,     return_tensors='''pt'''	,     padding='''max_length'''	,     truncation=lowerCamelCase_	,     max_length=35	,     ).input_ids

				hf_itm_model.load_state_dict(lowerCamelCase_       )
				hf_itm_model.eval()

				lowercase__     				=    hf_itm_model(lowerCamelCase_	,     lowerCamelCase_	,     use_itm_head=lowerCamelCase_       )
				lowercase__     				=    hf_itm_model(lowerCamelCase_	,     lowerCamelCase_	,     use_itm_head=lowerCamelCase_       )

				assert out[0].item() == 0.21_10_68_74_94_27_79_54
				assert torch.nn.functional.softmax(out_itm[0]	,     dim=1       )[:, 1].item() == 0.4_56_98_84_53_86_50_51_27

				if pytorch_dump_folder_path is not None:
								hf_itm_model.save_pretrained(pytorch_dump_folder_path + '''_itm'''       )


if __name__ == "__main__":
							A__			:    Optional[int]         =				argparse.ArgumentParser()
							parser.add_argument('--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model.')
							parser.add_argument('--config_path', default=None, type=str, help='Path to hf config.json of model to convert')
							A__			:    List[Any]         =				parser.parse_args()

							convert_blip_checkpoint(args.checkpoint_path, args.pytorch_dump_folder_path, args.config_path)




