

import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin




@custom_tokenizers
class      UpperCamelCase (     SCREAMING_SNAKE_CASE	,			unittest.TestCase   ):
    __UpperCamelCase         =BertJapaneseTokenizer
    __UpperCamelCase         =False
    __UpperCamelCase         =True

    def UpperCamelCase			(			self :      int    ):

     """simple docstring"""






     super().setUp()

     SCREAMING_SNAKE_CASE								=			[
         '[UNK]',
         '[CLS]',
         '[SEP]',
         'こんにちは',
         'こん',
         'にちは',
         'ばんは',
         '##こん',
         '##にちは',
         '##ばんは',
         '世界',
         '##世界',
         '、',
         '##、',
         '。',
         '##。',
     ]

     SCREAMING_SNAKE_CASE								=			os.path.join(self.tmpdirname							,    VOCAB_FILES_NAMES['vocab_file']    )
     with open(self.vocab_file							,    'w'							,    encoding='utf-8'    ) as vocab_writer:
      vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]    )    )

    def UpperCamelCase			(			self :      List[str]							,    snake_case__ :      int    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			'こんにちは、世界。 \nこんばんは、世界。'
     SCREAMING_SNAKE_CASE								=			'こんにちは 、 世界 。 こんばんは 、 世界 。'
     return input_text, output_text

    def UpperCamelCase			(			self :      Optional[Any]							,    snake_case__ :      Optional[Any]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE						,	SCREAMING_SNAKE_CASE								=			self.get_input_output_texts(snake_case__    )
     SCREAMING_SNAKE_CASE								=			tokenizer.encode(snake_case__							,    add_special_tokens=snake_case__    )
     SCREAMING_SNAKE_CASE								=			tokenizer.decode(snake_case__							,    clean_up_tokenization_spaces=snake_case__    )
     return text, ids

    def UpperCamelCase			(			self :      str    ):

     """simple docstring"""






     pass  # TODO add if relevant

    def UpperCamelCase			(			self :      List[str]    ):

     """simple docstring"""






     pass  # TODO add if relevant

    def UpperCamelCase			(			self :      Dict    ):

     """simple docstring"""






     pass  # TODO add if relevant

    def UpperCamelCase			(			self :      Union[str, Any]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			self.tokenizer_class(self.vocab_file    )

     SCREAMING_SNAKE_CASE								=			tokenizer.tokenize('こんにちは、世界。\nこんばんは、世界。'    )
     self.assertListEqual(snake_case__							,    ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']    )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__    )							,    [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]    )

    def UpperCamelCase			(			self :      Union[str, Any]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			self.tokenizer_class(self.vocab_file							,    word_tokenizer_type='mecab'    )
     self.assertIsNotNone(snake_case__    )

     SCREAMING_SNAKE_CASE								=			'こんにちは、世界。\nこんばんは、世界。'
     SCREAMING_SNAKE_CASE								=			tokenizer.tokenize(snake_case__    )
     self.assertListEqual(snake_case__							,    ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']    )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__    )							,    [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]    )

     SCREAMING_SNAKE_CASE								=			os.path.join(self.tmpdirname							,    'tokenizer.bin'    )
     with open(snake_case__							,    'wb'    ) as handle:
      pickle.dump(snake_case__							,    snake_case__    )

     with open(snake_case__							,    'rb'    ) as handle:
      SCREAMING_SNAKE_CASE								=			pickle.load(snake_case__    )

     SCREAMING_SNAKE_CASE								=			tokenizer_new.tokenize(snake_case__    )

     self.assertListEqual(snake_case__							,    snake_case__    )

    def UpperCamelCase			(			self :      Dict    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			MecabTokenizer(mecab_dic='ipadic'    )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '    )							,    ['アップルストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']							,    )

    def UpperCamelCase			(			self :      List[str]    ):

     """simple docstring"""






     try:
      SCREAMING_SNAKE_CASE								=			MecabTokenizer(mecab_dic='unidic_lite'    )
     except ModuleNotFoundError:
      return

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '    )							,    ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']							,    )

    def UpperCamelCase			(			self :      Optional[int]    ):

     """simple docstring"""






     try:
      SCREAMING_SNAKE_CASE								=			MecabTokenizer(mecab_dic='unidic'    )
     except ModuleNotFoundError:
      return

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '    )							,    ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']							,    )

    def UpperCamelCase			(			self :      str    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			MecabTokenizer(do_lower_case=snake_case__							,    mecab_dic='ipadic'    )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '    )							,    ['アップルストア', 'で', 'iphone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']							,    )

    def UpperCamelCase			(			self :      Union[str, Any]    ):

     """simple docstring"""






     try:
      SCREAMING_SNAKE_CASE								=			MecabTokenizer(
          do_lower_case=snake_case__							,    normalize_text=snake_case__							,    mecab_option='-d /usr/local/lib/mecab/dic/jumandic'    )
     except RuntimeError:
      # if dict doesn't exist in the system, previous code raises this error.
      return

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '    )							,    ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れた', '\u3000', '。']							,    )

    def UpperCamelCase			(			self :      Optional[int]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			MecabTokenizer(normalize_text=snake_case__							,    mecab_dic='ipadic'    )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '    )							,    ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れ', 'た', '　', '。']							,    )

    @require_sudachi
    def UpperCamelCase			(			self :      List[Any]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			self.tokenizer_class(self.vocab_file							,    word_tokenizer_type='sudachi'    )
     self.assertIsNotNone(snake_case__    )

     SCREAMING_SNAKE_CASE								=			'こんにちは、世界。\nこんばんは、世界。'
     SCREAMING_SNAKE_CASE								=			tokenizer.tokenize(snake_case__    )
     self.assertListEqual(snake_case__							,    ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']    )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__    )							,    [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]    )

     SCREAMING_SNAKE_CASE								=			os.path.join(self.tmpdirname							,    'tokenizer.bin'    )
     with open(snake_case__							,    'wb'    ) as handle:
      pickle.dump(snake_case__							,    snake_case__    )

     with open(snake_case__							,    'rb'    ) as handle:
      SCREAMING_SNAKE_CASE								=			pickle.load(snake_case__    )

     SCREAMING_SNAKE_CASE								=			tokenizer_new.tokenize(snake_case__    )

     self.assertListEqual(snake_case__							,    snake_case__    )

    @require_sudachi
    def UpperCamelCase			(			self :      Dict    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			SudachiTokenizer(sudachi_dict_type='core'    )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '    )							,    [' ', '\t', 'アップル', 'ストア', 'で', 'iPhone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']							,    )

    @require_sudachi
    def UpperCamelCase			(			self :      str    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			SudachiTokenizer(sudachi_dict_type='core'							,    sudachi_split_mode='A'    )

     self.assertListEqual(tokenizer.tokenize('外国人参政権'    )							,    ['外国', '人', '参政', '権']    )

    @require_sudachi
    def UpperCamelCase			(			self :      str    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			SudachiTokenizer(sudachi_dict_type='core'							,    sudachi_split_mode='B'    )

     self.assertListEqual(tokenizer.tokenize('外国人参政権'    )							,    ['外国人', '参政権']    )

    @require_sudachi
    def UpperCamelCase			(			self :      str    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			SudachiTokenizer(sudachi_dict_type='core'							,    sudachi_split_mode='C'    )

     self.assertListEqual(tokenizer.tokenize('外国人参政権'    )							,    ['外国人参政権']    )

    @require_sudachi
    def UpperCamelCase			(			self :      Optional[Any]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			SudachiTokenizer(do_lower_case=snake_case__							,    sudachi_dict_type='core'    )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '    )							,    [' ', '\t', 'アップル', 'ストア', 'で', 'iphone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']							,    )

    @require_sudachi
    def UpperCamelCase			(			self :      Optional[Any]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			SudachiTokenizer(normalize_text=snake_case__							,    sudachi_dict_type='core'    )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '    )							,    [' ', '\t', 'ｱｯﾌﾟﾙ', 'ストア', 'で', 'iPhone', '８', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', '\u3000', '。', ' ', ' ']							,    )

    @require_sudachi
    def UpperCamelCase			(			self :      Optional[int]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			SudachiTokenizer(trim_whitespace=snake_case__							,    sudachi_dict_type='core'    )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '    )							,    ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']							,    )

    @require_jumanpp
    def UpperCamelCase			(			self :      Any    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			self.tokenizer_class(self.vocab_file							,    word_tokenizer_type='jumanpp'    )
     self.assertIsNotNone(snake_case__    )

     SCREAMING_SNAKE_CASE								=			'こんにちは、世界。\nこんばんは、世界。'
     SCREAMING_SNAKE_CASE								=			tokenizer.tokenize(snake_case__    )
     self.assertListEqual(snake_case__							,    ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']    )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__    )							,    [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]    )

     SCREAMING_SNAKE_CASE								=			os.path.join(self.tmpdirname							,    'tokenizer.bin'    )
     with open(snake_case__							,    'wb'    ) as handle:
      pickle.dump(snake_case__							,    snake_case__    )

     with open(snake_case__							,    'rb'    ) as handle:
      SCREAMING_SNAKE_CASE								=			pickle.load(snake_case__    )

     SCREAMING_SNAKE_CASE								=			tokenizer_new.tokenize(snake_case__    )

     self.assertListEqual(snake_case__							,    snake_case__    )

    @require_jumanpp
    def UpperCamelCase			(			self :      Any    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			JumanppTokenizer()

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '    )							,    ['アップル', 'ストア', 'で', 'iPhone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']							,    )

    @require_jumanpp
    def UpperCamelCase			(			self :      int    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			JumanppTokenizer(do_lower_case=snake_case__    )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '    )							,    ['アップル', 'ストア', 'で', 'iphone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']							,    )

    @require_jumanpp
    def UpperCamelCase			(			self :      Tuple    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			JumanppTokenizer(normalize_text=snake_case__    )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '    )							,    ['ｱ', 'ｯ', 'ﾌ', 'ﾟ', 'ﾙ', 'ストア', 'で', 'iPhone', '８', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']							,    )

    @require_jumanpp
    def UpperCamelCase			(			self :      Optional[int]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			JumanppTokenizer(trim_whitespace=snake_case__    )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '    )							,    ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れた', '。']							,    )

    @require_jumanpp
    def UpperCamelCase			(			self :      Dict    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			JumanppTokenizer()

     self.assertListEqual(
         tokenizer.tokenize('ありがとうございますm(_ _)ｍ見つけるのが大変です。'    )							,    ['ありがとう', 'ございます', 'm(_ _)m', '見つける', 'の', 'が', '大変です', '。']							,    )

    def UpperCamelCase			(			self :      str    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			['[UNK]', '[CLS]', '[SEP]', 'こんにちは', 'こん', 'にちは', 'ばんは', '##こん', '##にちは', '##ばんは']

     SCREAMING_SNAKE_CASE								=			{}
     for i, token in enumerate(snake_case__    ):
      SCREAMING_SNAKE_CASE								=			i
     SCREAMING_SNAKE_CASE								=			WordpieceTokenizer(vocab=snake_case__							,    unk_token='[UNK]'    )

     self.assertListEqual(tokenizer.tokenize(''    )							,    []    )

     self.assertListEqual(tokenizer.tokenize('こんにちは'    )							,    ['こんにちは']    )

     self.assertListEqual(tokenizer.tokenize('こんばんは'    )							,    ['こん', '##ばんは']    )

     self.assertListEqual(tokenizer.tokenize('こんばんは こんばんにちは こんにちは'    )							,    ['こん', '##ばんは', '[UNK]', 'こんにちは']    )

    def UpperCamelCase			(			self :      Any    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			BertJapaneseTokenizer.from_pretrained('nlp-waseda/roberta-base-japanese-with-auto-jumanpp'    )
     SCREAMING_SNAKE_CASE								=			tokenizer.subword_tokenizer

     SCREAMING_SNAKE_CASE								=			subword_tokenizer.tokenize('国境 の 長い トンネル を 抜ける と 雪国 であった 。'    )
     self.assertListEqual(snake_case__							,    ['▁国境', '▁の', '▁長い', '▁トンネル', '▁を', '▁抜ける', '▁と', '▁雪', '国', '▁であった', '▁。']    )

     SCREAMING_SNAKE_CASE								=			subword_tokenizer.tokenize('こんばんは こんばん にち は こんにちは'    )
     self.assertListEqual(snake_case__							,    ['▁こん', 'ばん', 'は', '▁こん', 'ばん', '▁に', 'ち', '▁は', '▁こんにちは']    )




    def UpperCamelCase			(			self :      Any    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese'    )

     SCREAMING_SNAKE_CASE								=			tokenizer.encode('ありがとう。'							,    add_special_tokens=snake_case__    )
     SCREAMING_SNAKE_CASE								=			tokenizer.encode('どういたしまして。'							,    add_special_tokens=snake_case__    )

     SCREAMING_SNAKE_CASE								=			tokenizer.build_inputs_with_special_tokens(snake_case__    )
     SCREAMING_SNAKE_CASE								=			tokenizer.build_inputs_with_special_tokens(snake_case__							,    snake_case__    )

     # 2 is for "[CLS]", 3 is for "[SEP]"
     assert encoded_sentence == [2] + text + [3]
     assert encoded_pair == [2] + text + [3] + text_a + [3]




@custom_tokenizers
class      UpperCamelCase (     SCREAMING_SNAKE_CASE	,			unittest.TestCase   ):
    __UpperCamelCase         =BertJapaneseTokenizer
    __UpperCamelCase         =False

    def UpperCamelCase			(			self :      Any    ):

     """simple docstring"""






     super().setUp()

     SCREAMING_SNAKE_CASE								=			['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

     SCREAMING_SNAKE_CASE								=			os.path.join(self.tmpdirname							,    VOCAB_FILES_NAMES['vocab_file']    )
     with open(self.vocab_file							,    'w'							,    encoding='utf-8'    ) as vocab_writer:
      vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]    )    )

    def UpperCamelCase			(			self :      str							,    **snake_case__ :      Union[str, Any]    ):

     """simple docstring"""






     return BertJapaneseTokenizer.from_pretrained(self.tmpdirname							,    subword_tokenizer_type='character'							,    **snake_case__    )

    def UpperCamelCase			(			self :      List[Any]							,    snake_case__ :      Optional[Any]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			'こんにちは、世界。 \nこんばんは、世界。'
     SCREAMING_SNAKE_CASE								=			'こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'
     return input_text, output_text

    def UpperCamelCase			(			self :      List[str]    ):

     """simple docstring"""






     pass  # TODO add if relevant

    def UpperCamelCase			(			self :      Dict    ):

     """simple docstring"""






     pass  # TODO add if relevant

    def UpperCamelCase			(			self :      Dict    ):

     """simple docstring"""






     pass  # TODO add if relevant

    def UpperCamelCase			(			self :      Tuple    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			self.tokenizer_class(self.vocab_file							,    subword_tokenizer_type='character'    )

     SCREAMING_SNAKE_CASE								=			tokenizer.tokenize('こんにちは、世界。 \nこんばんは、世界。'    )
     self.assertListEqual(
         snake_case__							,    ['こ', 'ん', 'に', 'ち', 'は', '、', '世', '界', '。', 'こ', 'ん', 'ば', 'ん', 'は', '、', '世', '界', '。']    )
     self.assertListEqual(
         tokenizer.convert_tokens_to_ids(snake_case__    )							,    [3, 4, 5, 6, 7, 1_1, 9, 1_0, 1_2, 3, 4, 8, 4, 7, 1_1, 9, 1_0, 1_2]    )

    def UpperCamelCase			(			self :      Optional[int]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

     SCREAMING_SNAKE_CASE								=			{}
     for i, token in enumerate(snake_case__    ):
      SCREAMING_SNAKE_CASE								=			i
     SCREAMING_SNAKE_CASE								=			CharacterTokenizer(vocab=snake_case__							,    unk_token='[UNK]'    )

     self.assertListEqual(tokenizer.tokenize(''    )							,    []    )

     self.assertListEqual(tokenizer.tokenize('こんにちは'    )							,    ['こ', 'ん', 'に', 'ち', 'は']    )

     self.assertListEqual(tokenizer.tokenize('こんにちほ'    )							,    ['こ', 'ん', 'に', 'ち', '[UNK]']    )




    def UpperCamelCase			(			self :      Optional[Any]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese-char'    )

     SCREAMING_SNAKE_CASE								=			tokenizer.encode('ありがとう。'							,    add_special_tokens=snake_case__    )
     SCREAMING_SNAKE_CASE								=			tokenizer.encode('どういたしまして。'							,    add_special_tokens=snake_case__    )

     SCREAMING_SNAKE_CASE								=			tokenizer.build_inputs_with_special_tokens(snake_case__    )
     SCREAMING_SNAKE_CASE								=			tokenizer.build_inputs_with_special_tokens(snake_case__							,    snake_case__    )

     # 2 is for "[CLS]", 3 is for "[SEP]"
     assert encoded_sentence == [2] + text + [3]
     assert encoded_pair == [2] + text + [3] + text_a + [3]




@custom_tokenizers
class      UpperCamelCase (     unittest.TestCase   ):

    def UpperCamelCase			(			self :      List[str]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			'cl-tohoku/bert-base-japanese'
     SCREAMING_SNAKE_CASE								=			AutoTokenizer.from_pretrained(snake_case__    )
     self.assertIsInstance(snake_case__							,    snake_case__    )




class      UpperCamelCase (     unittest.TestCase   ):

    def UpperCamelCase			(			self :      Any    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			'cl-tohoku/bert-base-japanese'
     with self.assertLogs('transformers'							,    level='WARNING'    ) as cm:
      BertTokenizer.from_pretrained(snake_case__    )
      self.assertTrue(
          cm.records[0].message.startswith(
              'The tokenizer class you load from this checkpoint is not the same type as the class this function'
              ' is called from.'    )    )
     SCREAMING_SNAKE_CASE								=			'bert-base-cased'
     with self.assertLogs('transformers'							,    level='WARNING'    ) as cm:
      BertJapaneseTokenizer.from_pretrained(snake_case__    )
      self.assertTrue(
          cm.records[0].message.startswith(
              'The tokenizer class you load from this checkpoint is not the same type as the class this function'
              ' is called from.'    )    )





# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from ..models.whisper import WhisperForConditionalGeneration, WhisperProcessor
from .base import PipelineTool




class      UpperCamelCase (     SCREAMING_SNAKE_CASE   ):
    __UpperCamelCase         ="openai/whisper-base"
    __UpperCamelCase         =(
        "This is a tool that transcribes an audio into text. It takes an input named `audio` and returns the "
        "transcribed text."
    )
    __UpperCamelCase         ="transcriber"
    __UpperCamelCase         =WhisperProcessor
    __UpperCamelCase         =WhisperForConditionalGeneration

    __UpperCamelCase         =["audio"]
    __UpperCamelCase         =["text"]

    def UpperCamelCase			(			self :      Dict							,    snake_case__ :      Tuple    ):

     """simple docstring"""






     return self.pre_processor(snake_case__							,    return_tensors='pt'    ).input_features

    def UpperCamelCase			(			self :      Optional[int]							,    snake_case__ :      Tuple    ):

     """simple docstring"""






     return self.model.generate(inputs=snake_case__    )




    def UpperCamelCase			(			self :      str							,    snake_case__ :      Union[str, Any]    ):

     """simple docstring"""






     return self.pre_processor.batch_decode(snake_case__							,    skip_special_tokens=snake_case__    )[0]

