






'''simple docstring'''







import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 	SCREAMING_SNAKE_CASE					(					SCREAMING_SNAKE_CASE_	,   unittest.TestCase       ):
			snake_case__  =				GPTSanJapaneseTokenizer
			snake_case__  =				False
			snake_case__  =				{"do_clean_text": False, "add_prefix_space": False}






			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[Any]					)			->			int:
							super().setUp()

							# fmt: off
							a_		:  Union[str, Any]											=      ['''こん''', '''こんに''', '''にちは''', '''ばんは''', '''世界,㔺界''', '''、''', '''。''', '''<BR>''', '''<SP>''', '''<TAB>''', '''<URL>''', '''<EMAIL>''', '''<TEL>''', '''<DATE>''', '''<PRICE>''', '''<BLOCK>''', '''<KIGOU>''', '''<U2000U2BFF>''', '''<|emoji1|>''', '''<unk>''', '''<|bagoftoken|>''', '''<|endoftext|>''']
							# fmt: on
							a_		:  Any											=      {'''emoji''': {'''\ud83d\ude00''': '''<|emoji1|>'''}, '''emoji_inv''': {'''<|emoji1|>''': '''\ud83d\ude00'''}}  # 😀
							a_		:  Dict											=      {'''unk_token''': '''<unk>'''}

							a_		:  Union[str, Any]											=      os.path.join(self.tmpdirname  ,						VOCAB_FILES_NAMES['''vocab_file''']					)
							a_		:  List[str]											=      os.path.join(self.tmpdirname  ,						VOCAB_FILES_NAMES['''emoji_file''']					)
							with open(self.vocab_file  ,						'''w'''  ,						encoding='''utf-8'''					) as vocab_writer:
											vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]					)					)
							with open(self.emoji_file  ,						'''w'''					) as emoji_writer:
											emoji_writer.write(json.dumps(__SCREAMING_SNAKE_CASE					)					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[int]  ,						**__SCREAMING_SNAKE_CASE      :			Optional[Any]					)			->			Tuple:
							kwargs.update(self.special_tokens_map					)
							return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname  ,						**__SCREAMING_SNAKE_CASE					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[Any]  ,						__SCREAMING_SNAKE_CASE      :			Tuple					)			->			Optional[Any]:
							a_		:  Union[str, Any]											=      '''こんにちは、世界。 \nこんばんは、㔺界。😀'''
							a_		:  Dict											=      '''こんにちは、世界。 \nこんばんは、世界。😀'''
							return input_text, output_text






			def 			SCREAMING_SNAKE_CASE      ( self      :			List[Any]  ,						__SCREAMING_SNAKE_CASE      :			int					)			->			Optional[int]:
							a_				,			a_		:  List[str]											=      self.get_input_output_texts(__SCREAMING_SNAKE_CASE					)
							a_		:  List[str]											=      tokenizer.encode(__SCREAMING_SNAKE_CASE  ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
							a_		:  Optional[int]											=      tokenizer.decode(__SCREAMING_SNAKE_CASE  ,						clean_up_tokenization_spaces=__SCREAMING_SNAKE_CASE					)
							return text, ids






			def 			SCREAMING_SNAKE_CASE      ( self      :			List[str]					)			->			Optional[int]:
							pass  # TODO add if relevant






			def 			SCREAMING_SNAKE_CASE      ( self      :			Dict					)			->			int:
							pass  # TODO add if relevant






			def 			SCREAMING_SNAKE_CASE      ( self      :			Dict					)			->			Tuple:
							pass  # TODO add if relevant






			def 			SCREAMING_SNAKE_CASE      ( self      :			Union[str, Any]					)			->			str:
							a_		:  List[str]											=      self.get_tokenizer()

							# Testing tokenization
							a_		:  Tuple											=      '''こんにちは、世界。　こんばんは、㔺界。'''
							a_		:  Any											=      ['''こん''', '''にちは''', '''、''', '''世界''', '''。''', '''<SP>''', '''こん''', '''ばんは''', '''、''', '''㔺界''', '''。''']
							a_		:  Dict											=      tokenizer.tokenize(__SCREAMING_SNAKE_CASE					)
							self.assertListEqual(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)

							# Testing conversion to ids without special tokens
							a_		:  Union[str, Any]											=      [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
							a_		:  Tuple											=      tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE					)
							self.assertListEqual(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)

							# Testing conversion to ids with special tokens
							a_		:  Optional[int]											=      tokens + [tokenizer.unk_token]
							a_		:  List[Any]											=      [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
							a_		:  int											=      tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE					)
							self.assertListEqual(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Tuple					)			->			Union[str, Any]:
							a_		:  Tuple											=      self.get_tokenizer()

							# Testing tokenization
							a_		:  str											=      '''こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。'''
							a_		:  str											=      '''こんにちは、、、、世界。こんばんは、、、、世界。'''
							a_		:  Optional[int]											=      tokenizer.encode(__SCREAMING_SNAKE_CASE					)
							a_		:  str											=      tokenizer.decode(__SCREAMING_SNAKE_CASE					)
							self.assertEqual(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)






			@slow
			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[Any]					)			->			List[str]:
							a_		:  Tuple											=      self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese'''					)

							# Testing tokenization
							a_		:  Optional[Any]											=      '''こんにちは、世界。'''
							a_		:  Union[str, Any]											=      '''こんばんは、㔺界。😀'''
							a_		:  Union[str, Any]											=      '''こんにちは、世界。こんばんは、世界。😀'''
							a_		:  Union[str, Any]											=      tokenizer.encode(prefix_text + input_text					)
							a_		:  Dict											=      tokenizer.encode(''''''  ,						prefix_text=prefix_text + input_text					)
							a_		:  Optional[Any]											=      tokenizer.encode(__SCREAMING_SNAKE_CASE  ,						prefix_text=__SCREAMING_SNAKE_CASE					)
							a_		:  Optional[Any]											=      tokenizer.decode(__SCREAMING_SNAKE_CASE					)
							a_		:  Any											=      tokenizer.decode(__SCREAMING_SNAKE_CASE					)
							a_		:  Optional[Any]											=      tokenizer.decode(__SCREAMING_SNAKE_CASE					)
							self.assertEqual(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)
							self.assertEqual(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)
							self.assertEqual(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)






			@slow
			def 			SCREAMING_SNAKE_CASE      ( self      :			Tuple					)			->			Optional[Any]:
							a_		:  Any											=      self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese'''					)

							# Testing tokenization
							a_		:  Any											=      '''こんにちは、世界。'''
							a_		:  Tuple											=      '''こんばんは、㔺界。😀'''

							a_		:  Tuple											=      len(tokenizer.encode(__SCREAMING_SNAKE_CASE					)					) - 2
							a_		:  List[str]											=      len(tokenizer.encode(__SCREAMING_SNAKE_CASE					)					) - 2

							a_		:  List[str]											=      [1] + [0] * (len_prefix + len_text + 1)
							a_		:  int											=      [1] * (len_prefix + len_text + 1) + [0]
							a_		:  Optional[Any]											=      [1] + [1] * (len_prefix) + [0] * (len_text + 1)

							a_		:  List[str]											=      tokenizer(prefix_text + input_text					).token_type_ids
							a_		:  Union[str, Any]											=      tokenizer(''''''  ,						prefix_text=prefix_text + input_text					).token_type_ids
							a_		:  Tuple											=      tokenizer(__SCREAMING_SNAKE_CASE  ,						prefix_text=__SCREAMING_SNAKE_CASE					).token_type_ids
							self.assertListEqual(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)
							self.assertListEqual(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)
							self.assertListEqual(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)






			@slow
			def 			SCREAMING_SNAKE_CASE      ( self      :			str					)			->			Dict:
							a_		:  Tuple											=      self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese'''					)

							a_		:  int											=      tokenizer.encode('''あンいワ'''					)
							a_		:  List[Any]											=      tokenizer.encode(''''''  ,						prefix_text='''あンいワ'''					)
							a_		:  str											=      tokenizer.encode('''いワ'''  ,						prefix_text='''あン'''					)

							self.assertEqual(tokenizer.decode(__SCREAMING_SNAKE_CASE					)  ,						tokenizer.decode(__SCREAMING_SNAKE_CASE					)					)
							self.assertEqual(tokenizer.decode(__SCREAMING_SNAKE_CASE					)  ,						tokenizer.decode(__SCREAMING_SNAKE_CASE					)					)
							self.assertNotEqual(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)
							self.assertNotEqual(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)
							self.assertEqual(x_token_a[1]  ,						x_token_a[-1]					)  # SEG token
							self.assertEqual(x_token_a[1]  ,						x_token_a[3]					)  # SEG token






			@slow
			def 			SCREAMING_SNAKE_CASE      ( self      :			int					)			->			Optional[Any]:
							a_		:  Tuple											=      self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese'''					)

							a_		:  Optional[int]											=      [['''武田信玄''', '''は、'''], ['''織田信長''', '''の配下の、''']]
							a_		:  Any											=      tokenizer(__SCREAMING_SNAKE_CASE  ,						padding=__SCREAMING_SNAKE_CASE					)
							a_		:  Optional[Any]											=      tokenizer.batch_encode_plus(__SCREAMING_SNAKE_CASE  ,						padding=__SCREAMING_SNAKE_CASE					)

							# fmt: off
							a_		:  List[Any]											=      [[3_5993, 8640, 2_5948, 3_5998, 3_0647, 3_5675, 3_5999, 3_5999], [3_5993, 1_0382, 9868, 3_5998, 3_0646, 9459, 3_0646, 3_5675]]
							a_		:  Any											=      [[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
							a_		:  int											=      [[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
							# fmt: on
							self.assertListEqual(x_token.input_ids  ,						__SCREAMING_SNAKE_CASE					)
							self.assertListEqual(x_token.token_type_ids  ,						__SCREAMING_SNAKE_CASE					)
							self.assertListEqual(x_token.attention_mask  ,						__SCREAMING_SNAKE_CASE					)
							self.assertListEqual(x_token_a.input_ids  ,						__SCREAMING_SNAKE_CASE					)
							self.assertListEqual(x_token_a.token_type_ids  ,						__SCREAMING_SNAKE_CASE					)
							self.assertListEqual(x_token_a.attention_mask  ,						__SCREAMING_SNAKE_CASE					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[Any]					)			->			str:
							# Intentionally convert some words to accommodate character fluctuations unique to Japanese
							pass







			def 			SCREAMING_SNAKE_CASE      ( self      :			Any					)			->			Optional[int]:
							# tokenizer has no padding token
							pass












'''simple docstring'''







import copy
from dataclasses import dataclass, field
from typing import ClassVar, Dict

from ..features import ClassLabel, Features, Image
from .base import TaskTemplate





@dataclass(frozen=SCREAMING_SNAKE_CASE_       )
class 	SCREAMING_SNAKE_CASE					(					SCREAMING_SNAKE_CASE_       ):
			snake_case__  =				field(default="image-classification"	,   metadata={"include_in_asdict_even_if_is_default": True}       )
			snake_case__  =				Features({"image": Image()}       )
			snake_case__  =				Features({"labels": ClassLabel}       )
			snake_case__  =				"image"
			snake_case__  =				"labels"






			def 			SCREAMING_SNAKE_CASE      ( self      :			List[str]  ,						__SCREAMING_SNAKE_CASE      :			List[Any]					)			->			Any:
							if self.label_column not in features:
											raise ValueError(f'Column {self.label_column} is not present in features.'					)
							if not isinstance(features[self.label_column]  ,						__SCREAMING_SNAKE_CASE					):
											raise ValueError(f'Column {self.label_column} is not a ClassLabel.'					)
							a_		:  Optional[int]											=      copy.deepcopy(self					)
							a_		:  int											=      self.label_schema.copy()
							a_		:  Tuple											=      features[self.label_column]
							a_		:  str											=      label_schema
							return task_template







			@property
			def 			SCREAMING_SNAKE_CASE      ( self      :			Tuple					)			->			Dict[str, str]:
							return {
							    self.image_column: "image",
							    self.label_column: "labels",
							}




