






'''simple docstring'''







import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english





@require_tokenizers
class 	SCREAMING_SNAKE_CASE					(					SCREAMING_SNAKE_CASE_	,   unittest.TestCase       ):
			snake_case__  =				RoCBertTokenizer
			snake_case__  =				None
			snake_case__  =				False
			snake_case__  =				True
			snake_case__  =				filter_non_english






			def 			SCREAMING_SNAKE_CASE      ( self      :			Union[str, Any]					)			->			Dict:
							super().setUp()

							a_		:  Optional[int]											=      ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''你''', '''好''', '''是''', '''谁''', '''a''', '''b''', '''c''', '''d''']
							a_		:  List[Any]											=      {}
							a_		:  Any											=      {}
							for i, value in enumerate(__SCREAMING_SNAKE_CASE					):
											a_		:  Any											=      i
											a_		:  List[str]											=      i
							a_		:  Union[str, Any]											=      os.path.join(self.tmpdirname  ,						VOCAB_FILES_NAMES['''vocab_file''']					)
							a_		:  List[str]											=      os.path.join(self.tmpdirname  ,						VOCAB_FILES_NAMES['''word_shape_file''']					)
							a_		:  Any											=      os.path.join(self.tmpdirname  ,						VOCAB_FILES_NAMES['''word_pronunciation_file''']					)
							with open(self.vocab_file  ,						'''w'''  ,						encoding='''utf-8'''					) as vocab_writer:
											vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]					)					)
							with open(self.word_shape_file  ,						'''w'''  ,						encoding='''utf-8'''					) as word_shape_writer:
											json.dump(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE  ,						ensure_ascii=__SCREAMING_SNAKE_CASE					)
							with open(self.word_pronunciation_file  ,						'''w'''  ,						encoding='''utf-8'''					) as word_pronunciation_writer:
											json.dump(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE  ,						ensure_ascii=__SCREAMING_SNAKE_CASE					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Union[str, Any]					)			->			Dict:
							a_		:  Any											=      self.tokenizer_class(self.vocab_file  ,						self.word_shape_file  ,						self.word_pronunciation_file					)

							a_		:  Optional[int]											=      tokenizer.tokenize('''你好[SEP]你是谁'''					)
							self.assertListEqual(__SCREAMING_SNAKE_CASE  ,						['''你''', '''好''', '''[SEP]''', '''你''', '''是''', '''谁''']					)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE					)  ,						[5, 6, 2, 5, 7, 8]					)
							self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(__SCREAMING_SNAKE_CASE					)  ,						[5, 6, 2, 5, 7, 8]					)
							self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(__SCREAMING_SNAKE_CASE					)  ,						[5, 6, 2, 5, 7, 8]					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[Any]					)			->			List[str]:
							a_		:  str											=      RoCBertBasicTokenizer()

							self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''					)  ,						['''ah''', '''\u535A''', '''\u63A8''', '''zz''']					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			List[Any]					)			->			str:
							a_		:  Optional[Any]											=      RoCBertBasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE					)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''					)  ,						['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)  ,						['''hello''']					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[Any]					)			->			str:
							a_		:  List[str]											=      RoCBertBasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE  ,						strip_accents=__SCREAMING_SNAKE_CASE					)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)  ,						['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)  ,						['''h\u00E9llo''']					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Union[str, Any]					)			->			Tuple:
							a_		:  int											=      RoCBertBasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE  ,						strip_accents=__SCREAMING_SNAKE_CASE					)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)  ,						['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)  ,						['''hello''']					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			int					)			->			Optional[Any]:
							a_		:  Dict											=      RoCBertBasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE					)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)  ,						['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)  ,						['''hello''']					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			List[str]					)			->			str:
							a_		:  Any											=      RoCBertBasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE					)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''					)  ,						['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Union[str, Any]					)			->			List[str]:
							a_		:  int											=      RoCBertBasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE  ,						strip_accents=__SCREAMING_SNAKE_CASE					)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)  ,						['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			int					)			->			Tuple:
							a_		:  Optional[int]											=      RoCBertBasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE  ,						strip_accents=__SCREAMING_SNAKE_CASE					)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)  ,						['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			List[Any]					)			->			Tuple:
							a_		:  Any											=      RoCBertBasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE  ,						never_split=['''[UNK]''']					)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''					)  ,						['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			List[str]					)			->			int:
							a_		:  str											=      ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

							a_		:  Union[str, Any]											=      {}
							for i, token in enumerate(__SCREAMING_SNAKE_CASE					):
											a_		:  Union[str, Any]											=      i
							a_		:  int											=      RoCBertWordpieceTokenizer(vocab=__SCREAMING_SNAKE_CASE  ,						unk_token='''[UNK]'''					)

							self.assertListEqual(tokenizer.tokenize(''''''					)  ,						[]					)

							self.assertListEqual(tokenizer.tokenize('''unwanted running'''					)  ,						['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']					)

							self.assertListEqual(tokenizer.tokenize('''unwantedX running'''					)  ,						['''[UNK]''', '''runn''', '''##ing''']					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Any					)			->			int:
							self.assertTrue(_is_whitespace(''' '''					)					)
							self.assertTrue(_is_whitespace('''\t'''					)					)
							self.assertTrue(_is_whitespace('''\r'''					)					)
							self.assertTrue(_is_whitespace('''\n'''					)					)
							self.assertTrue(_is_whitespace('''\u00A0'''					)					)

							self.assertFalse(_is_whitespace('''A'''					)					)
							self.assertFalse(_is_whitespace('''-'''					)					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[Any]					)			->			List[str]:
							self.assertTrue(_is_control('''\u0005'''					)					)

							self.assertFalse(_is_control('''A'''					)					)
							self.assertFalse(_is_control(''' '''					)					)
							self.assertFalse(_is_control('''\t'''					)					)
							self.assertFalse(_is_control('''\r'''					)					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Union[str, Any]					)			->			Dict:
							self.assertTrue(_is_punctuation('''-'''					)					)
							self.assertTrue(_is_punctuation('''$'''					)					)
							self.assertTrue(_is_punctuation('''`'''					)					)
							self.assertTrue(_is_punctuation('''.'''					)					)

							self.assertFalse(_is_punctuation('''A'''					)					)
							self.assertFalse(_is_punctuation(''' '''					)					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Any					)			->			Optional[int]:
							a_		:  List[Any]											=      self.get_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(__SCREAMING_SNAKE_CASE					) for t in ['''Test''', '''\xad''', '''test''']]  ,						[['''[UNK]'''], [], ['''[UNK]''']]					)

							if self.test_rust_tokenizer:
											a_		:  Tuple											=      self.get_rust_tokenizer()
											self.assertListEqual(
											    [rust_tokenizer.tokenize(__SCREAMING_SNAKE_CASE					) for t in ['''Test''', '''\xad''', '''test''']]  ,						[['''[UNK]'''], [], ['''[UNK]''']]					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Dict					)			->			int:
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'					):
															a_		:  str											=      self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE  ,						**__SCREAMING_SNAKE_CASE					)

															a_		:  Dict											=      f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
															a_		:  str											=      tokenizer_r.encode_plus(
															    __SCREAMING_SNAKE_CASE  ,						return_attention_mask=__SCREAMING_SNAKE_CASE  ,						return_token_type_ids=__SCREAMING_SNAKE_CASE  ,						return_offsets_mapping=__SCREAMING_SNAKE_CASE  ,						add_special_tokens=__SCREAMING_SNAKE_CASE  ,						)

															a_		:  Optional[int]											=      tokenizer_r.do_lower_case if hasattr(__SCREAMING_SNAKE_CASE  ,						'''do_lower_case'''					) else False
															a_		:  List[str]											=      (
															    [
															        ((0, 0), tokenizer_r.cls_token),
															        ((0, 1), '''A'''),
															        ((1, 2), ''','''),
															        ((3, 5), '''na'''),
															        ((5, 6), '''##ï'''),
															        ((6, 8), '''##ve'''),
															        ((9, 15), tokenizer_r.mask_token),
															        ((16, 21), '''Allen'''),
															        ((21, 23), '''##NL'''),
															        ((23, 24), '''##P'''),
															        ((25, 33), '''sentence'''),
															        ((33, 34), '''.'''),
															        ((0, 0), tokenizer_r.sep_token),
															    ]
															    if not do_lower_case
															    else [
															        ((0, 0), tokenizer_r.cls_token),
															        ((0, 1), '''a'''),
															        ((1, 2), ''','''),
															        ((3, 8), '''naive'''),
															        ((9, 15), tokenizer_r.mask_token),
															        ((16, 21), '''allen'''),
															        ((21, 23), '''##nl'''),
															        ((23, 24), '''##p'''),
															        ((25, 33), '''sentence'''),
															        ((33, 34), '''.'''),
															        ((0, 0), tokenizer_r.sep_token),
															    ]
															)

															self.assertEqual(
															    [e[1] for e in expected_results]  ,						tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']					)					)
															self.assertEqual([e[0] for e in expected_results]  ,						tokens['''offset_mapping''']					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			List[str]					)			->			List[Any]:
							a_		:  List[Any]											=      ['''的''', '''人''', '''有''']
							a_		:  Optional[int]											=      ''''''.join(__SCREAMING_SNAKE_CASE					)
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'					):
															a_		:  str											=      True
															a_		:  List[str]											=      self.tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE  ,						**__SCREAMING_SNAKE_CASE					)
															a_		:  int											=      self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE  ,						**__SCREAMING_SNAKE_CASE					)

															a_		:  str											=      tokenizer_p.encode(__SCREAMING_SNAKE_CASE  ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
															a_		:  Optional[int]											=      tokenizer_r.encode(__SCREAMING_SNAKE_CASE  ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)

															a_		:  Union[str, Any]											=      tokenizer_r.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE					)
															a_		:  int											=      tokenizer_p.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE					)

															# it is expected that each Chinese character is not preceded by "##"
															self.assertListEqual(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)
															self.assertListEqual(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)

															a_		:  List[Any]											=      False
															a_		:  List[Any]											=      self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE  ,						**__SCREAMING_SNAKE_CASE					)
															a_		:  Optional[int]											=      self.tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE  ,						**__SCREAMING_SNAKE_CASE					)

															a_		:  int											=      tokenizer_r.encode(__SCREAMING_SNAKE_CASE  ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
															a_		:  Optional[Any]											=      tokenizer_p.encode(__SCREAMING_SNAKE_CASE  ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)

															a_		:  List[Any]											=      tokenizer_r.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE					)
															a_		:  int											=      tokenizer_p.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE					)

															# it is expected that only the first Chinese character is not preceded by "##".
															a_		:  Optional[int]											=      [
															    f'##{token}' if idx != 0 else token for idx, token in enumerate(__SCREAMING_SNAKE_CASE					)
															]
															self.assertListEqual(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)
															self.assertListEqual(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)






			@slow
			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[Any]					)			->			Tuple:
							a_		:  Optional[Any]											=      self.tokenizer_class(self.vocab_file  ,						self.word_shape_file  ,						self.word_pronunciation_file					)

							a_		:  str											=      tokenizer.encode('''你好'''  ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
							a_		:  Optional[Any]											=      tokenizer.encode('''你是谁'''  ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)

							a_		:  Union[str, Any]											=      tokenizer.build_inputs_with_special_tokens(__SCREAMING_SNAKE_CASE					)
							a_		:  Tuple											=      tokenizer.build_inputs_with_special_tokens(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)

							assert encoded_sentence == [1] + text + [2]
							assert encoded_pair == [1] + text + [2] + text_a + [2]







			def 			SCREAMING_SNAKE_CASE      ( self      :			str					)			->			Optional[int]:
							a_		:  Optional[int]											=      self.get_tokenizers(do_lower_case=__SCREAMING_SNAKE_CASE					)
							for tokenizer in tokenizers:
											with self.subTest(f'{tokenizer.__class__.__name__}'					):
															a_		:  Union[str, Any]											=      '''你好，你是谁'''
															a_		:  List[str]											=      tokenizer.tokenize(__SCREAMING_SNAKE_CASE					)
															a_		:  Tuple											=      tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE					)
															a_		:  List[Any]											=      tokenizer.convert_tokens_to_shape_ids(__SCREAMING_SNAKE_CASE					)
															a_		:  str											=      tokenizer.convert_tokens_to_pronunciation_ids(__SCREAMING_SNAKE_CASE					)
															a_		:  Union[str, Any]											=      tokenizer.prepare_for_model(
															    __SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE  ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)

															a_		:  List[Any]											=      tokenizer.encode_plus(__SCREAMING_SNAKE_CASE  ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)

															self.assertEqual(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)












'''simple docstring'''







import copy
from dataclasses import dataclass, field
from typing import ClassVar, Dict

from ..features import ClassLabel, Features, Image
from .base import TaskTemplate





@dataclass(frozen=SCREAMING_SNAKE_CASE_       )
class 	SCREAMING_SNAKE_CASE					(					SCREAMING_SNAKE_CASE_       ):
			snake_case__  =				field(default="image-classification"	,   metadata={"include_in_asdict_even_if_is_default": True}       )
			snake_case__  =				Features({"image": Image()}       )
			snake_case__  =				Features({"labels": ClassLabel}       )
			snake_case__  =				"image"
			snake_case__  =				"labels"






			def 			SCREAMING_SNAKE_CASE      ( self      :			List[str]  ,						__SCREAMING_SNAKE_CASE      :			List[Any]					)			->			Any:
							if self.label_column not in features:
											raise ValueError(f'Column {self.label_column} is not present in features.'					)
							if not isinstance(features[self.label_column]  ,						__SCREAMING_SNAKE_CASE					):
											raise ValueError(f'Column {self.label_column} is not a ClassLabel.'					)
							a_		:  Optional[int]											=      copy.deepcopy(self					)
							a_		:  int											=      self.label_schema.copy()
							a_		:  Tuple											=      features[self.label_column]
							a_		:  str											=      label_schema
							return task_template







			@property
			def 			SCREAMING_SNAKE_CASE      ( self      :			Tuple					)			->			Dict[str, str]:
							return {
							    self.image_column: "image",
							    self.label_column: "labels",
							}




