


'''simple docstring'''







import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__lowerCAmelCase       		=      logging.get_logger(__name__)

__lowerCAmelCase       		=      {
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

__lowerCAmelCase       		=      {
    "vocab_file": {"ctrl": "https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json"},
    "merges_file": {"ctrl": "https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt"},
}

__lowerCAmelCase       		=      {
    "ctrl": 256,
}

__lowerCAmelCase       		=      {
    "Pregnancy": 168_629,
    "Christianity": 7_675,
    "Explain": 106_423,
    "Fitness": 63_440,
    "Saving": 63_163,
    "Ask": 27_171,
    "Ass": 95_985,
    "Joke": 163_509,
    "Questions": 45_622,
    "Thoughts": 49_605,
    "Retail": 52_342,
    "Feminism": 164_338,
    "Writing": 11_992,
    "Atheism": 192_263,
    "Netflix": 48_616,
    "Computing": 39_639,
    "Opinion": 43_213,
    "Alone": 44_967,
    "Funny": 58_917,
    "Gaming": 40_358,
    "Human": 4_088,
    "India": 1_331,
    "Joker": 77_138,
    "Diet": 36_206,
    "Legal": 11_859,
    "Norman": 4_939,
    "Tip": 72_689,
    "Weight": 52_343,
    "Movies": 46_273,
    "Running": 23_425,
    "Science": 2_090,
    "Horror": 37_793,
    "Confession": 60_572,
    "Finance": 12_250,
    "Politics": 16_360,
    "Scary": 191_985,
    "Support": 12_654,
    "Technologies": 32_516,
    "Teenage": 66_160,
    "Event": 32_769,
    "Learned": 67_460,
    "Notion": 182_770,
    "Wikipedia": 37_583,
    "Books": 6_665,
    "Extract": 76_050,
    "Confessions": 102_701,
    "Conspiracy": 75_932,
    "Links": 63_674,
    "Narcissus": 150_425,
    "Relationship": 54_766,
    "Relationships": 134_796,
    "Reviews": 41_671,
    "News": 4_256,
    "Translation": 26_820,
    "multilingual": 128_406,
}





def 		_UpperCAmelCase   (							__A							:   Optional[int]      ):
				a_		:  Optional[Any]											=      set()
				a_		:  int											=      word[0]
				for char in word[1:]:
								pairs.add((prev_char, char)      )
								a_		:  Optional[int]											=      char

				a_		:  List[Any]											=      set(lowerCamelCase__      )
				return pairs





class 	SCREAMING_SNAKE_CASE					(					SCREAMING_SNAKE_CASE_       ):
			snake_case__  =				VOCAB_FILES_NAMES
			snake_case__  =				PRETRAINED_VOCAB_FILES_MAP
			snake_case__  =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			snake_case__  =				CONTROL_CODES






			def __init__( self      :			List[Any]  ,						__SCREAMING_SNAKE_CASE      :			int  ,						__SCREAMING_SNAKE_CASE      :			List[str]  ,						__SCREAMING_SNAKE_CASE      :			Any="<unk>"  ,						**__SCREAMING_SNAKE_CASE      :			Tuple					)			->			List[Any]:
							super().__init__(unk_token=UpperCamelCase_  ,						**UpperCamelCase_					)

							with open(UpperCamelCase_  ,						encoding='''utf-8'''					) as vocab_handle:
											a_		:  Union[str, Any]											=      json.load(UpperCamelCase_					)
							a_		:  Optional[int]											=      {v: k for k, v in self.encoder.items()}
							with open(UpperCamelCase_  ,						encoding='''utf-8'''					) as merges_handle:
											a_		:  List[Any]											=      merges_handle.read().split('''\n'''					)[1:-1]
							a_		:  Optional[int]											=      [tuple(merge.split()					) for merge in merges]
							a_		:  str											=      dict(zip(UpperCamelCase_  ,						range(len(UpperCamelCase_					)					)					)					)
							a_		:  List[Any]											=      {}






			@property
			def 			SCREAMING_SNAKE_CASE      ( self      :			Tuple					)			->			Tuple:
							return len(self.encoder					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			List[str]					)			->			int:
							return dict(self.encoder  ,						**self.added_tokens_encoder					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			List[str]  ,						__SCREAMING_SNAKE_CASE      :			Any					)			->			Optional[int]:
							if token in self.cache:
											return self.cache[token]
							a_		:  List[Any]											=      tuple(UpperCamelCase_					)
							a_		:  Union[str, Any]											=      tuple(list(word[:-1]					) + [word[-1] + '''</w>''']					)
							a_		:  Tuple											=      get_pairs(UpperCamelCase_					)

							if not pairs:
											return token

							while True:
											a_		:  int											=      min(UpperCamelCase_  ,						key=lambda __SCREAMING_SNAKE_CASE					: self.bpe_ranks.get(UpperCamelCase_  ,						float('''inf'''					)					)					)
											if bigram not in self.bpe_ranks:
															break
											a_		:  Optional[int]											=      bigram
											a_		:  int											=      []
											a_		:  Optional[Any]											=      0
											while i < len(UpperCamelCase_					):
															try:
																			a_		:  Any											=      word.index(UpperCamelCase_  ,						UpperCamelCase_					)
															except ValueError:
																			new_word.extend(word[i:]					)
																			break
															else:
																			new_word.extend(word[i:j]					)
																			a_		:  Optional[Any]											=      j

															if word[i] == first and i < len(UpperCamelCase_					) - 1 and word[i + 1] == second:
																			new_word.append(first + second					)
																			i += 2
															else:
																			new_word.append(word[i]					)
																			i += 1
											a_		:  Dict											=      tuple(UpperCamelCase_					)
											a_		:  int											=      new_word
											if len(UpperCamelCase_					) == 1:
															break
											else:
															a_		:  str											=      get_pairs(UpperCamelCase_					)
							a_		:  Any											=      "@@ ".join(UpperCamelCase_					)
							a_		:  Union[str, Any]											=      word[:-4]
							a_		:  Optional[Any]											=      word
							return word






			def 			SCREAMING_SNAKE_CASE      ( self      :			int  ,						__SCREAMING_SNAKE_CASE      :			List[Any]					)			->			Dict:
							a_		:  int											=      []

							a_		:  str											=      re.findall(r'''\S+\n?'''  ,						UpperCamelCase_					)

							for token in words:
											split_tokens.extend(list(self.bpe(UpperCamelCase_					).split(''' '''					)					)					)
							return split_tokens






			def 			SCREAMING_SNAKE_CASE      ( self      :			List[str]  ,						__SCREAMING_SNAKE_CASE      :			Optional[Any]					)			->			Any:
							return self.encoder.get(UpperCamelCase_  ,						self.encoder.get(self.unk_token					)					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Union[str, Any]  ,						__SCREAMING_SNAKE_CASE      :			Any					)			->			List[str]:
							return self.decoder.get(UpperCamelCase_  ,						self.unk_token					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			List[str]  ,						__SCREAMING_SNAKE_CASE      :			int					)			->			Optional[Any]:
							a_		:  List[Any]											=      " ".join(UpperCamelCase_					).replace('''@@ '''  ,						''''''					).strip()
							return out_string







			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[Any]  ,						__SCREAMING_SNAKE_CASE      :			Union[str, Any]  ,						__SCREAMING_SNAKE_CASE      :			Optional[Any] = None					)			->			List[str]:
							if not os.path.isdir(UpperCamelCase_					):
											logger.error(f'Vocabulary path ({save_directory}) should be a directory'					)
											return
							a_		:  int											=      os.path.join(
							    UpperCamelCase_  ,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)
							a_		:  List[Any]											=      os.path.join(
							    UpperCamelCase_  ,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']					)

							with open(UpperCamelCase_  ,						'''w'''  ,						encoding='''utf-8'''					) as f:
											f.write(json.dumps(self.encoder  ,						indent=2  ,						sort_keys=UpperCamelCase_  ,						ensure_ascii=UpperCamelCase_					) + '''\n'''					)

							a_		:  List[str]											=      0
							with open(UpperCamelCase_  ,						'''w'''  ,						encoding='''utf-8'''					) as writer:
											writer.write('''#version: 0.2\n'''					)
											for bpe_tokens, token_index in sorted(self.bpe_ranks.items()  ,						key=lambda __SCREAMING_SNAKE_CASE					: kv[1]					):
															if index != token_index:
																			logger.warning(
																			    f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
																			    ''' Please check that the tokenizer is not corrupted!'''					)
																			a_		:  List[str]											=      token_index
															writer.write(''' '''.join(UpperCamelCase_					) + '''\n'''					)
															index += 1

							return vocab_file, merge_file

			# def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
			#     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
			#     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
			#     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
			#     return ''.join(tokens_generated_so_far)












'''simple docstring'''







import copy
from dataclasses import dataclass, field
from typing import ClassVar, Dict

from ..features import ClassLabel, Features, Image
from .base import TaskTemplate





@dataclass(frozen=SCREAMING_SNAKE_CASE_       )
class 	SCREAMING_SNAKE_CASE					(					SCREAMING_SNAKE_CASE_       ):
			snake_case__  =				field(default="image-classification"	,   metadata={"include_in_asdict_even_if_is_default": True}       )
			snake_case__  =				Features({"image": Image()}       )
			snake_case__  =				Features({"labels": ClassLabel}       )
			snake_case__  =				"image"
			snake_case__  =				"labels"






			def 			SCREAMING_SNAKE_CASE      ( self      :			List[str]  ,						__SCREAMING_SNAKE_CASE      :			List[Any]					)			->			Any:
							if self.label_column not in features:
											raise ValueError(f'Column {self.label_column} is not present in features.'					)
							if not isinstance(features[self.label_column]  ,						__SCREAMING_SNAKE_CASE					):
											raise ValueError(f'Column {self.label_column} is not a ClassLabel.'					)
							a_		:  Optional[int]											=      copy.deepcopy(self					)
							a_		:  int											=      self.label_schema.copy()
							a_		:  Tuple											=      features[self.label_column]
							a_		:  str											=      label_schema
							return task_template







			@property
			def 			SCREAMING_SNAKE_CASE      ( self      :			Tuple					)			->			Dict[str, str]:
							return {
							    self.image_column: "image",
							    self.label_column: "labels",
							}




