






'''simple docstring'''







import re
import string
from collections import Counter

import sacrebleu
import sacremoses
from packaging import version

import datasets


__lowerCAmelCase       		=      '\n@inproceedings{xu-etal-2016-optimizing,\n    title = {Optimizing Statistical Machine Translation for Text Simplification},\n    authors={Xu, Wei and Napoles, Courtney and Pavlick, Ellie and Chen, Quanze and Callison-Burch, Chris},\n    journal = {Transactions of the Association for Computational Linguistics},\n    volume = {4},\n    year={2016},\n    url = {https://www.aclweb.org/anthology/Q16-1029},\n    pages = {401--415\n},\n@inproceedings{post-2018-call,\n    title = "A Call for Clarity in Reporting {BLEU} Scores",\n    author = "Post, Matt",\n    booktitle = "Proceedings of the Third Conference on Machine Translation: Research Papers",\n    month = oct,\n    year = "2018",\n    address = "Belgium, Brussels",\n    publisher = "Association for Computational Linguistics",\n    url = "https://www.aclweb.org/anthology/W18-6319",\n    pages = "186--191",\n}\n'

__lowerCAmelCase       		=      '\\nWIKI_SPLIT is the combination of three metrics SARI, EXACT and SACREBLEU\nIt can be used to evaluate the quality of machine-generated texts.\n'


__lowerCAmelCase       		=      '\nCalculates sari score (between 0 and 100) given a list of source and predicted\nsentences, and a list of lists of reference sentences. It also computes the BLEU score as well as the exact match score.\nArgs:\n    sources: list of source sentences where each sentence should be a string.\n    predictions: list of predicted sentences where each sentence should be a string.\n    references: list of lists of reference sentences where each sentence should be a string.\nReturns:\n    sari: sari score\n    sacrebleu: sacrebleu score\n    exact: exact score\n\nExamples:\n    >>> sources=["About 95 species are currently accepted ."]\n    >>> predictions=["About 95 you now get in ."]\n    >>> references=[["About 95 species are currently known ."]]\n    >>> wiki_split = datasets.load_metric("wiki_split")\n    >>> results = wiki_split.compute(sources=sources, predictions=predictions, references=references)\n    >>> print(results)\n    {\'sari\': 21.805555555555557, \'sacrebleu\': 14.535768424205482, \'exact\': 0.0}\n'





def 		_UpperCAmelCase   (							__A							:   Union[str, Any]      ):

				def remove_articles(__A							:   Optional[Any]      ):
								a_		:  List[Any]											=      re.compile(R'''\b(a|an|the)\b'''						,		re.UNICODE      )
								return re.sub(__A						,		''' '''						,		__A      )

				def white_space_fix(__A							:   Optional[int]      ):
								return " ".join(text.split()      )

				def remove_punc(__A							:   List[Any]      ):
								a_		:  str											=      set(string.punctuation      )
								return "".join(ch for ch in text if ch not in exclude      )

				def lower(__A							:   Tuple      ):
								return text.lower()

				return white_space_fix(remove_articles(remove_punc(lower(__A      )      )      )      )





def 		_UpperCAmelCase   (							__A							:   str						,		__A							:   Optional[Any]      ):
				return int(normalize_answer(__A      ) == normalize_answer(__A      )      )





def 		_UpperCAmelCase   (							__A							:   Optional[int]						,		__A							:   List[Any]      ):
				a_		:  str											=      [any(compute_exact(__A						,		__A      ) for ref in refs      ) for pred, refs in zip(__A						,		__A      )]
				return (sum(__A      ) / len(__A      )) * 1_00





def 		_UpperCAmelCase   (							__A							:   Optional[Any]						,		__A							:   Any						,		__A							:   Any						,		__A							:   List[Any]      ):
				a_		:  str											=      [rgram for rgrams in rgramslist for rgram in rgrams]
				a_		:  Optional[int]											=      Counter(__A      )

				a_		:  Dict											=      Counter(__A      )
				a_		:  Tuple											=      Counter()
				for sgram, scount in sgramcounter.items():
								a_		:  Dict											=      scount * numref

				a_		:  List[str]											=      Counter(__A      )
				a_		:  Dict											=      Counter()
				for cgram, ccount in cgramcounter.items():
								a_		:  List[str]											=      ccount * numref

				# KEEP
				a_		:  str											=      sgramcounter_rep & cgramcounter_rep
				a_		:  int											=      keepgramcounter_rep & rgramcounter
				a_		:  Tuple											=      sgramcounter_rep & rgramcounter

				a_		:  Dict											=      0
				a_		:  str											=      0
				for keepgram in keepgramcountergood_rep:
								keeptmpscorea += keepgramcountergood_rep[keepgram] / keepgramcounter_rep[keepgram]
								# Fix an alleged bug [2] in the keep score computation.
								# keeptmpscore2 += keepgramcountergood_rep[keepgram] / keepgramcounterall_rep[keepgram]
								keeptmpscorea += keepgramcountergood_rep[keepgram]
				# Define 0/0=1 instead of 0 to give higher scores for predictions that match
				#      a target exactly.
				a_		:  List[str]											=      1
				a_		:  str											=      1
				if len(__A      ) > 0:
								a_		:  List[Any]											=      keeptmpscorea / len(__A      )
				if len(__A      ) > 0:
								# Fix an alleged bug [2] in the keep score computation.
								# keepscore_recall = keeptmpscore2 / len(keepgramcounterall_rep)
								a_		:  int											=      keeptmpscorea / sum(keepgramcounterall_rep.values()      )
				a_		:  List[Any]											=      0
				if keepscore_precision > 0 or keepscore_recall > 0:
								a_		:  str											=      2 * keepscore_precision * keepscore_recall / (keepscore_precision + keepscore_recall)

				# DELETION
				a_		:  Optional[int]											=      sgramcounter_rep - cgramcounter_rep
				a_		:  Dict											=      delgramcounter_rep - rgramcounter
				a_		:  Optional[Any]											=      sgramcounter_rep - rgramcounter
				a_		:  int											=      0
				a_		:  Any											=      0
				for delgram in delgramcountergood_rep:
								deltmpscorea += delgramcountergood_rep[delgram] / delgramcounter_rep[delgram]
								deltmpscorea += delgramcountergood_rep[delgram] / delgramcounterall_rep[delgram]
				# Define 0/0=1 instead of 0 to give higher scores for predictions that match
				# a target exactly.
				a_		:  int											=      1
				if len(__A      ) > 0:
								a_		:  Tuple											=      deltmpscorea / len(__A      )

				# ADDITION
				a_		:  Optional[Any]											=      set(__A      ) - set(__A      )
				a_		:  Optional[int]											=      set(__A      ) & set(__A      )
				a_		:  Any											=      set(__A      ) - set(__A      )

				a_		:  str											=      0
				for addgram in addgramcountergood:
								addtmpscore += 1

				# Define 0/0=1 instead of 0 to give higher scores for predictions that match
				# a target exactly.
				a_		:  List[str]											=      1
				a_		:  Optional[Any]											=      1
				if len(__A      ) > 0:
								a_		:  Optional[int]											=      addtmpscore / len(__A      )
				if len(__A      ) > 0:
								a_		:  Dict											=      addtmpscore / len(__A      )
				a_		:  str											=      0
				if addscore_precision > 0 or addscore_recall > 0:
								a_		:  List[Any]											=      2 * addscore_precision * addscore_recall / (addscore_precision + addscore_recall)

				return (keepscore, delscore_precision, addscore)





def 		_UpperCAmelCase   (							__A							:   Optional[Any]						,		__A							:   List[str]						,		__A							:   Optional[Any]      ):
				a_		:  Tuple											=      len(__A      )

				a_		:  List[str]											=      ssent.split(''' '''      )
				a_		:  Optional[Any]											=      csent.split(''' '''      )
				a_		:  Dict											=      []
				a_		:  Dict											=      []
				a_		:  List[str]											=      []
				a_		:  Any											=      []
				a_		:  int											=      []
				a_		:  int											=      []

				a_		:  int											=      []
				a_		:  List[str]											=      []
				a_		:  int											=      []
				a_		:  str											=      []
				for rsent in rsents:
								a_		:  int											=      rsent.split(''' '''      )
								a_		:  Union[str, Any]											=      []
								a_		:  Dict											=      []
								a_		:  Any											=      []
								ragramslist.append(__A      )
								for i in range(0						,		len(__A      ) - 1      ):
												if i < len(__A      ) - 1:
																a_		:  Dict											=      ragrams[i] + ''' ''' + ragrams[i + 1]
																ragrams.append(__A      )
												if i < len(__A      ) - 2:
																a_		:  Tuple											=      ragrams[i] + ''' ''' + ragrams[i + 1] + ''' ''' + ragrams[i + 2]
																ragrams.append(__A      )
												if i < len(__A      ) - 3:
																a_		:  Any											=      ragrams[i] + ''' ''' + ragrams[i + 1] + ''' ''' + ragrams[i + 2] + ''' ''' + ragrams[i + 3]
																ragrams.append(__A      )
								ragramslist.append(__A      )
								ragramslist.append(__A      )
								ragramslist.append(__A      )

				for i in range(0						,		len(__A      ) - 1      ):
								if i < len(__A      ) - 1:
												a_		:  Dict											=      sagrams[i] + ''' ''' + sagrams[i + 1]
												sagrams.append(__A      )
								if i < len(__A      ) - 2:
												a_		:  Optional[Any]											=      sagrams[i] + ''' ''' + sagrams[i + 1] + ''' ''' + sagrams[i + 2]
												sagrams.append(__A      )
								if i < len(__A      ) - 3:
												a_		:  Optional[int]											=      sagrams[i] + ''' ''' + sagrams[i + 1] + ''' ''' + sagrams[i + 2] + ''' ''' + sagrams[i + 3]
												sagrams.append(__A      )

				for i in range(0						,		len(__A      ) - 1      ):
								if i < len(__A      ) - 1:
												a_		:  Optional[int]											=      cagrams[i] + ''' ''' + cagrams[i + 1]
												cagrams.append(__A      )
								if i < len(__A      ) - 2:
												a_		:  Union[str, Any]											=      cagrams[i] + ''' ''' + cagrams[i + 1] + ''' ''' + cagrams[i + 2]
												cagrams.append(__A      )
								if i < len(__A      ) - 3:
												a_		:  List[Any]											=      cagrams[i] + ''' ''' + cagrams[i + 1] + ''' ''' + cagrams[i + 2] + ''' ''' + cagrams[i + 3]
												cagrams.append(__A      )

				((a_)				,			(a_)				,			(a_))		:  str											=      SARIngram(__A						,		__A						,		__A						,		__A      )
				((a_)				,			(a_)				,			(a_))		:  List[str]											=      SARIngram(__A						,		__A						,		__A						,		__A      )
				((a_)				,			(a_)				,			(a_))		:  Union[str, Any]											=      SARIngram(__A						,		__A						,		__A						,		__A      )
				((a_)				,			(a_)				,			(a_))		:  Optional[Any]											=      SARIngram(__A						,		__A						,		__A						,		__A      )
				a_		:  Tuple											=      sum([keepascore, keepascore, keepascore, keepascore]      ) / 4
				a_		:  List[str]											=      sum([delascore, delascore, delascore, delascore]      ) / 4
				a_		:  Union[str, Any]											=      sum([addascore, addascore, addascore, addascore]      ) / 4
				a_		:  Union[str, Any]											=      (avgkeepscore + avgdelscore + avgaddscore) / 3
				return finalscore





def 		_UpperCAmelCase   (							__A							:   Any						,		__A							:   bool = True						,		__A							:   str = "13a"						,		__A							:   bool = True      ):
				# Normalization is requried for the ASSET dataset (one of the primary
				# datasets in sentence simplification) to allow using space
				# to split the sentence. Even though Wiki-Auto and TURK datasets,
				# do not require normalization, we do it for consistency.
				# Code adapted from the EASSE library [1] written by the authors of the ASSET dataset.
				# [1] https://github.com/feralvam/easse/blob/580bba7e1378fc8289c663f864e0487188fe8067/easse/utils/preprocessing.py#L7

				if lowercase:
								a_		:  Optional[Any]											=      sentence.lower()

				if tokenizer in ["13a", "intl"]:
								if version.parse(sacrebleu.__version__      ).major >= 2:
												a_		:  Dict											=      sacrebleu.metrics.bleu._get_tokenizer(__A      )()(__A      )
								else:
												a_		:  Union[str, Any]											=      sacrebleu.TOKENIZERS[tokenizer]()(__A      )
				elif tokenizer == "moses":
								a_		:  Dict											=      sacremoses.MosesTokenizer().tokenize(__A						,		return_str=__A						,		escape=__A      )
				elif tokenizer == "penn":
								a_		:  Tuple											=      sacremoses.MosesTokenizer().penn_tokenize(__A						,		return_str=__A      )
				else:
								a_		:  Any											=      sentence

				if not return_str:
								a_		:  Optional[Any]											=      normalized_sent.split()

				return normalized_sent





def 		_UpperCAmelCase   (							__A							:   Dict						,		__A							:   int						,		__A							:   Optional[Any]      ):
				if not (len(__A      ) == len(__A      ) == len(__A      )):
								raise ValueError('''Sources length must match predictions and references lengths.'''      )
				a_		:  Optional[Any]											=      0
				for src, pred, refs in zip(__A						,		__A						,		__A      ):
								sari_score += SARIsent(normalize(__A      )						,		normalize(__A      )						,		[normalize(__A      ) for sent in refs]      )
				a_		:  str											=      sari_score / len(__A      )
				return 1_00 * sari_score





def 		_UpperCAmelCase   (							__A							:   Dict						,		__A							:   str						,		__A							:   str="exp"						,		__A							:   int=None						,		__A							:   int=False						,		__A							:   Union[str, Any]=False						,		__A							:   str=False						,		):
				a_		:  Optional[Any]											=      len(references[0]      )
				if any(len(__A      ) != references_per_prediction for refs in references      ):
								raise ValueError('''Sacrebleu requires the same number of references for each prediction'''      )
				a_		:  Dict											=      [[refs[i] for refs in references] for i in range(__A      )]
				a_		:  List[str]											=      sacrebleu.corpus_bleu(
				    __A						,		__A						,		smooth_method=__A						,		smooth_value=__A						,		force=__A						,		lowercase=__A						,		use_effective_order=__A						,		)
				return output.score





@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION	,   _KWARGS_DESCRIPTION       )
class 	SCREAMING_SNAKE_CASE					(					datasets.Metric       ):






			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[int]					)			->			Optional[Any]:
							return datasets.MetricInfo(
							    description=_DESCRIPTION  ,						citation=_CITATION  ,						inputs_description=_KWARGS_DESCRIPTION  ,						features=datasets.Features(
							        {
							            '''predictions''': datasets.Value('''string'''  ,						id='''sequence'''					),
							            '''references''': datasets.Sequence(datasets.Value('''string'''  ,						id='''sequence'''					)  ,						id='''references'''					),
							        }					)  ,						codebase_urls=[
							        '''https://github.com/huggingface/transformers/blob/master/src/transformers/data/metrics/squad_metrics.py''',
							        '''https://github.com/cocoxu/simplification/blob/master/SARI.py''',
							        '''https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/utils/sari_hook.py''',
							        '''https://github.com/mjpost/sacreBLEU''',
							    ]  ,						reference_urls=[
							        '''https://www.aclweb.org/anthology/Q16-1029.pdf''',
							        '''https://github.com/mjpost/sacreBLEU''',
							        '''https://en.wikipedia.org/wiki/BLEU''',
							        '''https://towardsdatascience.com/evaluating-text-output-in-nlp-bleu-at-your-own-risk-e8609665a213''',
							    ]  ,						)







			def 			SCREAMING_SNAKE_CASE      ( self      :			Union[str, Any]  ,						__SCREAMING_SNAKE_CASE      :			Union[str, Any]  ,						__SCREAMING_SNAKE_CASE      :			Any  ,						__SCREAMING_SNAKE_CASE      :			Tuple					)			->			Optional[int]:
							a_		:  Any											=      {}
							result.update({'''sari''': compute_sari(sources=__SCREAMING_SNAKE_CASE  ,						predictions=__SCREAMING_SNAKE_CASE  ,						references=__SCREAMING_SNAKE_CASE					)}					)
							result.update({'''sacrebleu''': compute_sacrebleu(predictions=__SCREAMING_SNAKE_CASE  ,						references=__SCREAMING_SNAKE_CASE					)}					)
							result.update({'''exact''': compute_em(predictions=__SCREAMING_SNAKE_CASE  ,						references=__SCREAMING_SNAKE_CASE					)}					)
							return result












'''simple docstring'''







import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__lowerCAmelCase       		=      logging.get_logger(__name__)

__lowerCAmelCase       		=      {'vocab_file': 'sentencepiece.bpe.model'}

__lowerCAmelCase       		=      {
    'vocab_file': {
        'moussaKam/mbarthez': 'https://huggingface.co/moussaKam/mbarthez/resolve/main/sentencepiece.bpe.model',
        'moussaKam/barthez': 'https://huggingface.co/moussaKam/barthez/resolve/main/sentencepiece.bpe.model',
        'moussaKam/barthez-orangesum-title': (
            'https://huggingface.co/moussaKam/barthez-orangesum-title/resolve/main/sentencepiece.bpe.model'
        ),
    },
}

__lowerCAmelCase       		=      {
    'moussaKam/mbarthez': 1_024,
    'moussaKam/barthez': 1_024,
    'moussaKam/barthez-orangesum-title': 1_024,
}

__lowerCAmelCase       		=      '▁'





class 	SCREAMING_SNAKE_CASE					(					SCREAMING_SNAKE_CASE_       ):
			snake_case__  =				VOCAB_FILES_NAMES
			snake_case__  =				PRETRAINED_VOCAB_FILES_MAP
			snake_case__  =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			snake_case__  =				["input_ids", "attention_mask"]






			def __init__( self      :			Dict  ,						__SCREAMING_SNAKE_CASE      :			str  ,						__SCREAMING_SNAKE_CASE      :			Dict="<s>"  ,						__SCREAMING_SNAKE_CASE      :			List[Any]="</s>"  ,						__SCREAMING_SNAKE_CASE      :			List[str]="</s>"  ,						__SCREAMING_SNAKE_CASE      :			List[str]="<s>"  ,						__SCREAMING_SNAKE_CASE      :			Dict="<unk>"  ,						__SCREAMING_SNAKE_CASE      :			int="<pad>"  ,						__SCREAMING_SNAKE_CASE      :			Tuple="<mask>"  ,						__SCREAMING_SNAKE_CASE      :			Optional[Dict[str, Any]] = None  ,						**__SCREAMING_SNAKE_CASE      :			Optional[Any]  ,						)			->			None:
							# Mask token behave like a normal word, i.e. include the space before it
							a_		:  Tuple											=      AddedToken(__SCREAMING_SNAKE_CASE  ,						lstrip=__SCREAMING_SNAKE_CASE  ,						rstrip=__SCREAMING_SNAKE_CASE					) if isinstance(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					) else mask_token

							a_		:  List[str]											=      {} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    bos_token=__SCREAMING_SNAKE_CASE  ,						eos_token=__SCREAMING_SNAKE_CASE  ,						unk_token=__SCREAMING_SNAKE_CASE  ,						sep_token=__SCREAMING_SNAKE_CASE  ,						cls_token=__SCREAMING_SNAKE_CASE  ,						pad_token=__SCREAMING_SNAKE_CASE  ,						mask_token=__SCREAMING_SNAKE_CASE  ,						sp_model_kwargs=self.sp_model_kwargs  ,						**__SCREAMING_SNAKE_CASE  ,						)

							a_		:  Tuple											=      vocab_file
							a_		:  Optional[Any]											=      spm.SentencePieceProcessor(**self.sp_model_kwargs					)
							self.sp_model.Load(str(__SCREAMING_SNAKE_CASE					)					)

							a_		:  Optional[Any]											=      {'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

							a_		:  Any											=      len(self.sp_model					) - 1
							a_		:  Optional[Any]											=      {v: k for k, v in self.fairseq_tokens_to_ids.items()}






			def 			SCREAMING_SNAKE_CASE      ( self      :			str  ,						__SCREAMING_SNAKE_CASE      :			List[int]  ,						__SCREAMING_SNAKE_CASE      :			Optional[List[int]] = None					)			->			List[int]:

							if token_ids_a is None:
											return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
							a_		:  List[str]											=      [self.cls_token_id]
							a_		:  Optional[Any]											=      [self.sep_token_id]
							return cls + token_ids_a + sep + sep + token_ids_a + sep






			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[int]  ,						__SCREAMING_SNAKE_CASE      :			List[int]  ,						__SCREAMING_SNAKE_CASE      :			Optional[List[int]] = None  ,						__SCREAMING_SNAKE_CASE      :			bool = False					)			->			List[int]:
							if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=__SCREAMING_SNAKE_CASE  ,						token_ids_a=__SCREAMING_SNAKE_CASE  ,						already_has_special_tokens=__SCREAMING_SNAKE_CASE					)

							if token_ids_a is None:
											return [1] + ([0] * len(__SCREAMING_SNAKE_CASE					)) + [1]
							return [1] + ([0] * len(__SCREAMING_SNAKE_CASE					)) + [1, 1] + ([0] * len(__SCREAMING_SNAKE_CASE					)) + [1]






			def 			SCREAMING_SNAKE_CASE      ( self      :			Union[str, Any]  ,						__SCREAMING_SNAKE_CASE      :			List[int]  ,						__SCREAMING_SNAKE_CASE      :			Optional[List[int]] = None					)			->			List[int]:
							a_		:  List[str]											=      [self.sep_token_id]
							a_		:  Any											=      [self.cls_token_id]

							if token_ids_a is None:
											return len(cls + token_ids_a + sep					) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]






			@property
			def 			SCREAMING_SNAKE_CASE      ( self      :			Any					)			->			Union[str, Any]:
							return len(self.sp_model					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			str					)			->			Tuple:
							a_		:  int											=      {self.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE					): i for i in range(self.vocab_size					)}
							vocab.update(self.added_tokens_encoder					)
							return vocab






			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[Any]  ,						__SCREAMING_SNAKE_CASE      :			str					)			->			List[str]:
							return self.sp_model.encode(__SCREAMING_SNAKE_CASE  ,						out_type=__SCREAMING_SNAKE_CASE					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			str  ,						__SCREAMING_SNAKE_CASE      :			List[Any]					)			->			Dict:
							if token in self.fairseq_tokens_to_ids:
											return self.fairseq_tokens_to_ids[token]
							a_		:  Optional[int]											=      self.sp_model.PieceToId(__SCREAMING_SNAKE_CASE					)

							return spm_id if spm_id else self.unk_token_id






			def 			SCREAMING_SNAKE_CASE      ( self      :			List[str]  ,						__SCREAMING_SNAKE_CASE      :			int					)			->			str:
							if index in self.fairseq_ids_to_tokens:
											return self.fairseq_ids_to_tokens[index]
							return self.sp_model.IdToPiece(__SCREAMING_SNAKE_CASE					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			str  ,						__SCREAMING_SNAKE_CASE      :			Optional[Any]					)			->			Optional[int]:
							a_		:  Dict											=      []
							a_		:  List[Any]											=      ''''''
							a_		:  Dict											=      False
							for token in tokens:
											# make sure that special tokens are not decoded using sentencepiece model
											if token in self.all_special_tokens:
															if not prev_is_special:
																			out_string += " "
															out_string += self.sp_model.decode(__SCREAMING_SNAKE_CASE					) + token
															a_		:  Dict											=      True
															a_		:  Optional[Any]											=      []
											else:
															current_sub_tokens.append(__SCREAMING_SNAKE_CASE					)
															a_		:  Tuple											=      False
							out_string += self.sp_model.decode(__SCREAMING_SNAKE_CASE					)
							return out_string.strip()






			def __getstate__( self      :			Dict					)			->			int:
							a_		:  Dict											=      self.__dict__.copy()
							a_		:  List[str]											=      None
							return state






			def __setstate__( self      :			Dict  ,						__SCREAMING_SNAKE_CASE      :			Optional[Any]					)			->			Any:
							a_		:  Optional[Any]											=      d

							# for backward compatibility
							if not hasattr(self  ,						'''sp_model_kwargs'''					):
											a_		:  Union[str, Any]											=      {}

							a_		:  int											=      spm.SentencePieceProcessor(**self.sp_model_kwargs					)
							self.sp_model.Load(self.vocab_file					)







			def 			SCREAMING_SNAKE_CASE      ( self      :			str  ,						__SCREAMING_SNAKE_CASE      :			str  ,						__SCREAMING_SNAKE_CASE      :			Optional[str] = None					)			->			Tuple[str]:
							if not os.path.isdir(__SCREAMING_SNAKE_CASE					):
											logger.error(f'Vocabulary path ({save_directory}) should be a directory'					)
											return
							a_		:  Union[str, Any]											=      os.path.join(
							    __SCREAMING_SNAKE_CASE  ,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)

							if os.path.abspath(self.vocab_file					) != os.path.abspath(__SCREAMING_SNAKE_CASE					) and os.path.isfile(self.vocab_file					):
											copyfile(self.vocab_file  ,						__SCREAMING_SNAKE_CASE					)
							elif not os.path.isfile(self.vocab_file					):
											with open(__SCREAMING_SNAKE_CASE  ,						'''wb'''					) as fi:
															a_		:  Any											=      self.sp_model.serialized_model_proto()
															fi.write(__SCREAMING_SNAKE_CASE					)

							return (out_vocab_file,)




