



'''simple docstring'''


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
						from .tokenization_mbart import MBartTokenizer
else:
						lowercase_      :				int   					=    None


lowercase_      :				List[Any]   					=    logging.get_logger(__name__)


lowercase_      :				List[str]   					=    {'''vocab_file''': '''sentencepiece.bpe.model''', '''tokenizer_file''': '''tokenizer.json'''}

lowercase_      :				Dict   					=    {
    '''vocab_file''': {
        '''facebook/mbart-large-en-ro''': (
            '''https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model'''
        ),
        '''facebook/mbart-large-cc25''': (
            '''https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/mbart-large-en-ro''': '''https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json''',
        '''facebook/mbart-large-cc25''': '''https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json''',
    },
}

lowercase_      :				Union[str, Any]   					=    {
    '''facebook/mbart-large-en-ro''': 1024,
    '''facebook/mbart-large-cc25''': 1024,
}

# fmt: off
lowercase_      :				Dict   					=    ['''ar_AR''', '''cs_CZ''', '''de_DE''', '''en_XX''', '''es_XX''', '''et_EE''', '''fi_FI''', '''fr_XX''', '''gu_IN''', '''hi_IN''', '''it_IT''', '''ja_XX''', '''kk_KZ''', '''ko_KR''', '''lt_LT''', '''lv_LV''', '''my_MM''', '''ne_NP''', '''nl_XX''', '''ro_RO''', '''ru_RU''', '''si_LK''', '''tr_TR''', '''vi_VN''', '''zh_CN''']




class      __UpperCamelCase       (_UpperCAmelCase			):
	__A			  =					VOCAB_FILES_NAMES
	__A			  =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	__A			  =					PRETRAINED_VOCAB_FILES_MAP
	__A			  =					['''input_ids''', '''attention_mask''']
	__A			  =					MBartTokenizer

	__A			  =					[]
	__A			  =					[]







	def __init__(    self ,				_lowerCAmelCase=None ,				_lowerCAmelCase=None ,				_lowerCAmelCase="<s>" ,				_lowerCAmelCase="</s>" ,				_lowerCAmelCase="</s>" ,				_lowerCAmelCase="<s>" ,				_lowerCAmelCase="<unk>" ,				_lowerCAmelCase="<pad>" ,				_lowerCAmelCase="<mask>" ,				_lowerCAmelCase=None ,				_lowerCAmelCase=None ,				_lowerCAmelCase=None ,				**_lowerCAmelCase ,				)    ->    Optional[int]:
							'''simple docstring'''






							lowercase        =							AddedToken(_lowerCAmelCase ,				lstrip=_lowerCAmelCase ,				rstrip=_lowerCAmelCase						) if isinstance(_lowerCAmelCase ,				_lowerCAmelCase						) else mask_token

							super().__init__(
							    vocab_file=_lowerCAmelCase ,				tokenizer_file=_lowerCAmelCase ,				bos_token=_lowerCAmelCase ,				eos_token=_lowerCAmelCase ,				sep_token=_lowerCAmelCase ,				cls_token=_lowerCAmelCase ,				unk_token=_lowerCAmelCase ,				pad_token=_lowerCAmelCase ,				mask_token=_lowerCAmelCase ,				src_lang=_lowerCAmelCase ,				tgt_lang=_lowerCAmelCase ,				additional_special_tokens=_lowerCAmelCase ,				**_lowerCAmelCase ,				)

							lowercase        =							vocab_file
							lowercase        =							False if not self.vocab_file else True

							lowercase        =							FAIRSEQ_LANGUAGE_CODES.copy()

							if additional_special_tokens is not None:
													# Only add those special tokens if they are not already there.
													_additional_special_tokens.extend(
													    [t for t in additional_special_tokens if t not in _additional_special_tokens]						)

							self.add_special_tokens({"""additional_special_tokens""": _additional_special_tokens}						)
							lowercase        =							{
							    lang_code: self.convert_tokens_to_ids(_lowerCAmelCase						) for lang_code in FAIRSEQ_LANGUAGE_CODES
							}

							lowercase        =							src_lang if src_lang is not None else """en_XX"""
							lowercase        =							self.convert_tokens_to_ids(self._src_lang						)
							lowercase        =							tgt_lang
							self.set_src_lang_special_tokens(self._src_lang						)







	@property
	def   _a   (    self						)    ->    str:
							'''simple docstring'''






							return self._src_lang







	@src_lang.setter
	def   _a   (    self ,				_lowerCAmelCase						)    ->    None:
							'''simple docstring'''






							lowercase        =							new_src_lang
							self.set_src_lang_special_tokens(self._src_lang						)







	def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase = None						)    ->    List[int]:
							'''simple docstring'''






							if token_ids_a is None:
													return self.prefix_tokens + token_ids_a + self.suffix_tokens
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens







	def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase = None						)    ->    List[int]:
							'''simple docstring'''






							lowercase        =							[self.sep_token_id]
							lowercase        =							[self.cls_token_id]

							if token_ids_a is None:
													return len(cls + token_ids_a + sep						) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]







	def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				**_lowerCAmelCase						)    ->    Optional[int]:
							'''simple docstring'''






							if src_lang is None or tgt_lang is None:
													raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""						)
							lowercase        =							src_lang
							lowercase        =							self(_lowerCAmelCase ,				add_special_tokens=_lowerCAmelCase ,				return_tensors=_lowerCAmelCase ,				**_lowerCAmelCase						)
							lowercase        =							self.convert_tokens_to_ids(_lowerCAmelCase						)
							lowercase        =							tgt_lang_id
							return inputs







	def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase = "en_XX" ,				_lowerCAmelCase = None ,				_lowerCAmelCase = "ro_RO" ,				**_lowerCAmelCase ,				)    ->    BatchEncoding:
							'''simple docstring'''






							lowercase        =							src_lang
							lowercase        =							tgt_lang
							return super().prepare_seqaseq_batch(_lowerCAmelCase ,				_lowerCAmelCase ,				**_lowerCAmelCase						)







	def   _a   (    self						)    ->    List[Any]:
							'''simple docstring'''






							return self.set_src_lang_special_tokens(self.src_lang						)







	def   _a   (    self						)    ->    Any:
							'''simple docstring'''






							return self.set_tgt_lang_special_tokens(self.tgt_lang						)







	def   _a   (    self ,				_lowerCAmelCase						)    ->    None:
							'''simple docstring'''






							lowercase        =							self.convert_tokens_to_ids(_lowerCAmelCase						)
							lowercase        =							[]
							lowercase        =							[self.eos_token_id, self.cur_lang_code]

							lowercase        =							self.convert_ids_to_tokens(self.prefix_tokens						)
							lowercase        =							self.convert_ids_to_tokens(self.suffix_tokens						)

							lowercase        =							processors.TemplateProcessing(
							    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str ,				pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str ,				special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str ,				self.prefix_tokens + self.suffix_tokens						)						) ,				)







	def   _a   (    self ,				_lowerCAmelCase						)    ->    None:
							'''simple docstring'''






							lowercase        =							self.convert_tokens_to_ids(_lowerCAmelCase						)
							lowercase        =							[]
							lowercase        =							[self.eos_token_id, self.cur_lang_code]

							lowercase        =							self.convert_ids_to_tokens(self.prefix_tokens						)
							lowercase        =							self.convert_ids_to_tokens(self.suffix_tokens						)

							lowercase        =							processors.TemplateProcessing(
							    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str ,				pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str ,				special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str ,				self.prefix_tokens + self.suffix_tokens						)						) ,				)







	def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase = None						)    ->    Tuple[str]:
							'''simple docstring'''






							if not self.can_save_slow_tokenizer:
													raise ValueError(
													    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
													    """tokenizer."""						)

							if not os.path.isdir(_lowerCAmelCase						):
													logger.error(F"""Vocabulary path ({save_directory}) should be a directory."""						)
													return
							lowercase        =							os.path.join(
							    _lowerCAmelCase ,				(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)

							if os.path.abspath(self.vocab_file						) != os.path.abspath(_lowerCAmelCase						):
													copyfile(self.vocab_file ,				_lowerCAmelCase						)

							return (out_vocab_file,)







'''simple docstring'''


from random import shuffle

import tensorflow as tf
from numpy import array







def      SCREAMING_SNAKE_CASE				(				lowercase_    :      List[str] ,  lowercase_    :      Optional[int]						):
						lowercase        =							int(lowercase_						)
						assert noofclusters < len(lowercase_						)

						# Find out the dimensionality
						lowercase        =							len(vectors[0]						)

						# Will help select random centroids from among the available vectors
						lowercase        =							list(range(len(lowercase_						)						)						)
						shuffle(lowercase_						)

						# GRAPH OF COMPUTATION
						# We initialize a new graph and set it as the default during each run
						# of this algorithm. This ensures that as this function is called
						# multiple times, the default graph doesn't keep getting crowded with
						# unused ops and Variables from previous function calls.

						lowercase        =							tf.Graph()

						with graph.as_default():
												# SESSION OF COMPUTATION

												lowercase        =							tf.Session()

												##CONSTRUCTING THE ELEMENTS OF COMPUTATION

												##First lets ensure we have a Variable vector for each centroid,
												##initialized to one of the vectors from the available data points
												lowercase        =							[
												    tf.Variable(vectors[vector_indices[i]]						) for i in range(lowercase_						)
												]
												##These nodes will assign the centroid Variables the appropriate
												##values
												lowercase        =							tf.placeholder("""float64""" ,  [dim]						)
												lowercase        =							[]
												for centroid in centroids:
																		cent_assigns.append(tf.assign(lowercase_ ,  lowercase_						)						)

												##Variables for cluster assignments of individual vectors(initialized
												##to 0 at first)
												lowercase        =							[tf.Variable(0						) for i in range(len(lowercase_						)						)]
												##These nodes will assign an assignment Variable the appropriate
												##value
												lowercase        =							tf.placeholder("""int32"""						)
												lowercase        =							[]
												for assignment in assignments:
																		cluster_assigns.append(tf.assign(lowercase_ ,  lowercase_						)						)

												##Now lets construct the node that will compute the mean
												# The placeholder for the input
												lowercase        =							tf.placeholder("""float""" ,  [None, dim]						)
												# The Node/op takes the input and computes a mean along the 0th
												# dimension, i.e. the list of input vectors
												lowercase        =							tf.reduce_mean(lowercase_ ,  0						)

												##Node for computing Euclidean distances
												# Placeholders for input
												lowercase        =							tf.placeholder("""float""" ,  [dim]						)
												lowercase        =							tf.placeholder("""float""" ,  [dim]						)
												lowercase        =							tf.sqrt(tf.reduce_sum(tf.pow(tf.sub(lowercase_ ,  lowercase_						) ,  2						)						)						)

												##This node will figure out which cluster to assign a vector to,
												##based on Euclidean distances of the vector from the centroids.
												# Placeholder for input
												lowercase        =							tf.placeholder("""float""" ,  [noofclusters]						)
												lowercase        =							tf.argmin(lowercase_ ,  0						)

												##INITIALIZING STATE VARIABLES

												##This will help initialization of all Variables defined with respect
												##to the graph. The Variable-initializer should be defined after
												##all the Variables have been constructed, so that each of them
												##will be included in the initialization.
												lowercase        =							tf.initialize_all_variables()

												# Initialize all variables
												sess.run(lowercase_						)

												##CLUSTERING ITERATIONS

												# Now perform the Expectation-Maximization steps of K-Means clustering
												# iterations. To keep things simple, we will only do a set number of
												# iterations, instead of using a Stopping Criterion.
												lowercase        =							100
												for _ in range(lowercase_						):
																		##EXPECTATION STEP
																		##Based on the centroid locations till last iteration, compute
																		##the _expected_ centroid assignments.
																		# Iterate over each vector
																		for vector_n in range(len(lowercase_						)						):
																								lowercase        =							vectors[vector_n]
																								# Compute Euclidean distance between this vector and each
																								# centroid. Remember that this list cannot be named
																								#'centroid_distances', since that is the input to the
																								# cluster assignment node.
																								lowercase        =							[
																								    sess.run(lowercase_ ,  feed_dict={va: vect, va: sess.run(lowercase_						)}						)
																								    for centroid in centroids
																								]
																								# Now use the cluster assignment node, with the distances
																								# as the input
																								lowercase        =							sess.run(
																								    lowercase_ ,  feed_dict={centroid_distances: distances}						)
																								# Now assign the value to the appropriate state variable
																								sess.run(
																								    cluster_assigns[vector_n] ,  feed_dict={assignment_value: assignment}						)

																		##MAXIMIZATION STEP
																		# Based on the expected state computed from the Expectation Step,
																		# compute the locations of the centroids so as to maximize the
																		# overall objective of minimizing within-cluster Sum-of-Squares
																		for cluster_n in range(lowercase_						):
																								# Collect all the vectors assigned to this cluster
																								lowercase        =							[
																								    vectors[i]
																								    for i in range(len(lowercase_						)						)
																								    if sess.run(assignments[i]						) == cluster_n
																								]
																								# Compute new centroid location
																								lowercase        =							sess.run(
																								    lowercase_ ,  feed_dict={mean_input: array(lowercase_						)}						)
																								# Assign value to appropriate variable
																								sess.run(
																								    cent_assigns[cluster_n] ,  feed_dict={centroid_value: new_location}						)

        # Return centroids and assignments
												lowercase        =							sess.run(lowercase_						)
												lowercase        =							sess.run(lowercase_						)
												return centroids, assignments


