



'''simple docstring'''


import unittest

from transformers import XLMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch

      from transformers import (
          XLMForMultipleChoice,
          XLMForQuestionAnswering,
          XLMForQuestionAnsweringSimple,
          XLMForSequenceClassification,
          XLMForTokenClassification,
          XLMModel,
          XLMWithLMHeadModel,
      )
      from transformers.models.xlm.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_LIST




class      __UpperCamelCase       :







 def __init__(    self ,				_lowerCAmelCase ,				_lowerCAmelCase=13 ,				_lowerCAmelCase=7 ,				_lowerCAmelCase=True ,				_lowerCAmelCase=True ,				_lowerCAmelCase=True ,				_lowerCAmelCase=True ,				_lowerCAmelCase=True ,				_lowerCAmelCase=False ,				_lowerCAmelCase=False ,				_lowerCAmelCase=False ,				_lowerCAmelCase=2 ,				_lowerCAmelCase=99 ,				_lowerCAmelCase=0 ,				_lowerCAmelCase=32 ,				_lowerCAmelCase=5 ,				_lowerCAmelCase=4 ,				_lowerCAmelCase=0.1 ,				_lowerCAmelCase=0.1 ,				_lowerCAmelCase=512 ,				_lowerCAmelCase=2 ,				_lowerCAmelCase=0.02 ,				_lowerCAmelCase=2 ,				_lowerCAmelCase=4 ,				_lowerCAmelCase="last" ,				_lowerCAmelCase=True ,				_lowerCAmelCase=None ,				_lowerCAmelCase=0 ,				)    ->    List[str]:
       '''simple docstring'''






       lowercase        =							parent
       lowercase        =							batch_size
       lowercase        =							seq_length
       lowercase        =							is_training
       lowercase        =							use_input_lengths
       lowercase        =							use_token_type_ids
       lowercase        =							use_labels
       lowercase        =							gelu_activation
       lowercase        =							sinusoidal_embeddings
       lowercase        =							causal
       lowercase        =							asm
       lowercase        =							n_langs
       lowercase        =							vocab_size
       lowercase        =							n_special
       lowercase        =							hidden_size
       lowercase        =							num_hidden_layers
       lowercase        =							num_attention_heads
       lowercase        =							hidden_dropout_prob
       lowercase        =							attention_probs_dropout_prob
       lowercase        =							max_position_embeddings
       lowercase        =							type_sequence_label_size
       lowercase        =							initializer_range
       lowercase        =							num_labels
       lowercase        =							num_choices
       lowercase        =							summary_type
       lowercase        =							use_proj
       lowercase        =							scope
       lowercase        =							bos_token_id







 def   _a   (    self						)    ->    Tuple:
       '''simple docstring'''






       lowercase        =							ids_tensor([self.batch_size, self.seq_length] ,				self.vocab_size						)
       lowercase        =							random_attention_mask([self.batch_size, self.seq_length]						)

       lowercase        =							None
       if self.use_input_lengths:
             lowercase        =							(
                 ids_tensor([self.batch_size] ,				vocab_size=2						) + self.seq_length - 2
             )  # small variation of seq_length

       lowercase        =							None
       if self.use_token_type_ids:
             lowercase        =							ids_tensor([self.batch_size, self.seq_length] ,				self.n_langs						)

       lowercase        =							None
       lowercase        =							None
       lowercase        =							None
       if self.use_labels:
             lowercase        =							ids_tensor([self.batch_size] ,				self.type_sequence_label_size						)
             lowercase        =							ids_tensor([self.batch_size, self.seq_length] ,				self.num_labels						)
             lowercase        =							ids_tensor([self.batch_size] ,				2						).float()
             lowercase        =							ids_tensor([self.batch_size] ,				self.num_choices						)

       lowercase        =							self.get_config()

       return (
           config,
           input_ids,
           token_type_ids,
           input_lengths,
           sequence_labels,
           token_labels,
           is_impossible_labels,
           choice_labels,
           input_mask,
       )







 def   _a   (    self						)    ->    int:
       '''simple docstring'''






       return XLMConfig(
           vocab_size=self.vocab_size ,				n_special=self.n_special ,				emb_dim=self.hidden_size ,				n_layers=self.num_hidden_layers ,				n_heads=self.num_attention_heads ,				dropout=self.hidden_dropout_prob ,				attention_dropout=self.attention_probs_dropout_prob ,				gelu_activation=self.gelu_activation ,				sinusoidal_embeddings=self.sinusoidal_embeddings ,				asm=self.asm ,				causal=self.causal ,				n_langs=self.n_langs ,				max_position_embeddings=self.max_position_embeddings ,				initializer_range=self.initializer_range ,				summary_type=self.summary_type ,				use_proj=self.use_proj ,				num_labels=self.num_labels ,				bos_token_id=self.bos_token_id ,				)







 def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				)    ->    str:
       '''simple docstring'''






       lowercase        =							XLMModel(config=lowerCamelCase_						)
       model.to(lowerCamelCase_						)
       model.eval()
       lowercase        =							model(lowerCamelCase_ ,				lengths=lowerCamelCase_ ,				langs=lowerCamelCase_						)
       lowercase        =							model(lowerCamelCase_ ,				langs=lowerCamelCase_						)
       lowercase        =							model(lowerCamelCase_						)
       self.parent.assertEqual(result.last_hidden_state.shape ,				(self.batch_size, self.seq_length, self.hidden_size)						)







 def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				)    ->    Tuple:
       '''simple docstring'''






       lowercase        =							XLMWithLMHeadModel(lowerCamelCase_						)
       model.to(lowerCamelCase_						)
       model.eval()

       lowercase        =							model(lowerCamelCase_ ,				token_type_ids=lowerCamelCase_ ,				labels=lowerCamelCase_						)
       self.parent.assertEqual(result.loss.shape ,				()						)
       self.parent.assertEqual(result.logits.shape ,				(self.batch_size, self.seq_length, self.vocab_size)						)







 def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				)    ->    Optional[Any]:
       '''simple docstring'''






       lowercase        =							XLMForQuestionAnsweringSimple(lowerCamelCase_						)
       model.to(lowerCamelCase_						)
       model.eval()

       lowercase        =							model(lowerCamelCase_						)

       lowercase        =							model(lowerCamelCase_ ,				start_positions=lowerCamelCase_ ,				end_positions=lowerCamelCase_						)
       lowercase        =							outputs
       self.parent.assertEqual(result.start_logits.shape ,				(self.batch_size, self.seq_length)						)
       self.parent.assertEqual(result.end_logits.shape ,				(self.batch_size, self.seq_length)						)







 def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				)    ->    int:
       '''simple docstring'''






       lowercase        =							XLMForQuestionAnswering(lowerCamelCase_						)
       model.to(lowerCamelCase_						)
       model.eval()

       lowercase        =							model(lowerCamelCase_						)

       lowercase        =							model(
           lowerCamelCase_ ,				start_positions=lowerCamelCase_ ,				end_positions=lowerCamelCase_ ,				cls_index=lowerCamelCase_ ,				is_impossible=lowerCamelCase_ ,				p_mask=lowerCamelCase_ ,				)

       lowercase        =							model(
           lowerCamelCase_ ,				start_positions=lowerCamelCase_ ,				end_positions=lowerCamelCase_ ,				cls_index=lowerCamelCase_ ,				is_impossible=lowerCamelCase_ ,				)

       (lowercase					)        =							result_with_labels.to_tuple()

       lowercase        =							model(lowerCamelCase_ ,				start_positions=lowerCamelCase_ ,				end_positions=lowerCamelCase_						)

       (lowercase					)        =							result_with_labels.to_tuple()

       self.parent.assertEqual(result_with_labels.loss.shape ,				()						)
       self.parent.assertEqual(result.start_top_log_probs.shape ,				(self.batch_size, model.config.start_n_top)						)
       self.parent.assertEqual(result.start_top_index.shape ,				(self.batch_size, model.config.start_n_top)						)
       self.parent.assertEqual(
           result.end_top_log_probs.shape ,				(self.batch_size, model.config.start_n_top * model.config.end_n_top)						)
       self.parent.assertEqual(
           result.end_top_index.shape ,				(self.batch_size, model.config.start_n_top * model.config.end_n_top)						)
       self.parent.assertEqual(result.cls_logits.shape ,				(self.batch_size,)						)







 def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				)    ->    List[str]:
       '''simple docstring'''






       lowercase        =							XLMForSequenceClassification(lowerCamelCase_						)
       model.to(lowerCamelCase_						)
       model.eval()

       lowercase        =							model(lowerCamelCase_						)
       lowercase        =							model(lowerCamelCase_ ,				labels=lowerCamelCase_						)
       self.parent.assertEqual(result.loss.shape ,				()						)
       self.parent.assertEqual(result.logits.shape ,				(self.batch_size, self.type_sequence_label_size)						)







 def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				)    ->    Any:
       '''simple docstring'''






       lowercase        =							self.num_labels
       lowercase        =							XLMForTokenClassification(lowerCamelCase_						)
       model.to(lowerCamelCase_						)
       model.eval()

       lowercase        =							model(lowerCamelCase_ ,				attention_mask=lowerCamelCase_ ,				labels=lowerCamelCase_						)
       self.parent.assertEqual(result.logits.shape ,				(self.batch_size, self.seq_length, self.num_labels)						)







 def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				)    ->    Optional[Any]:
       '''simple docstring'''






       lowercase        =							self.num_choices
       lowercase        =							XLMForMultipleChoice(config=lowerCamelCase_						)
       model.to(lowerCamelCase_						)
       model.eval()
       lowercase        =							input_ids.unsqueeze(1						).expand(-1 ,				self.num_choices ,				-1						).contiguous()
       lowercase        =							token_type_ids.unsqueeze(1						).expand(-1 ,				self.num_choices ,				-1						).contiguous()
       lowercase        =							input_mask.unsqueeze(1						).expand(-1 ,				self.num_choices ,				-1						).contiguous()
       lowercase        =							model(
           lowerCamelCase_ ,				attention_mask=lowerCamelCase_ ,				token_type_ids=lowerCamelCase_ ,				labels=lowerCamelCase_ ,				)
       self.parent.assertEqual(result.logits.shape ,				(self.batch_size, self.num_choices)						)







 def   _a   (    self						)    ->    int:
       '''simple docstring'''






       lowercase        =							self.prepare_config_and_inputs()
       (
           lowercase					
       )        =							config_and_inputs
       lowercase        =							{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''lengths''': input_lengths}
       return config, inputs_dict




@require_torch
class      __UpperCamelCase       (_UpperCAmelCase			,   _UpperCAmelCase			,   _UpperCAmelCase			,   unittest.TestCase			):
 __A			  =					(
     (
         XLMModel,
         XLMWithLMHeadModel,
         XLMForQuestionAnswering,
         XLMForSequenceClassification,
         XLMForQuestionAnsweringSimple,
         XLMForTokenClassification,
         XLMForMultipleChoice,
     )
     if is_torch_available()
     else ()
 )
 __A			  =					(
     (XLMWithLMHeadModel,) if is_torch_available() else ()
 )  # TODO (PVP): Check other models whether language generation is also applicable
 __A			  =					(
     {
         '''feature-extraction''': XLMModel,
         '''fill-mask''': XLMWithLMHeadModel,
         '''question-answering''': XLMForQuestionAnsweringSimple,
         '''text-classification''': XLMForSequenceClassification,
         '''text-generation''': XLMWithLMHeadModel,
         '''token-classification''': XLMForTokenClassification,
         '''zero-shot''': XLMForSequenceClassification,
     }
     if is_torch_available()
     else {}
 )







 def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase						)    ->    Tuple:
       '''simple docstring'''






       if (
           pipeline_test_casse_name == "QAPipelineTests"
           and tokenizer_name is not None
           and not tokenizer_name.endswith("""Fast"""						)
       ):
             # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
             # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
             # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
             return True

       return False







 def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase=False						)    ->    Any:
       '''simple docstring'''






       lowercase        =							super()._prepare_for_class(lowerCamelCase_ ,				lowerCamelCase_ ,				return_labels=lowerCamelCase_						)

       if return_labels:
             if model_class.__name__ == "XLMForQuestionAnswering":
                   lowercase        =							torch.zeros(
                       self.model_tester.batch_size ,				dtype=torch.long ,				device=lowerCamelCase_						)
                   lowercase        =							torch.zeros(
                       self.model_tester.batch_size ,				dtype=torch.long ,				device=lowerCamelCase_						)

       return inputs_dict







 def   _a   (    self						)    ->    int:
       '''simple docstring'''






       lowercase        =							XLMModelTester(self						)
       lowercase        =							ConfigTester(self ,				config_class=lowerCamelCase_ ,				emb_dim=37						)







 def   _a   (    self						)    ->    Dict:
       '''simple docstring'''






       self.config_tester.run_common_tests()







 def   _a   (    self						)    ->    int:
       '''simple docstring'''






       lowercase        =							self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_model(*lowerCamelCase_						)







 def   _a   (    self						)    ->    List[Any]:
       '''simple docstring'''






       lowercase        =							self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_lm_head(*lowerCamelCase_						)







 def   _a   (    self						)    ->    Union[str, Any]:
       '''simple docstring'''






       lowercase        =							self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_simple_qa(*lowerCamelCase_						)







 def   _a   (    self						)    ->    Optional[Any]:
       '''simple docstring'''






       lowercase        =							self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_qa(*lowerCamelCase_						)







 def   _a   (    self						)    ->    Optional[Any]:
       '''simple docstring'''






       lowercase        =							self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_sequence_classif(*lowerCamelCase_						)







 def   _a   (    self						)    ->    Tuple:
       '''simple docstring'''






       lowercase        =							self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_token_classif(*lowerCamelCase_						)







 def   _a   (    self						)    ->    Dict:
       '''simple docstring'''






       lowercase        =							self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_for_multiple_choice(*lowerCamelCase_						)







 def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase=False ,				_lowerCAmelCase=1						)    ->    Union[str, Any]:
       '''simple docstring'''






       self.assertIsInstance(lowerCamelCase_ ,				lowerCamelCase_						)
       self.assertListEqual(
           [isinstance(lowerCamelCase_ ,				lowerCamelCase_						) for iter_attentions in attentions] ,				[True] * len(lowerCamelCase_						)						)
       self.assertEqual(len(lowerCamelCase_						) ,				(max_length - min_length) * num_beam_groups						)

       for idx, iter_attentions in enumerate(lowerCamelCase_						):
             # adds PAD dummy token
             lowercase        =							min_length + idx + 1
             lowercase        =							min_length + idx + 1

             lowercase        =							(
                 batch_size * num_beam_groups,
                 config.num_attention_heads,
                 tgt_len,
                 src_len,
             )
             # check attn size
             self.assertListEqual(
                 [layer_attention.shape for layer_attention in iter_attentions] ,				[expected_shape] * len(lowerCamelCase_						)						)







 def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase=False ,				_lowerCAmelCase=1						)    ->    Tuple:
       '''simple docstring'''






       self.assertIsInstance(lowerCamelCase_ ,				lowerCamelCase_						)
       self.assertListEqual(
           [isinstance(lowerCamelCase_ ,				lowerCamelCase_						) for iter_hidden_states in hidden_states] ,				[True] * len(lowerCamelCase_						) ,				)
       self.assertEqual(len(lowerCamelCase_						) ,				(max_length - min_length) * num_beam_groups						)

       for idx, iter_hidden_states in enumerate(lowerCamelCase_						):
             # adds PAD dummy token
             lowercase        =							min_length + idx + 1
             lowercase        =							(batch_size * num_beam_groups, seq_len, config.hidden_size)
             # check hidden size
             self.assertListEqual(
                 [layer_hidden_states.shape for layer_hidden_states in iter_hidden_states] ,				[expected_shape] * len(lowerCamelCase_						) ,				)
       pass







 @slow
 def   _a   (    self						)    ->    Union[str, Any]:
       '''simple docstring'''






       for model_name in XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             lowercase        =							XLMModel.from_pretrained(lowerCamelCase_						)
             self.assertIsNotNone(lowerCamelCase_						)




@require_torch
class      __UpperCamelCase       (unittest.TestCase			):







 @slow
 def   _a   (    self						)    ->    Dict:
       '''simple docstring'''






       lowercase        =							XLMWithLMHeadModel.from_pretrained("""xlm-mlm-en-2048"""						)
       model.to(lowerCamelCase_						)
       lowercase        =							torch.tensor([[14, 447]] ,				dtype=torch.long ,				device=lowerCamelCase_						)  # the president
       lowercase        =							[
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
       ]  # the president the president the president the president the president the president the president the president the president the president
       # TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
       lowercase        =							model.generate(lowerCamelCase_ ,				do_sample=lowerCamelCase_						)
       self.assertListEqual(output_ids[0].cpu().numpy().tolist() ,				lowerCamelCase_						)







'''simple docstring'''


import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowercase_      :				Optional[Any]   					=    logging.get_logger(__name__)

lowercase_      :				int   					=    {'''vocab_file''': '''spm_char.model'''}

lowercase_      :				int   					=    {
    '''vocab_file''': {
        '''microsoft/speecht5_asr''': '''https://huggingface.co/microsoft/speecht5_asr/resolve/main/spm_char.model''',
        '''microsoft/speecht5_tts''': '''https://huggingface.co/microsoft/speecht5_tts/resolve/main/spm_char.model''',
        '''microsoft/speecht5_vc''': '''https://huggingface.co/microsoft/speecht5_vc/resolve/main/spm_char.model''',
    }
}

lowercase_      :				Optional[Any]   					=    {
    '''microsoft/speecht5_asr''': 1024,
    '''microsoft/speecht5_tts''': 1024,
    '''microsoft/speecht5_vc''': 1024,
}




class      __UpperCamelCase       (_UpperCAmelCase			):
	__A			  =					VOCAB_FILES_NAMES
	__A			  =					PRETRAINED_VOCAB_FILES_MAP
	__A			  =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	__A			  =					['''input_ids''', '''attention_mask''']







	def __init__(    self ,				_lowerCAmelCase ,				_lowerCAmelCase="<s>" ,				_lowerCAmelCase="</s>" ,				_lowerCAmelCase="<unk>" ,				_lowerCAmelCase="<pad>" ,				_lowerCAmelCase = None ,				**_lowerCAmelCase ,				)    ->    None:
							'''simple docstring'''






							lowercase        =							{} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    bos_token=_lowerCAmelCase ,				eos_token=_lowerCAmelCase ,				unk_token=_lowerCAmelCase ,				pad_token=_lowerCAmelCase ,				sp_model_kwargs=self.sp_model_kwargs ,				**_lowerCAmelCase ,				)

							lowercase        =							vocab_file

							lowercase        =							spm.SentencePieceProcessor(**self.sp_model_kwargs						)
							self.sp_model.Load(_lowerCAmelCase						)







	@property
	def   _a   (    self						)    ->    List[Any]:
							'''simple docstring'''






							return self.sp_model.get_piece_size()







	def   _a   (    self						)    ->    str:
							'''simple docstring'''






							lowercase        =							{self.convert_ids_to_tokens(_lowerCAmelCase						): i for i in range(self.vocab_size						)}
							vocab.update(self.added_tokens_encoder						)
							return vocab







	def __getstate__(    self						)    ->    Union[str, Any]:
							'''simple docstring'''






							lowercase        =							self.__dict__.copy()
							lowercase        =							None
							return state







	def __setstate__(    self ,				_lowerCAmelCase						)    ->    Optional[int]:
							'''simple docstring'''






							lowercase        =							d

							# for backward compatibility
							if not hasattr(self ,				"""sp_model_kwargs"""						):
													lowercase        =							{}

							lowercase        =							spm.SentencePieceProcessor(**self.sp_model_kwargs						)
							self.sp_model.Load(self.vocab_file						)







	def   _a   (    self ,				_lowerCAmelCase						)    ->    List[str]:
							'''simple docstring'''






							return self.sp_model.encode(_lowerCAmelCase ,				out_type=_lowerCAmelCase						)







	def   _a   (    self ,				_lowerCAmelCase						)    ->    List[Any]:
							'''simple docstring'''






							return self.sp_model.piece_to_id(_lowerCAmelCase						)







	def   _a   (    self ,				_lowerCAmelCase						)    ->    str:
							'''simple docstring'''






							lowercase        =							self.sp_model.IdToPiece(_lowerCAmelCase						)
							return token







	def   _a   (    self ,				_lowerCAmelCase						)    ->    Optional[Any]:
							'''simple docstring'''






							lowercase        =							[]
							lowercase        =							""""""
							for token in tokens:
													# make sure that special tokens are not decoded using sentencepiece model
													if token in self.all_special_tokens:
																			out_string += self.sp_model.decode(_lowerCAmelCase						) + token
																			lowercase        =							[]
													else:
																			current_sub_tokens.append(_lowerCAmelCase						)
							out_string += self.sp_model.decode(_lowerCAmelCase						)
							return out_string.strip()







	def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase=None						)    ->    List[int]:
							'''simple docstring'''






							if token_ids_a is None:
													return token_ids_a + [self.eos_token_id]
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return token_ids_a + token_ids_a + [self.eos_token_id]







	def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase = None ,				_lowerCAmelCase = False						)    ->    List[int]:
							'''simple docstring'''






							if already_has_special_tokens:
													return super().get_special_tokens_mask(
													    token_ids_a=_lowerCAmelCase ,				token_ids_a=_lowerCAmelCase ,				already_has_special_tokens=_lowerCAmelCase						)

							lowercase        =							[1]
							if token_ids_a is None:
													return ([0] * len(_lowerCAmelCase						)) + suffix_ones
							return ([0] * len(_lowerCAmelCase						)) + ([0] * len(_lowerCAmelCase						)) + suffix_ones







	def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase = None						)    ->    Tuple[str]:
							'''simple docstring'''






							if not os.path.isdir(_lowerCAmelCase						):
													logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""						)
													return
							lowercase        =							os.path.join(
							    _lowerCAmelCase ,				(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)

							if os.path.abspath(self.vocab_file						) != os.path.abspath(_lowerCAmelCase						) and os.path.isfile(self.vocab_file						):
													copyfile(self.vocab_file ,				_lowerCAmelCase						)
							elif not os.path.isfile(self.vocab_file						):
													with open(_lowerCAmelCase ,				"""wb"""						) as fi:
																			lowercase        =							self.sp_model.serialized_model_proto()
																			fi.write(_lowerCAmelCase						)

							return (out_vocab_file,)


