






from argparse import ArgumentParser, Namespace

from ..utils import logging
from . import BaseTransformersCLICommand

def   _lowerCAmelCase   (						A__	):
      return ConvertCommand(
          args.model_type						,  args.tf_checkpoint						,  args.pytorch_dump_output						,  args.config						,  args.finetuning_task_name	)


a__      :				List[Any]   	=			"\ntransformers can only be used from the commandline to convert TensorFlow models in PyTorch, In that case, it requires\nTensorFlow to be installed. Please see https://www.tensorflow.org/install/ for installation instructions.\n"



class       UpperCAmelCase__(							__lowerCAmelCase	):

   '''simple docstring'''




   @staticmethod
   def 			UpperCAmelCase		(						lowerCAmelCase  :			ArgumentParser)	->				Optional[int]:




         """simple docstring"""

         lowercase__     =  parser.add_parser(
             'convert'			,				help='CLI tool to run convert model from original author checkpoints to Transformers PyTorch checkpoints.'			,				)
         train_parser.add_argument('--model_type'			,				type=lowerCamelCase__			,				required=lowerCamelCase__			,				help='Model\'s type.')
         train_parser.add_argument(
             '--tf_checkpoint'			,				type=lowerCamelCase__			,				required=lowerCamelCase__			,				help='TensorFlow checkpoint path or folder.')
         train_parser.add_argument(
             '--pytorch_dump_output'			,				type=lowerCamelCase__			,				required=lowerCamelCase__			,				help='Path to the PyTorch saved model output.')
         train_parser.add_argument('--config'			,				type=lowerCamelCase__			,				default=''			,				help='Configuration file path or folder.')
         train_parser.add_argument(
             '--finetuning_task_name'			,				type=lowerCamelCase__			,				default=lowerCamelCase__			,				help='Optional fine-tuning task name if the TF model was a finetuned model.'			,				)
         train_parser.set_defaults(func=lowerCamelCase__)


   def __init__(						self  :			Optional[Any]			,				lowerCAmelCase  :			str			,				lowerCAmelCase  :			str			,				lowerCAmelCase  :			str			,				lowerCAmelCase  :			str			,				lowerCAmelCase  :			str			,				*lowerCAmelCase  :			Optional[int]			,				)	->				List[Any]:




         """simple docstring"""

         lowercase__     =  logging.get_logger('transformers-cli/converting')

         self._logger.info(f'''Loading model {model_type}''')
         lowercase__     =  model_type
         lowercase__     =  tf_checkpoint
         lowercase__     =  pytorch_dump_output
         lowercase__     =  config
         lowercase__     =  finetuning_task_name



   def 			UpperCAmelCase		(						self  :			Optional[Any])	->				Optional[Any]:




         """simple docstring"""

         if self._model_type == "albert":
               try:
                     from ..models.albert.convert_albert_original_tf_checkpoint_to_pytorch import (
                         convert_tf_checkpoint_to_pytorch,
                     )
               except ImportError:
                     raise ImportError(lowerCamelCase__)

               convert_tf_checkpoint_to_pytorch(self._tf_checkpoint			,				self._config			,				self._pytorch_dump_output)
         elif self._model_type == "bert":
               try:
                     from ..models.bert.convert_bert_original_tf_checkpoint_to_pytorch import (
                         convert_tf_checkpoint_to_pytorch,
                     )
               except ImportError:
                     raise ImportError(lowerCamelCase__)

               convert_tf_checkpoint_to_pytorch(self._tf_checkpoint			,				self._config			,				self._pytorch_dump_output)
         elif self._model_type == "funnel":
               try:
                     from ..models.funnel.convert_funnel_original_tf_checkpoint_to_pytorch import (
                         convert_tf_checkpoint_to_pytorch,
                     )
               except ImportError:
                     raise ImportError(lowerCamelCase__)

               convert_tf_checkpoint_to_pytorch(self._tf_checkpoint			,				self._config			,				self._pytorch_dump_output)
         elif self._model_type == "t5":
               try:
                     from ..models.ta.convert_ta_original_tf_checkpoint_to_pytorch import convert_tf_checkpoint_to_pytorch
               except ImportError:
                     raise ImportError(lowerCamelCase__)

               convert_tf_checkpoint_to_pytorch(self._tf_checkpoint			,				self._config			,				self._pytorch_dump_output)
         elif self._model_type == "gpt":
               from ..models.openai.convert_openai_original_tf_checkpoint_to_pytorch import (
                   convert_openai_checkpoint_to_pytorch,
               )

               convert_openai_checkpoint_to_pytorch(self._tf_checkpoint			,				self._config			,				self._pytorch_dump_output)
         elif self._model_type == "transfo_xl":
               try:
                     from ..models.transfo_xl.convert_transfo_xl_original_tf_checkpoint_to_pytorch import (
                         convert_transfo_xl_checkpoint_to_pytorch,
                     )
               except ImportError:
                     raise ImportError(lowerCamelCase__)

               if "ckpt" in self._tf_checkpoint.lower():
                     lowercase__     =  self._tf_checkpoint
                     lowercase__     =  ''''''
               else:
                     lowercase__     =  self._tf_checkpoint
                     lowercase__     =  ''''''
               convert_transfo_xl_checkpoint_to_pytorch(
                   lowerCamelCase__			,				self._config			,				self._pytorch_dump_output			,				lowerCamelCase__)
         elif self._model_type == "gpt2":
               try:
                     from ..models.gpta.convert_gpta_original_tf_checkpoint_to_pytorch import (
                         convert_gpta_checkpoint_to_pytorch,
                     )
               except ImportError:
                     raise ImportError(lowerCamelCase__)

               convert_gpta_checkpoint_to_pytorch(self._tf_checkpoint			,				self._config			,				self._pytorch_dump_output)
         elif self._model_type == "xlnet":
               try:
                     from ..models.xlnet.convert_xlnet_original_tf_checkpoint_to_pytorch import (
                         convert_xlnet_checkpoint_to_pytorch,
                     )
               except ImportError:
                     raise ImportError(lowerCamelCase__)

               convert_xlnet_checkpoint_to_pytorch(
                   self._tf_checkpoint			,				self._config			,				self._pytorch_dump_output			,				self._finetuning_task_name)
         elif self._model_type == "xlm":
               from ..models.xlm.convert_xlm_original_pytorch_checkpoint_to_pytorch import (
                   convert_xlm_checkpoint_to_pytorch,
               )

               convert_xlm_checkpoint_to_pytorch(self._tf_checkpoint			,				self._pytorch_dump_output)
         elif self._model_type == "lxmert":
               from ..models.lxmert.convert_lxmert_original_tf_checkpoint_to_pytorch import (
                   convert_lxmert_checkpoint_to_pytorch,
               )

               convert_lxmert_checkpoint_to_pytorch(self._tf_checkpoint			,				self._pytorch_dump_output)
         elif self._model_type == "rembert":
               from ..models.rembert.convert_rembert_tf_checkpoint_to_pytorch import (
                   convert_rembert_tf_checkpoint_to_pytorch,
               )

               convert_rembert_tf_checkpoint_to_pytorch(self._tf_checkpoint			,				self._config			,				self._pytorch_dump_output)
         else:
               raise ValueError(
                   '--model_type should be selected in the list [bert, gpt, gpt2, t5, transfo_xl, xlnet, xlm, lxmert]')





import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bart import BartTokenizer


a__      :				List[Any]   	=			logging.get_logger(__name__)


a__      :				Optional[int]   	=			{"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

# See all BART models at https://huggingface.co/models?filter=bart
a__      :				List[Any]   	=			{
    "vocab_file": {
        "facebook/bart-base": "https://huggingface.co/facebook/bart-base/resolve/main/vocab.json",
        "facebook/bart-large": "https://huggingface.co/facebook/bart-large/resolve/main/vocab.json",
        "facebook/bart-large-mnli": "https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json",
        "facebook/bart-large-cnn": "https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json",
        "facebook/bart-large-xsum": "https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json",
        "yjernite/bart_eli5": "https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json",
    },
    "merges_file": {
        "facebook/bart-base": "https://huggingface.co/facebook/bart-base/resolve/main/merges.txt",
        "facebook/bart-large": "https://huggingface.co/facebook/bart-large/resolve/main/merges.txt",
        "facebook/bart-large-mnli": "https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt",
        "facebook/bart-large-cnn": "https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt",
        "facebook/bart-large-xsum": "https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt",
        "yjernite/bart_eli5": "https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "facebook/bart-base": "https://huggingface.co/facebook/bart-base/resolve/main/tokenizer.json",
        "facebook/bart-large": "https://huggingface.co/facebook/bart-large/resolve/main/tokenizer.json",
        "facebook/bart-large-mnli": "https://huggingface.co/facebook/bart-large-mnli/resolve/main/tokenizer.json",
        "facebook/bart-large-cnn": "https://huggingface.co/facebook/bart-large-cnn/resolve/main/tokenizer.json",
        "facebook/bart-large-xsum": "https://huggingface.co/facebook/bart-large-xsum/resolve/main/tokenizer.json",
        "yjernite/bart_eli5": "https://huggingface.co/yjernite/bart_eli5/resolve/main/tokenizer.json",
    },
}

a__      :				int   	=			{
    "facebook/bart-base": 10_24,
    "facebook/bart-large": 10_24,
    "facebook/bart-large-mnli": 10_24,
    "facebook/bart-large-cnn": 10_24,
    "facebook/bart-large-xsum": 10_24,
    "yjernite/bart_eli5": 10_24,
}



class       UpperCAmelCase__(							lowerCamelCase	):

   '''simple docstring'''




   A    :  Optional[Any]       				=    VOCAB_FILES_NAMES
   A    :  Dict       				=    PRETRAINED_VOCAB_FILES_MAP
   A    :  str       				=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   A    :  int       				=    ["input_ids", "attention_mask"]
   A    :  Any       				=    BartTokenizer


   def __init__(						self  :			List[Any]			,				lowerCAmelCase  :			Any=None			,				lowerCAmelCase  :			Optional[int]=None			,				lowerCAmelCase  :			List[Any]=None			,				lowerCAmelCase  :			str="replace"			,				lowerCAmelCase  :			str="<s>"			,				lowerCAmelCase  :			int="</s>"			,				lowerCAmelCase  :			Optional[int]="</s>"			,				lowerCAmelCase  :			Union[str, Any]="<s>"			,				lowerCAmelCase  :			str="<unk>"			,				lowerCAmelCase  :			int="<pad>"			,				lowerCAmelCase  :			int="<mask>"			,				lowerCAmelCase  :			Dict=False			,				lowerCAmelCase  :			List[Any]=True			,				**lowerCAmelCase  :			Optional[Any]			,				)	->				Optional[Any]:




         """simple docstring"""

         super().__init__(
             lowerCAmelCase			,				lowerCAmelCase			,				tokenizer_file=lowerCAmelCase			,				errors=lowerCAmelCase			,				bos_token=lowerCAmelCase			,				eos_token=lowerCAmelCase			,				sep_token=lowerCAmelCase			,				cls_token=lowerCAmelCase			,				unk_token=lowerCAmelCase			,				pad_token=lowerCAmelCase			,				mask_token=lowerCAmelCase			,				add_prefix_space=lowerCAmelCase			,				trim_offsets=lowerCAmelCase			,				**lowerCAmelCase			,				)

         lowercase__     =  json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
         if pre_tok_state.get('add_prefix_space'			,				lowerCAmelCase) != add_prefix_space:
               lowercase__     =  getattr(lowerCAmelCase			,				pre_tok_state.pop('type'))
               lowercase__     =  add_prefix_space
               lowercase__     =  pre_tok_class(**lowerCAmelCase)

         lowercase__     =  add_prefix_space

         # the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
         lowercase__     =  'post_processor'
         lowercase__     =  getattr(self.backend_tokenizer			,				lowerCAmelCase			,				lowerCAmelCase)
         if tokenizer_component_instance:
               lowercase__     =  json.loads(tokenizer_component_instance.__getstate__())

               # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
               if "sep" in state:
                     lowercase__     =  tuple(state['sep'])
               if "cls" in state:
                     lowercase__     =  tuple(state['cls'])

               lowercase__     =  False

               if state.get('add_prefix_space'			,				lowerCAmelCase) != add_prefix_space:
                     lowercase__     =  add_prefix_space
                     lowercase__     =  True

               if state.get('trim_offsets'			,				lowerCAmelCase) != trim_offsets:
                     lowercase__     =  trim_offsets
                     lowercase__     =  True

               if changes_to_apply:
                     lowercase__     =  getattr(lowerCAmelCase			,				state.pop('type'))
                     lowercase__     =  component_class(**lowerCAmelCase)
                     setattr(self.backend_tokenizer			,				lowerCAmelCase			,				lowerCAmelCase)


   @property
   def 			UpperCAmelCase		(						self  :			Union[str, Any])	->				str:




         """simple docstring"""

         if self._mask_token is None:
               if self.verbose:
                     logger.error('Using mask_token, but it is not set yet.')
               return None
         return str(self._mask_token)


   @mask_token.setter
   def 			UpperCAmelCase		(						self  :			Tuple			,				lowerCAmelCase  :			int)	->				Optional[int]:




         """simple docstring"""

         lowercase__     =  AddedToken(lowerCAmelCase			,				lstrip=lowerCAmelCase			,				rstrip=lowerCAmelCase) if isinstance(lowerCAmelCase			,				lowerCAmelCase) else value
         lowercase__     =  value


   def 			UpperCAmelCase		(						self  :			List[str]			,				*lowerCAmelCase  :			Tuple			,				**lowerCAmelCase  :			Optional[int])	->				BatchEncoding:




         """simple docstring"""

         lowercase__     =  kwargs.get('is_split_into_words'			,				lowerCAmelCase)

         if is_split_into_words and not self.add_prefix_space:
               raise ValueError(
                   f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
                   'to use it with pretokenized inputs.')

         return super()._batch_encode_plus(*lowerCAmelCase			,				**lowerCAmelCase)


   def 			UpperCAmelCase		(						self  :			str			,				*lowerCAmelCase  :			Tuple			,				**lowerCAmelCase  :			str)	->				BatchEncoding:




         """simple docstring"""

         lowercase__     =  kwargs.get('is_split_into_words'			,				lowerCAmelCase)

         if is_split_into_words and not self.add_prefix_space:
               raise ValueError(
                   f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
                   'to use it with pretokenized inputs.')

         return super()._encode_plus(*lowerCAmelCase			,				**lowerCAmelCase)


   def 			UpperCAmelCase		(						self  :			Dict			,				lowerCAmelCase  :			str			,				lowerCAmelCase  :			Optional[str] = None)	->				Tuple[str]:




         """simple docstring"""

         lowercase__     =  self._tokenizer.model.save(lowerCAmelCase			,				name=lowerCAmelCase)
         return tuple(lowerCAmelCase)


   def 			UpperCAmelCase		(						self  :			Any			,				lowerCAmelCase  :			str			,				lowerCAmelCase  :			Optional[int]=None)	->				Tuple:




         """simple docstring"""

         lowercase__     =  [self.bos_token_id] + token_ids_a + [self.eos_token_id]
         if token_ids_a is None:
               return output

         return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]



   def 			UpperCAmelCase		(						self  :			Union[str, Any]			,				lowerCAmelCase  :			List[int]			,				lowerCAmelCase  :			Optional[List[int]] = None)	->				List[int]:




         """simple docstring"""

         lowercase__     =  [self.sep_token_id]
         lowercase__     =  [self.cls_token_id]

         if token_ids_a is None:
               return len(cls + token_ids_a + sep) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]
