



import argparse
import json

import numpy
import torch

from transformers.models.xlm.tokenization_xlm import VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()






def  UpperCAmelCase__    (    lowercase__						,       lowercase__   ) ->      int:
      # Load checkpoint
      __lowercase					=				torch.load(__A						,       map_location="""cpu"""   )

      __lowercase					=				chkpt["""model"""]

      # We have the base model one level deeper than the original XLM repository
      __lowercase					=				{}
      for k, v in state_dict.items():
            if "pred_layer" in k:
                  __lowercase					=				v
            else:
                  __lowercase					=				v

      __lowercase					=				chkpt["""params"""]
      __lowercase					=				{n: v for n, v in config.items() if not isinstance(__A						,       (torch.FloatTensor, numpy.ndarray)   )}

      __lowercase					=				chkpt["""dico_word2id"""]
      __lowercase					=				{s + """</w>""" if s.find("""@@"""   ) == -1 and i > 13 else s.replace("""@@"""						,       """"""   ): i for s, i in vocab.items()}

      # Save pytorch-model
      __lowercase					=				pytorch_dump_folder_path + """/""" + WEIGHTS_NAME
      __lowercase					=				pytorch_dump_folder_path + """/""" + CONFIG_NAME
      __lowercase					=				pytorch_dump_folder_path + """/""" + VOCAB_FILES_NAMES["""vocab_file"""]

      print(F"Save PyTorch model to {pytorch_weights_dump_path}"   )
      torch.save(__A						,       __A   )

      print(F"Save configuration file to {pytorch_config_dump_path}"   )
      with open(__A						,       """w"""						,       encoding="""utf-8"""   ) as f:
            f.write(json.dumps(__A						,       indent=2   ) + """\n"""   )

      print(F"Save vocab file to {pytorch_config_dump_path}"   )
      with open(__A						,       """w"""						,       encoding="""utf-8"""   ) as f:
            f.write(json.dumps(__A						,       indent=2   ) + """\n"""   )


if __name__ == "__main__":
   UpperCamelCase__ =						argparse.ArgumentParser()
   # Required parameters
   parser.add_argument(
       "--xlm_checkpoint_path", default=None, type=str, required=True, help="Path the official PyTorch dump."
   )
   parser.add_argument(
       "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
   )
   UpperCamelCase__ =						parser.parse_args()
   convert_xlm_checkpoint_to_pytorch(args.xlm_checkpoint_path, args.pytorch_dump_folder_path)










import os
import unittest

from transformers import LxmertTokenizer, LxmertTokenizerFast
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 			_lowerCAmelCase   ( _UpperCAmelCase    ,					unittest.TestCase			):






			"""simple docstring"""





			lowercase__      :  Tuple      =			LxmertTokenizer
			lowercase__      :  List[str]      =			LxmertTokenizerFast
			lowercase__      :  Optional[Any]      =			True
			lowercase__      :  List[Any]      =			True





			def 		snake_case__			(				self	:       Tuple   )						->					Tuple:
									"""simple docstring"""







									super().setUp()

									__lowercase					=				[
									    """[UNK]""",
									    """[CLS]""",
									    """[SEP]""",
									    """want""",
									    """##want""",
									    """##ed""",
									    """wa""",
									    """un""",
									    """runn""",
									    """##ing""",
									    """,""",
									    """low""",
									    """lowest""",
									]
									__lowercase					=				os.path.join(self.tmpdirname			,     VOCAB_FILES_NAMES["""vocab_file"""]   )
									with open(self.vocab_file			,     """w"""			,     encoding="""utf-8"""   ) as vocab_writer:
															vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]   )   )





			def 		snake_case__			(				self	:       Optional[int]			,     lowercase	:       int   )						->					List[Any]:
									"""simple docstring"""







									__lowercase					=				"""UNwant\u00E9d,running"""
									__lowercase					=				"""unwanted, running"""
									return input_text, output_text





			def 		snake_case__			(				self	:       str   )						->					Any:
									"""simple docstring"""







									__lowercase					=				self.tokenizer_class(self.vocab_file   )

									__lowercase					=				tokenizer.tokenize("""UNwant\u00E9d,running"""   )
									self.assertListEqual(lowercase			,     ["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]   )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase   )			,     [7, 4, 5, 10, 8, 9]   )







			def 		snake_case__			(				self	:       Union[str, Any]   )						->					int:
									"""simple docstring"""







									if not self.test_rust_tokenizer:
															return

									__lowercase					=				self.get_tokenizer()
									__lowercase					=				self.get_rust_tokenizer()

									__lowercase					=				"""I was born in 92000, and this is falsé."""

									__lowercase					=				tokenizer.tokenize(lowercase   )
									__lowercase					=				rust_tokenizer.tokenize(lowercase   )
									self.assertListEqual(lowercase			,     lowercase   )

									__lowercase					=				tokenizer.encode(lowercase			,     add_special_tokens=lowercase   )
									__lowercase					=				rust_tokenizer.encode(lowercase			,     add_special_tokens=lowercase   )
									self.assertListEqual(lowercase			,     lowercase   )

									__lowercase					=				self.get_rust_tokenizer()
									__lowercase					=				tokenizer.encode(lowercase   )
									__lowercase					=				rust_tokenizer.encode(lowercase   )
									self.assertListEqual(lowercase			,     lowercase   )





