


import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class    UpperCAmelCase_	(							__lowerCamelCase,     unittest.TestCase):
       '''simple docstring'''
       __UpperCamelCase      :     int							=					CLIPTokenizer
       __UpperCamelCase      :     int							=					CLIPTokenizerFast
       __UpperCamelCase      :     Optional[Any]							=					True
       __UpperCamelCase      :     Tuple							=					{}
       __UpperCamelCase      :     str							=					False






       def  _lowercase   (     self						):




              """simple docstring"""




              super().setUp()

              # fmt: off
              UpperCamelCase    :	Optional[Any]   	=		['''l''', '''o''', '''w''', '''e''', '''r''', '''s''', '''t''', '''i''', '''d''', '''n''', '''lo''', '''l</w>''', '''w</w>''', '''r</w>''', '''t</w>''', '''low</w>''', '''er</w>''', '''lowest</w>''', '''newer</w>''', '''wider''', '''<unk>''', '''<|startoftext|>''', '''<|endoftext|>''']
              # fmt: on
              UpperCamelCase    :	List[Any]   	=		dict(zip(SCREAMING_SNAKE_CASE_ ,					range(len(SCREAMING_SNAKE_CASE_						)						)						)						)
              UpperCamelCase    :	str   	=		['''#version: 0.2''', '''l o''', '''lo w</w>''', '''e r</w>''']
              UpperCamelCase    :	str   	=		{'''unk_token''': '''<unk>'''}

              UpperCamelCase    :	Tuple   	=		os.path.join(self.tmpdirname ,					VOCAB_FILES_NAMES['''vocab_file''']						)
              UpperCamelCase    :	Union[str, Any]   	=		os.path.join(self.tmpdirname ,					VOCAB_FILES_NAMES['''merges_file''']						)
              with open(self.vocab_file ,					'''w''' ,					encoding='''utf-8'''						) as fp:
                     fp.write(json.dumps(SCREAMING_SNAKE_CASE_						) + '''\n'''						)
              with open(self.merges_file ,					'''w''' ,					encoding='''utf-8'''						) as fp:
                     fp.write('''\n'''.join(SCREAMING_SNAKE_CASE_						)						)






       def  _lowercase   (     self ,					**__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              kwargs.update(self.special_tokens_map						)
              return CLIPTokenizer.from_pretrained(self.tmpdirname ,					**SCREAMING_SNAKE_CASE_						)






       def  _lowercase   (     self ,					**__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              kwargs.update(self.special_tokens_map						)
              return CLIPTokenizerFast.from_pretrained(self.tmpdirname ,					**SCREAMING_SNAKE_CASE_						)






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              UpperCamelCase    :	Tuple   	=		'''lower newer'''
              UpperCamelCase    :	Tuple   	=		'''lower newer'''
              return input_text, output_text






       def  _lowercase   (     self						):




              """simple docstring"""




              UpperCamelCase    :	str   	=		CLIPTokenizer(self.vocab_file ,					self.merges_file ,					**self.special_tokens_map						)
              UpperCamelCase    :	int   	=		'''lower newer'''
              UpperCamelCase    :	List[str]   	=		['''lo''', '''w''', '''er</w>''', '''n''', '''e''', '''w''', '''er</w>''']
              UpperCamelCase    :	str   	=		tokenizer.tokenize(SCREAMING_SNAKE_CASE_						)
              self.assertListEqual(SCREAMING_SNAKE_CASE_ ,					SCREAMING_SNAKE_CASE_						)

              UpperCamelCase    :	List[str]   	=		tokens + [tokenizer.unk_token]
              UpperCamelCase    :	Optional[int]   	=		[10, 2, 16, 9, 3, 2, 16, 20]
              self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_						) ,					SCREAMING_SNAKE_CASE_						)






       @require_ftfy
       def  _lowercase   (     self						):




              """simple docstring"""




              for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                     with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""						):
                            UpperCamelCase    :	Tuple   	=		self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_ ,					**SCREAMING_SNAKE_CASE_						)
                            UpperCamelCase    :	Optional[int]   	=		self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_ ,					**SCREAMING_SNAKE_CASE_						)

                            UpperCamelCase    :	Optional[Any]   	=		'''A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'''
                            UpperCamelCase    :	Optional[Any]   	=		tokenizer_s.tokenize(SCREAMING_SNAKE_CASE_						)
                            UpperCamelCase    :	List[str]   	=		tokenizer_r.tokenize(SCREAMING_SNAKE_CASE_						)

                            self.assertListEqual(SCREAMING_SNAKE_CASE_ ,					SCREAMING_SNAKE_CASE_						)

                            # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
                            # with Tilde) encoded in 2 different ways
                            UpperCamelCase    :	Optional[Any]   	=		'''xa\u0303y''' + ''' ''' + '''x\xe3y'''
                            UpperCamelCase    :	List[Any]   	=		tokenizer_s.tokenize(SCREAMING_SNAKE_CASE_						)
                            UpperCamelCase    :	Any   	=		tokenizer_r.tokenize(SCREAMING_SNAKE_CASE_						)

                            self.assertListEqual(SCREAMING_SNAKE_CASE_ ,					SCREAMING_SNAKE_CASE_						)

                            # Test that the tokenization is identical on unicode of space type
                            UpperCamelCase    :	Union[str, Any]   	=		[
                                '''\u0009''',  # (horizontal tab, '\t')
                                '''\u000B''',  # (vertical tab)
                                '''\u000C''',  # (form feed)
                                '''\u0020''',  # (space, ' ')
                                '''\u200E''',  # (left-to-right mark):w
                                '''\u200F''',  # (right-to-left mark)
                            ]
                            for unicode_seq in spaces_unicodes:
                                   UpperCamelCase    :	int   	=		tokenizer_s.tokenize(SCREAMING_SNAKE_CASE_						)
                                   UpperCamelCase    :	Optional[int]   	=		tokenizer_r.tokenize(SCREAMING_SNAKE_CASE_						)

                                   self.assertListEqual(SCREAMING_SNAKE_CASE_ ,					SCREAMING_SNAKE_CASE_						)

                            # Test that the tokenization is identical on unicode of line break type
                            UpperCamelCase    :	List[Any]   	=		[
                                '''\u000A''',  # (line feed, '\n')
                                '''\r\n''',  # (carriage return and line feed, '\r\n')
                                '''\u000D''',  # (carriage return, '\r')
                                '''\r''',  # (carriage return, '\r')
                                '''\u000D''',  # (carriage return, '\r')
                                '''\u2028''',  # (line separator)
                                '''\u2029''',  # (paragraph separator)
                                # "\u0085", # (next line)
                            ]

                            # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
                            # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
                            # space (and thus into an empty list).

                            for unicode_seq in line_break_unicodes:
                                   UpperCamelCase    :	Any   	=		tokenizer_s.tokenize(SCREAMING_SNAKE_CASE_						)
                                   UpperCamelCase    :	List[Any]   	=		tokenizer_r.tokenize(SCREAMING_SNAKE_CASE_						)

                                   self.assertListEqual(SCREAMING_SNAKE_CASE_ ,					SCREAMING_SNAKE_CASE_						)






       def  _lowercase   (     self						):




              """simple docstring"""




              for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                     with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""						):
                            UpperCamelCase    :	List[Any]   	=		'''hello'''  # `hello` is a token in the vocabulary of `pretrained_name`
                            UpperCamelCase    :	Any   	=		f"""{text_of_1_token} {text_of_1_token}"""

                            UpperCamelCase    :	str   	=		self.rust_tokenizer_class.from_pretrained(
                                SCREAMING_SNAKE_CASE_ ,					use_fast=SCREAMING_SNAKE_CASE_ ,					)
                            UpperCamelCase    :	Optional[int]   	=		tokenizer_r(SCREAMING_SNAKE_CASE_ ,					return_offsets_mapping=SCREAMING_SNAKE_CASE_ ,					add_special_tokens=SCREAMING_SNAKE_CASE_						)
                            self.assertEqual(encoding.offset_mapping[0] ,					(0, len(SCREAMING_SNAKE_CASE_						))						)
                            self.assertEqual(
                                encoding.offset_mapping[1] ,					(len(SCREAMING_SNAKE_CASE_						) + 1, len(SCREAMING_SNAKE_CASE_						) + 1 + len(SCREAMING_SNAKE_CASE_						)) ,					)

                            UpperCamelCase    :	Tuple   	=		f""" {text}"""

                            UpperCamelCase    :	Any   	=		self.rust_tokenizer_class.from_pretrained(
                                SCREAMING_SNAKE_CASE_ ,					use_fast=SCREAMING_SNAKE_CASE_ ,					)
                            UpperCamelCase    :	Optional[int]   	=		tokenizer_r(SCREAMING_SNAKE_CASE_ ,					return_offsets_mapping=SCREAMING_SNAKE_CASE_ ,					add_special_tokens=SCREAMING_SNAKE_CASE_						)
                            self.assertEqual(encoding.offset_mapping[0] ,					(1, 1 + len(SCREAMING_SNAKE_CASE_						))						)
                            self.assertEqual(
                                encoding.offset_mapping[1] ,					(1 + len(SCREAMING_SNAKE_CASE_						) + 1, 1 + len(SCREAMING_SNAKE_CASE_						) + 1 + len(SCREAMING_SNAKE_CASE_						)) ,					)






       def  _lowercase   (     self						):




              """simple docstring"""




              with self.assertRaises(SCREAMING_SNAKE_CASE_						) as context:
                     self.rust_tokenizer_class.from_pretrained('''robot-test/old-clip-tokenizer'''						)

              self.assertTrue(
                  context.exception.args[0].startswith(
                      '''The `backend_tokenizer` provided does not match the expected format.'''						)						)






       @require_ftfy
       def  _lowercase   (     self						):




              """simple docstring"""




              super().test_tokenization_python_rust_equals()






       def  _lowercase   (     self						):




              """simple docstring"""




              pass









import json
import os
import shutil
import warnings
from argparse import ArgumentParser, Namespace
from pathlib import Path
from typing import List

from ..utils import logging
from . import BaseTransformersCLICommand


try:
  from cookiecutter.main import cookiecutter

  __UpperCAmelCase : List[Any] 			=				True
except ImportError:
  __UpperCAmelCase : List[str] 			=				False

__UpperCAmelCase : Any 			=				logging.get_logger(__name__)  # pylint: disable=invalid-name


def      a	(							SCREAMING_SNAKE_CASE_      :		Namespace       ):






       """simple docstring"""





       return AddNewModelCommand(args.testing							,			args.testing_file							,			path=args.path       )



class    UpperCAmelCase_	(							_a):
       '''simple docstring'''






       @staticmethod
       def  _lowercase   (     __SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              UpperCamelCase    :	List[Any]   	=		parser.add_parser('''add-new-model'''						)
              add_new_model_parser.add_argument('''--testing''' ,					action='''store_true''' ,					help='''If in testing mode.'''						)
              add_new_model_parser.add_argument('''--testing_file''' ,					type=__SCREAMING_SNAKE_CASE ,					help='''Configuration file on which to run.'''						)
              add_new_model_parser.add_argument(
                  '''--path''' ,					type=__SCREAMING_SNAKE_CASE ,					help='''Path to cookiecutter. Should only be used for testing purposes.'''						)
              add_new_model_parser.set_defaults(func=__SCREAMING_SNAKE_CASE						)






       def __init__(     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE=None ,					*__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              UpperCamelCase    :	Tuple   	=		testing
              UpperCamelCase    :	Any   	=		testing_file
              UpperCamelCase    :	Dict   	=		path






       def  _lowercase   (     self						):




              """simple docstring"""




              warnings.warn(
                  '''The command `transformers-cli add-new-model` is deprecated and will be removed in v5 of Transformers. '''
                  '''It is not actively maintained anymore, so might give a result that won\'t pass all tests and quality '''
                  '''checks, you should use `transformers-cli add-new-model-like` instead.'''						)
              if not _has_cookiecutter:
                     raise ImportError(
                         '''Model creation dependencies are required to use the `add_new_model` command. Install them by running '''
                         '''the following at the root of your `transformers` clone:\n\n\t$ pip install -e .[modelcreation]\n'''						)
              # Ensure that there is no other `cookiecutter-template-xxx` directory in the current working directory
              UpperCamelCase    :	List[str]   	=		[directory for directory in os.listdir() if '''cookiecutter-template-''' == directory[:22]]
              if len(__SCREAMING_SNAKE_CASE						) > 0:
                     raise ValueError(
                         '''Several directories starting with `cookiecutter-template-` in current working directory. '''
                         '''Please clean your directory by removing all folders starting with `cookiecutter-template-` or '''
                         '''change your working directory.'''						)

              UpperCamelCase    :	Dict   	=		(
                  Path(__SCREAMING_SNAKE_CASE						).parent.parent.parent.parent if self._path is None else Path(self._path						).parent.parent
              )
              UpperCamelCase    :	List[Any]   	=		path_to_transformer_root / '''templates''' / '''adding_a_new_model'''

              # Execute cookiecutter
              if not self._testing:
                     cookiecutter(str(__SCREAMING_SNAKE_CASE						)						)
              else:
                     with open(self._testing_file ,					'''r'''						) as configuration_file:
                            UpperCamelCase    :	Tuple   	=		json.load(__SCREAMING_SNAKE_CASE						)

                     cookiecutter(
                         str(path_to_cookiecutter if self._path is None else self._path						) ,					no_input=__SCREAMING_SNAKE_CASE ,					extra_context=__SCREAMING_SNAKE_CASE ,					)

              UpperCamelCase    :	Dict   	=		[directory for directory in os.listdir() if '''cookiecutter-template-''' in directory[:22]][0]

              # Retrieve configuration
              with open(directory + '''/configuration.json''' ,					'''r'''						) as configuration_file:
                     UpperCamelCase    :	Tuple   	=		json.load(__SCREAMING_SNAKE_CASE						)

              UpperCamelCase    :	Dict   	=		configuration['''lowercase_modelname''']
              UpperCamelCase    :	int   	=		configuration['''generate_tensorflow_pytorch_and_flax''']
              os.remove(f"""{directory}/configuration.json"""						)

              UpperCamelCase    :	str   	=		'''PyTorch''' in generate_tensorflow_pytorch_and_flax
              UpperCamelCase    :	Any   	=		'''TensorFlow''' in generate_tensorflow_pytorch_and_flax
              UpperCamelCase    :	Union[str, Any]   	=		'''Flax''' in generate_tensorflow_pytorch_and_flax

              UpperCamelCase    :	Optional[Any]   	=		f"""{path_to_transformer_root}/src/transformers/models/{lowercase_model_name}"""
              os.makedirs(__SCREAMING_SNAKE_CASE ,					exist_ok=__SCREAMING_SNAKE_CASE						)
              os.makedirs(f"""{path_to_transformer_root}/tests/models/{lowercase_model_name}""" ,					exist_ok=__SCREAMING_SNAKE_CASE						)

              # Tests require submodules as they have parent imports
              with open(f"""{path_to_transformer_root}/tests/models/{lowercase_model_name}/__init__.py""" ,					'''w'''						):
                     pass

              shutil.move(
                  f"""{directory}/__init__.py""" ,					f"""{model_dir}/__init__.py""" ,					)
              shutil.move(
                  f"""{directory}/configuration_{lowercase_model_name}.py""" ,					f"""{model_dir}/configuration_{lowercase_model_name}.py""" ,					)

              def remove_copy_lines(__SCREAMING_SNAKE_CASE						):
                     with open(__SCREAMING_SNAKE_CASE ,					'''r'''						) as f:
                            UpperCamelCase    :	Any   	=		f.readlines()
                     with open(__SCREAMING_SNAKE_CASE ,					'''w'''						) as f:
                            for line in lines:
                                   if "# Copied from transformers." not in line:
                                          f.write(__SCREAMING_SNAKE_CASE						)

              if output_pytorch:
                     if not self._testing:
                            remove_copy_lines(f"""{directory}/modeling_{lowercase_model_name}.py"""						)

                     shutil.move(
                         f"""{directory}/modeling_{lowercase_model_name}.py""" ,					f"""{model_dir}/modeling_{lowercase_model_name}.py""" ,					)

                     shutil.move(
                         f"""{directory}/test_modeling_{lowercase_model_name}.py""" ,					f"""{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_{lowercase_model_name}.py""" ,					)
              else:
                     os.remove(f"""{directory}/modeling_{lowercase_model_name}.py"""						)
                     os.remove(f"""{directory}/test_modeling_{lowercase_model_name}.py"""						)

              if output_tensorflow:
                     if not self._testing:
                            remove_copy_lines(f"""{directory}/modeling_tf_{lowercase_model_name}.py"""						)

                     shutil.move(
                         f"""{directory}/modeling_tf_{lowercase_model_name}.py""" ,					f"""{model_dir}/modeling_tf_{lowercase_model_name}.py""" ,					)

                     shutil.move(
                         f"""{directory}/test_modeling_tf_{lowercase_model_name}.py""" ,					f"""{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_tf_{lowercase_model_name}.py""" ,					)
              else:
                     os.remove(f"""{directory}/modeling_tf_{lowercase_model_name}.py"""						)
                     os.remove(f"""{directory}/test_modeling_tf_{lowercase_model_name}.py"""						)

              if output_flax:
                     if not self._testing:
                            remove_copy_lines(f"""{directory}/modeling_flax_{lowercase_model_name}.py"""						)

                     shutil.move(
                         f"""{directory}/modeling_flax_{lowercase_model_name}.py""" ,					f"""{model_dir}/modeling_flax_{lowercase_model_name}.py""" ,					)

                     shutil.move(
                         f"""{directory}/test_modeling_flax_{lowercase_model_name}.py""" ,					f"""{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_flax_{lowercase_model_name}.py""" ,					)
              else:
                     os.remove(f"""{directory}/modeling_flax_{lowercase_model_name}.py"""						)
                     os.remove(f"""{directory}/test_modeling_flax_{lowercase_model_name}.py"""						)

              shutil.move(
                  f"""{directory}/{lowercase_model_name}.md""" ,					f"""{path_to_transformer_root}/docs/source/en/model_doc/{lowercase_model_name}.md""" ,					)

              shutil.move(
                  f"""{directory}/tokenization_{lowercase_model_name}.py""" ,					f"""{model_dir}/tokenization_{lowercase_model_name}.py""" ,					)

              shutil.move(
                  f"""{directory}/tokenization_fast_{lowercase_model_name}.py""" ,					f"""{model_dir}/tokenization_{lowercase_model_name}_fast.py""" ,					)

              from os import fdopen, remove
              from shutil import copymode, move
              from tempfile import mkstemp

              def replace(__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						):
                     # Create temp file
                     UpperCamelCase	, UpperCamelCase    :	Optional[Any]   	=		mkstemp()
                     UpperCamelCase    :	Tuple   	=		False
                     with fdopen(__SCREAMING_SNAKE_CASE ,					'''w'''						) as new_file:
                            with open(__SCREAMING_SNAKE_CASE						) as old_file:
                                   for line in old_file:
                                          new_file.write(__SCREAMING_SNAKE_CASE						)
                                          if line_to_copy_below in line:
                                                 UpperCamelCase    :	Optional[int]   	=		True
                                                 for line_to_copy in lines_to_copy:
                                                        new_file.write(__SCREAMING_SNAKE_CASE						)

                     if not line_found:
                            raise ValueError(f"""Line {line_to_copy_below} was not found in file."""						)

                     # Copy the file permissions from the old file to the new file
                     copymode(__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						)
                     # Remove original file
                     remove(__SCREAMING_SNAKE_CASE						)
                     # Move new file
                     move(__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						)

              def skip_units(__SCREAMING_SNAKE_CASE						):
                     return (
                         ("generating PyTorch" in line and not output_pytorch)
                         or ("generating TensorFlow" in line and not output_tensorflow)
                         or ("generating Flax" in line and not output_flax)
                     )

              def replace_in_files(__SCREAMING_SNAKE_CASE						):
                     with open(__SCREAMING_SNAKE_CASE						) as datafile:
                            UpperCamelCase    :	int   	=		[]
                            UpperCamelCase    :	Dict   	=		False
                            UpperCamelCase    :	List[Any]   	=		False
                            for line in datafile:
                                   if "# To replace in: " in line and "##" not in line:
                                          UpperCamelCase    :	Dict   	=		line.split('''"'''						)[1]
                                          UpperCamelCase    :	int   	=		skip_units(__SCREAMING_SNAKE_CASE						)
                                   elif "# Below: " in line and "##" not in line:
                                          UpperCamelCase    :	Dict   	=		line.split('''"'''						)[1]
                                          UpperCamelCase    :	List[str]   	=		skip_units(__SCREAMING_SNAKE_CASE						)
                                   elif "# End." in line and "##" not in line:
                                          if not skip_file and not skip_snippet:
                                                 replace(__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						)

                                          UpperCamelCase    :	Dict   	=		[]
                                   elif "# Replace with" in line and "##" not in line:
                                          UpperCamelCase    :	Tuple   	=		[]
                                   elif "##" not in line:
                                          lines_to_copy.append(__SCREAMING_SNAKE_CASE						)

                     remove(__SCREAMING_SNAKE_CASE						)

              replace_in_files(f"""{directory}/to_replace_{lowercase_model_name}.py"""						)
              os.rmdir(__SCREAMING_SNAKE_CASE						)

