


import argparse
import os
import re

import packaging.version


_UpperCAmelCase			:  Optional[int]     ="""examples/"""
_UpperCAmelCase			:  Any     ={
    """examples""": (re.compile(R"""^check_min_version\(\"[^\"]+\"\)\s*$""", re.MULTILINE), """check_min_version(\"VERSION\")\n"""),
    """init""": (re.compile(R"""^__version__\s+=\s+\"([^\"]+)\"\s*$""", re.MULTILINE), """__version__ = \"VERSION\"\n"""),
    """setup""": (re.compile(R"""^(\s*)version\s*=\s*\"[^\"]+\",""", re.MULTILINE), R"""\1version=\"VERSION\","""),
    """doc""": (re.compile(R"""^(\s*)release\s*=\s*\"[^\"]+\"$""", re.MULTILINE), """release = \"VERSION\"\n"""),
}
_UpperCAmelCase			:  str     ={
    """init""": """src/transformers/__init__.py""",
    """setup""": """setup.py""",
}
_UpperCAmelCase			:  Union[str, Any]     ="""README.md"""



def 			lowerCAmelCase (		lowerCAmelCase_ ,   lowerCAmelCase_ ,   lowerCAmelCase_ ):
				with open(lowercase__ ,   '''r''' ,   encoding='''utf-8''' ,   newline='''\n''' ) as f:
								lowerCAmelCase_ :      List[str]    				=				f.read()
				lowerCAmelCase_				,  lowerCAmelCase_ :      str    				=				REPLACE_PATTERNS[pattern]
				lowerCAmelCase_ :      List[Any]    				=				replace.replace('''VERSION''' ,   lowercase__ )
				lowerCAmelCase_ :      Optional[int]    				=				re_pattern.sub(lowercase__ ,   lowercase__ )
				with open(lowercase__ ,   '''w''' ,   encoding='''utf-8''' ,   newline='''\n''' ) as f:
								f.write(lowercase__ )



def 			lowerCAmelCase (		lowerCAmelCase_ ):
				for folder, directories, fnames in os.walk(lowercase__ ):
								# Removing some of the folders with non-actively maintained examples from the walk
								if "research_projects" in directories:
												directories.remove('''research_projects''' )
								if "legacy" in directories:
												directories.remove('''legacy''' )
								for fname in fnames:
												if fname.endswith('''.py''' ):
																update_version_in_file(os.path.join(lowercase__ ,   lowercase__ ) ,   lowercase__ ,   pattern='''examples''' )



def 			lowerCAmelCase (		lowerCAmelCase_ ,   lowerCAmelCase_=False ):
				for pattern, fname in REPLACE_FILES.items():
								update_version_in_file(lowercase__ ,   lowercase__ ,   lowercase__ )
				if not patch:
								update_version_in_examples(lowercase__ )



def 			lowerCAmelCase (		):
				lowerCAmelCase_ :      Optional[int]    				=				'''🤗 Transformers currently provides the following architectures'''
				lowerCAmelCase_ :      int    				=				'''1. Want to contribute a new model?'''
				with open(lowercase__ ,   '''r''' ,   encoding='''utf-8''' ,   newline='''\n''' ) as f:
								lowerCAmelCase_ :      Tuple    				=				f.readlines()

				# Find the start of the list.
				lowerCAmelCase_ :      List[Any]    				=				0
				while not lines[start_index].startswith(_start_prompt ):
								start_index += 1
				start_index += 1

				lowerCAmelCase_ :      Optional[Any]    				=				start_index
				# Update the lines in the model list.
				while not lines[index].startswith(_end_prompt ):
								if lines[index].startswith('''1.''' ):
												lowerCAmelCase_ :      Optional[Any]    				=				lines[index].replace(
												    '''https://huggingface.co/docs/transformers/main/model_doc''' ,   '''https://huggingface.co/docs/transformers/model_doc''' ,   )
								index += 1

				with open(lowercase__ ,   '''w''' ,   encoding='''utf-8''' ,   newline='''\n''' ) as f:
								f.writelines(lowercase__ )



def 			lowerCAmelCase (		):
				with open(REPLACE_FILES['''init'''] ,   '''r''' ) as f:
								lowerCAmelCase_ :      Any    				=				f.read()
				lowerCAmelCase_ :      List[Any]    				=				REPLACE_PATTERNS['''init'''][0].search(lowercase__ ).groups()[0]
				return packaging.version.parse(lowercase__ )



def 			lowerCAmelCase (		lowerCAmelCase_=False ):
				lowerCAmelCase_ :      Optional[int]    				=				get_version()
				if patch and default_version.is_devrelease:
								raise ValueError('''Can\'t create a patch version from the dev branch, checkout a released version!''' )
				if default_version.is_devrelease:
								lowerCAmelCase_ :      int    				=				default_version.base_version
				elif patch:
								lowerCAmelCase_ :      List[str]    				=				f"""{default_version.major}.{default_version.minor}.{default_version.micro + 1}"""
				else:
								lowerCAmelCase_ :      Optional[Any]    				=				f"""{default_version.major}.{default_version.minor + 1}.0"""

				# Now let's ask nicely if that's the right one.
				lowerCAmelCase_ :      Optional[int]    				=				input(f"""Which version are you releasing? [{default_version}]""" )
				if len(lowercase__ ) == 0:
								lowerCAmelCase_ :      List[Any]    				=				default_version

				print(f"""Updating version to {version}.""" )
				global_version_update(lowercase__ ,   patch=lowercase__ )
				if not patch:
								print('''Cleaning main README, don\'t forget to run `make fix-copies`.''' )
								clean_main_ref_in_model_list()



def 			lowerCAmelCase (		):
				lowerCAmelCase_ :      List[str]    				=				get_version()
				lowerCAmelCase_ :      str    				=				f"""{current_version.major}.{current_version.minor + 1}.0.dev0"""
				lowerCAmelCase_ :      Union[str, Any]    				=				current_version.base_version

				# Check with the user we got that right.
				lowerCAmelCase_ :      int    				=				input(f"""Which version are we developing now? [{dev_version}]""" )
				if len(lowercase__ ) == 0:
								lowerCAmelCase_ :      List[str]    				=				dev_version

				print(f"""Updating version to {version}.""" )
				global_version_update(lowercase__ )
				print('''Cleaning main README, don\'t forget to run `make fix-copies`.''' )
				clean_main_ref_in_model_list()


if __name__ == "__main__":
						_UpperCAmelCase			:  Dict     =argparse.ArgumentParser()
						parser.add_argument("""--post_release""", action="""store_true""", help="""Whether this is pre or post release.""")
						parser.add_argument("""--patch""", action="""store_true""", help="""Whether or not this is a patch release.""")
						_UpperCAmelCase			:  int     =parser.parse_args()
						if not args.post_release:
												pre_release_work(patch=args.patch)
						elif args.patch:
												print("""Nothing to do after a patch :-)""")
						else:
												post_release_work()



import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
						from .tokenization_xlnet import XLNetTokenizer
else:
						_UpperCAmelCase			:  Dict     =None


_UpperCAmelCase			:  Tuple     =logging.get_logger(__name__)

_UpperCAmelCase			:  Any     ={"""vocab_file""": """spiece.model""", """tokenizer_file""": """tokenizer.json"""}

_UpperCAmelCase			:  Any     ={
    """vocab_file""": {
        """xlnet-base-cased""": """https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model""",
        """xlnet-large-cased""": """https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model""",
    },
    """tokenizer_file""": {
        """xlnet-base-cased""": """https://huggingface.co/xlnet-base-cased/resolve/main/tokenizer.json""",
        """xlnet-large-cased""": """https://huggingface.co/xlnet-large-cased/resolve/main/tokenizer.json""",
    },
}

_UpperCAmelCase			:  Dict     ={
    """xlnet-base-cased""": None,
    """xlnet-large-cased""": None,
}

_UpperCAmelCase			:  Tuple     ="""▁"""

# Segments (not really needed)
_UpperCAmelCase			:  str     =0
_UpperCAmelCase			:  List[str]     =1
_UpperCAmelCase			:  int     =2
_UpperCAmelCase			:  Any     =3
_UpperCAmelCase			:  List[Any]     =4







class 				snake_case__(							UpperCAmelCase__							):







						'''simple docstring'''




						SCREAMING_SNAKE_CASE__      :  int					=							VOCAB_FILES_NAMES
						SCREAMING_SNAKE_CASE__      :  Dict					=							PRETRAINED_VOCAB_FILES_MAP
						SCREAMING_SNAKE_CASE__      :  Optional[Any]					=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						SCREAMING_SNAKE_CASE__      :  Any					=							"""left"""
						SCREAMING_SNAKE_CASE__      :  List[Any]					=							XLNetTokenizer




						def __init__(				self   ,						__lowercase=None   ,						__lowercase=None   ,						__lowercase=False   ,						__lowercase=True   ,						__lowercase=False   ,						__lowercase="<s>"   ,						__lowercase="</s>"   ,						__lowercase="<unk>"   ,						__lowercase="<sep>"   ,						__lowercase="<pad>"   ,						__lowercase="<cls>"   ,						__lowercase="<mask>"   ,						__lowercase=["<eop>", "<eod>"]   ,						**__lowercase   ,						)					->    List[Any]:
										# Mask token behave like a normal word, i.e. include the space before it
										lowerCAmelCase_ :      Any    				=				AddedToken(__lowercase   ,						lstrip=__lowercase   ,						rstrip=__lowercase							) if isinstance(__lowercase   ,						__lowercase							) else mask_token

										super().__init__(
										    vocab_file=__lowercase   ,						tokenizer_file=__lowercase   ,						do_lower_case=__lowercase   ,						remove_space=__lowercase   ,						keep_accents=__lowercase   ,						bos_token=__lowercase   ,						eos_token=__lowercase   ,						unk_token=__lowercase   ,						sep_token=__lowercase   ,						pad_token=__lowercase   ,						cls_token=__lowercase   ,						mask_token=__lowercase   ,						additional_special_tokens=__lowercase   ,						**__lowercase   ,						)

										lowerCAmelCase_ :      List[Any]    				=				3
										lowerCAmelCase_ :      Dict    				=				do_lower_case
										lowerCAmelCase_ :      Dict    				=				remove_space
										lowerCAmelCase_ :      List[str]    				=				keep_accents
										lowerCAmelCase_ :      List[str]    				=				vocab_file
										lowerCAmelCase_ :      str    				=				False if not self.vocab_file else True




						def 						lowercase_ (				self   ,						__lowercase   ,						__lowercase = None							)					->    List[int]:
										lowerCAmelCase_ :      Tuple    				=				[self.sep_token_id]
										lowerCAmelCase_ :      Any    				=				[self.cls_token_id]
										if token_ids_a is None:
														return token_ids_a + sep + cls
										return token_ids_a + sep + token_ids_a + sep + cls




						def 						lowercase_ (				self   ,						__lowercase   ,						__lowercase = None							)					->    List[int]:
										lowerCAmelCase_ :      Optional[Any]    				=				[self.sep_token_id]
										lowerCAmelCase_ :      List[Any]    				=				[2]

										if token_ids_a is None:
														return len(token_ids_a + sep							) * [0] + cls_segment_id
										return len(token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1] + cls_segment_id




						def 						lowercase_ (				self   ,						__lowercase   ,						__lowercase = None							)					->    Tuple[str]:
										if not self.can_save_slow_tokenizer:
														raise ValueError(
														    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
														    '''tokenizer.'''							)

										if not os.path.isdir(__lowercase							):
														logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""							)
														return
										lowerCAmelCase_ :      str    				=				os.path.join(
										    __lowercase   ,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']							)

										if os.path.abspath(self.vocab_file							) != os.path.abspath(__lowercase							):
														copyfile(self.vocab_file   ,						__lowercase							)

										return (out_vocab_file,)