



import argparse
import os
import re


a_			=					"""src/transformers"""

# Pattern that looks at the indentation in a line.
a_			=					re.compile(R"""^(\s*)\S""")
# Pattern that matches `"key":" and puts `key` in group 0.
a_			=					re.compile(R"""^\s*\"([^\"]+)\":""")
# Pattern that matches `_import_structure["key"]` and puts `key` in group 0.
a_			=					re.compile(R"""^\s*_import_structure\[\"([^\"]+)\"\]""")
# Pattern that matches `"key",` and puts `key` in group 0.
a_			=					re.compile(R"""^\s*\"([^\"]+)\",\s*$""")
# Pattern that matches any `[stuff]` and puts `stuff` in group 0.
a_			=					re.compile(R"""\[([^\]]+)\]""")





def 		a__			(		_UpperCamelCase			: int							):
   __lowerCamelCase													=     _re_indent.search(_UpperCamelCase							)
   return "" if search is None else search.groups()[0]





def 		a__			(		_UpperCamelCase			: List[str] ,_UpperCamelCase			: Dict="" ,_UpperCamelCase			: Dict=None ,_UpperCamelCase			: Dict=None							):
   __lowerCamelCase													=     0
   __lowerCamelCase													=     code.split('''\n'''							)
   if start_prompt is not None:
      while not lines[index].startswith(_UpperCamelCase							):
         index += 1
      __lowerCamelCase													=     ['''\n'''.join(lines[:index]							)]
   else:
      __lowerCamelCase													=     []

   # We split into blocks until we get to the `end_prompt` (or the end of the block).
   __lowerCamelCase													=     [lines[index]]
   index += 1
   while index < len(_UpperCamelCase							) and (end_prompt is None or not lines[index].startswith(_UpperCamelCase							)):
      if len(lines[index]							) > 0 and get_indent(lines[index]							) == indent_level:
         if len(_UpperCamelCase							) > 0 and get_indent(current_block[-1]							).startswith(indent_level + ''' '''							):
            current_block.append(lines[index]							)
            blocks.append('''\n'''.join(_UpperCamelCase							)							)
            if index < len(_UpperCamelCase							) - 1:
               __lowerCamelCase													=     [lines[index + 1]]
               index += 1
            else:
               __lowerCamelCase													=     []
         else:
            blocks.append('''\n'''.join(_UpperCamelCase							)							)
            __lowerCamelCase													=     [lines[index]]
      else:
         current_block.append(lines[index]							)
      index += 1

   # Adds current block if it's nonempty.
   if len(_UpperCamelCase							) > 0:
      blocks.append('''\n'''.join(_UpperCamelCase							)							)

   # Add final block after end_prompt if provided.
   if end_prompt is not None and index < len(_UpperCamelCase							):
      blocks.append('''\n'''.join(lines[index:]							)							)

   return blocks





def 		a__			(		_UpperCamelCase			: Optional[Any]							):

   def _inner(_UpperCamelCase			: List[Any]							):
      return key(_UpperCamelCase							).lower().replace('''_''' ,''''''							)

   return _inner





def 		a__			(		_UpperCamelCase			: Tuple ,_UpperCamelCase			: Optional[int]=None							):

   # If no key is provided, we use a noop.
   def noop(_UpperCamelCase			: Optional[Any]							):
      return x

   if key is None:
      __lowerCamelCase													=     noop
   # Constants are all uppercase, they go first.
   __lowerCamelCase													=     [obj for obj in objects if key(_UpperCamelCase							).isupper()]
   # Classes are not all uppercase but start with a capital, they go second.
   __lowerCamelCase													=     [obj for obj in objects if key(_UpperCamelCase							)[0].isupper() and not key(_UpperCamelCase							).isupper()]
   # Functions begin with a lowercase, they go last.
   __lowerCamelCase													=     [obj for obj in objects if not key(_UpperCamelCase							)[0].isupper()]

   __lowerCamelCase													=     ignore_underscore(_UpperCamelCase							)
   return sorted(_UpperCamelCase ,key=_UpperCamelCase							) + sorted(_UpperCamelCase ,key=_UpperCamelCase							) + sorted(_UpperCamelCase ,key=_UpperCamelCase							)





def 		a__			(		_UpperCamelCase			: Tuple							):

   # This inner function sort imports between [ ].
   def _replace(_UpperCamelCase			: int							):
      __lowerCamelCase													=     match.groups()[0]
      if "," not in imports:
         return F"""[{imports}]"""
      __lowerCamelCase													=     [part.strip().replace('''"''' ,''''''							) for part in imports.split(''','''							)]
      # We will have a final empty element if the line finished with a comma.
      if len(keys[-1]							) == 0:
         __lowerCamelCase													=     keys[:-1]
      return "[" + ", ".join([F"""\"{k}\"""" for k in sort_objects(_UpperCamelCase							)]							) + "]"

   __lowerCamelCase													=     import_statement.split('''\n'''							)
   if len(_UpperCamelCase							) > 3:
      # Here we have to sort internal imports that are on several lines (one per name):
      # key: [
      #     "object1",
      #     "object2",
      #     ...
      # ]

      # We may have to ignore one or two lines on each side.
      __lowerCamelCase													=     2 if lines[1].strip() == '''[''' else 1
      __lowerCamelCase													=     [(i, _re_strip_line.search(_UpperCamelCase							).groups()[0]) for i, line in enumerate(lines[idx:-idx]							)]
      __lowerCamelCase													=     sort_objects(_UpperCamelCase ,key=lambda _UpperCamelCase							: x[1]							)
      __lowerCamelCase													=     [lines[x[0] + idx] for x in sorted_indices]
      return "\n".join(lines[:idx] + sorted_lines + lines[-idx:]							)
   elif len(_UpperCamelCase							) == 3:
      # Here we have to sort internal imports that are on one separate line:
      # key: [
      #     "object1", "object2", ...
      # ]
      if _re_bracket_content.search(lines[1]							) is not None:
         __lowerCamelCase													=     _re_bracket_content.sub(_replace ,lines[1]							)
      else:
         __lowerCamelCase													=     [part.strip().replace('''"''' ,''''''							) for part in lines[1].split(''','''							)]
         # We will have a final empty element if the line finished with a comma.
         if len(keys[-1]							) == 0:
            __lowerCamelCase													=     keys[:-1]
         __lowerCamelCase													=     get_indent(lines[1]							) + ''', '''.join([F"""\"{k}\"""" for k in sort_objects(_UpperCamelCase							)]							)
      return "\n".join(_UpperCamelCase							)
   else:
      # Finally we have to deal with imports fitting on one line
      __lowerCamelCase													=     _re_bracket_content.sub(_replace ,_UpperCamelCase							)
      return import_statement





def 		a__			(		_UpperCamelCase			: List[Any] ,_UpperCamelCase			: Tuple=True							):
   with open(_UpperCamelCase ,encoding='''utf-8'''							) as f:
      __lowerCamelCase													=     f.read()

   if "_import_structure" not in code:
      return

   # Blocks of indent level 0
   __lowerCamelCase													=     split_code_in_indented_blocks(
       _UpperCamelCase ,start_prompt='''_import_structure = {''' ,end_prompt='''if TYPE_CHECKING:'''							)

   # We ignore block 0 (everything untils start_prompt) and the last block (everything after end_prompt).
   for block_idx in range(1 ,len(_UpperCamelCase							) - 1							):
      # Check if the block contains some `_import_structure`s thingy to sort.
      __lowerCamelCase													=     main_blocks[block_idx]
      __lowerCamelCase													=     block.split('''\n'''							)

      # Get to the start of the imports.
      __lowerCamelCase													=     0
      while line_idx < len(_UpperCamelCase							) and "_import_structure" not in block_lines[line_idx]:
         # Skip dummy import blocks
         if "import dummy" in block_lines[line_idx]:
            __lowerCamelCase													=     len(_UpperCamelCase							)
         else:
            line_idx += 1
      if line_idx >= len(_UpperCamelCase							):
         continue

      # Ignore beginning and last line: they don't contain anything.
      __lowerCamelCase													=     '''\n'''.join(block_lines[line_idx:-1]							)
      __lowerCamelCase													=     get_indent(block_lines[1]							)
      # Slit the internal block into blocks of indent level 1.
      __lowerCamelCase													=     split_code_in_indented_blocks(_UpperCamelCase ,indent_level=_UpperCamelCase							)
      # We have two categories of import key: list or _import_structure[key].append/extend
      __lowerCamelCase													=     _re_direct_key if '''_import_structure = {''' in block_lines[0] else _re_indirect_key
      # Grab the keys, but there is a trap: some lines are empty or just comments.
      __lowerCamelCase													=     [(pattern.search(_UpperCamelCase							).groups()[0] if pattern.search(_UpperCamelCase							) is not None else None) for b in internal_blocks]
      # We only sort the lines with a key.
      __lowerCamelCase													=     [(i, key) for i, key in enumerate(_UpperCamelCase							) if key is not None]
      __lowerCamelCase													=     [x[0] for x in sorted(_UpperCamelCase ,key=lambda _UpperCamelCase							: x[1]							)]

      # We reorder the blocks by leaving empty lines/comments as they were and reorder the rest.
      __lowerCamelCase													=     0
      __lowerCamelCase													=     []
      for i in range(len(_UpperCamelCase							)							):
         if keys[i] is None:
            reorderded_blocks.append(internal_blocks[i]							)
         else:
            __lowerCamelCase													=     sort_objects_in_import(internal_blocks[sorted_indices[count]]							)
            reorderded_blocks.append(_UpperCamelCase							)
            count += 1

        # And we put our main block back together with its first and last line.
      __lowerCamelCase													=     '''\n'''.join(block_lines[:line_idx] + reorderded_blocks + [block_lines[-1]]							)

   if code != "\n".join(_UpperCamelCase							):
      if check_only:
         return True
      else:
         print(F"""Overwriting {file}."""							)
         with open(_UpperCamelCase ,'''w''' ,encoding='''utf-8'''							) as f:
            f.write('''\n'''.join(_UpperCamelCase							)							)





def 		a__			(		_UpperCamelCase			: Union[str, Any]=True							):
   __lowerCamelCase													=     []
   for root, _, files in os.walk(_UpperCamelCase							):
      if "__init__.py" in files:
         __lowerCamelCase													=     sort_imports(os.path.join(_UpperCamelCase ,'''__init__.py'''							) ,check_only=_UpperCamelCase							)
         if result:
            __lowerCamelCase													=     [os.path.join(_UpperCamelCase ,'''__init__.py'''							)]
   if len(_UpperCamelCase							) > 0:
      raise ValueError(F"""Would overwrite {len(_UpperCamelCase							)} files, run `make style`."""							)


if __name__ == "__main__":
  a_			=					argparse.ArgumentParser()
  parser.add_argument("""--check_only""", action="""store_true""", help="""Whether to only check or fix style.""")
  a_			=					parser.parse_args()

  sort_imports_in_all_inits(check_only=args.check_only)











import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


a_			=					logging.get_logger(__name__)
a_			=					{"""vocab_file""": """spiece.model"""}

a_			=					{
    """vocab_file""": {
        """albert-base-v1""": """https://huggingface.co/albert-base-v1/resolve/main/spiece.model""",
        """albert-large-v1""": """https://huggingface.co/albert-large-v1/resolve/main/spiece.model""",
        """albert-xlarge-v1""": """https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model""",
        """albert-xxlarge-v1""": """https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model""",
        """albert-base-v2""": """https://huggingface.co/albert-base-v2/resolve/main/spiece.model""",
        """albert-large-v2""": """https://huggingface.co/albert-large-v2/resolve/main/spiece.model""",
        """albert-xlarge-v2""": """https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model""",
        """albert-xxlarge-v2""": """https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model""",
    }
}

a_			=					{
    """albert-base-v1""": 512,
    """albert-large-v1""": 512,
    """albert-xlarge-v1""": 512,
    """albert-xxlarge-v1""": 512,
    """albert-base-v2""": 512,
    """albert-large-v2""": 512,
    """albert-xlarge-v2""": 512,
    """albert-xxlarge-v2""": 512,
}

a_			=					"""▁"""



class  __lowerCAmelCase     (	lowerCAmelCase__   ):
 lowerCAmelCase__       					=  VOCAB_FILES_NAMES
 lowerCAmelCase__       					=  PRETRAINED_VOCAB_FILES_MAP
 lowerCAmelCase__       					=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES




 def __init__(						self	,				__UpperCAmelCase	,				__UpperCAmelCase=True	,				__UpperCAmelCase=True	,				__UpperCAmelCase=False	,				__UpperCAmelCase="[CLS]"	,				__UpperCAmelCase="[SEP]"	,				__UpperCAmelCase="<unk>"	,				__UpperCAmelCase="[SEP]"	,				__UpperCAmelCase="<pad>"	,				__UpperCAmelCase="[CLS]"	,				__UpperCAmelCase="[MASK]"	,				__UpperCAmelCase = None	,				**__UpperCAmelCase	,				):



    '''simple docstring'''
    # Mask token behave like a normal word, i.e. include the space before it and
    # is included in the raw text, there should be a match in a non-normalized sentence.
    __lowerCamelCase													=    (
        AddedToken(__UpperCAmelCase	,				lstrip=__UpperCAmelCase	,				rstrip=__UpperCAmelCase	,				normalized=__UpperCAmelCase				)
        if isinstance(__UpperCAmelCase	,				__UpperCAmelCase				)
        else mask_token
    )

    __lowerCamelCase													=    {} if sp_model_kwargs is None else sp_model_kwargs

    super().__init__(
        do_lower_case=__UpperCAmelCase	,				remove_space=__UpperCAmelCase	,				keep_accents=__UpperCAmelCase	,				bos_token=__UpperCAmelCase	,				eos_token=__UpperCAmelCase	,				unk_token=__UpperCAmelCase	,				sep_token=__UpperCAmelCase	,				pad_token=__UpperCAmelCase	,				cls_token=__UpperCAmelCase	,				mask_token=__UpperCAmelCase	,				sp_model_kwargs=self.sp_model_kwargs	,				**__UpperCAmelCase	,				)

    __lowerCamelCase													=    do_lower_case
    __lowerCamelCase													=    remove_space
    __lowerCamelCase													=    keep_accents
    __lowerCamelCase													=    vocab_file

    __lowerCamelCase													=    spm.SentencePieceProcessor(**self.sp_model_kwargs				)
    self.sp_model.Load(__UpperCAmelCase				)




 @property
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    return len(self.sp_model				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    {self.convert_ids_to_tokens(__UpperCAmelCase				): i for i in range(self.vocab_size				)}
    vocab.update(self.added_tokens_encoder				)
    return vocab




 def __getstate__(						self				):



    '''simple docstring'''
    __lowerCamelCase													=    self.__dict__.copy()
    __lowerCamelCase													=    None
    return state




 def __setstate__(						self	,				__UpperCAmelCase				):



    '''simple docstring'''
    __lowerCamelCase													=    d

    # for backward compatibility
    if not hasattr(self	,				'''sp_model_kwargs'''				):
       __lowerCamelCase													=    {}

    __lowerCamelCase													=    spm.SentencePieceProcessor(**self.sp_model_kwargs				)
    self.sp_model.Load(self.vocab_file				)




 def        lowerCamelCase      (						self	,				__UpperCAmelCase				):



    '''simple docstring'''
    if self.remove_space:
       __lowerCamelCase													=    ''' '''.join(inputs.strip().split()				)
    else:
       __lowerCamelCase													=    inputs
    __lowerCamelCase													=    outputs.replace('''``'''	,				'''"'''				).replace('''\'\''''	,				'''"'''				)

    if not self.keep_accents:
       __lowerCamelCase													=    unicodedata.normalize('''NFKD'''	,				__UpperCAmelCase				)
       __lowerCamelCase													=    ''''''.join([c for c in outputs if not unicodedata.combining(__UpperCAmelCase				)]				)
    if self.do_lower_case:
       __lowerCamelCase													=    outputs.lower()

    return outputs




 def        lowerCamelCase      (						self	,				__UpperCAmelCase				):



    '''simple docstring'''
    __lowerCamelCase													=    self.preprocess_text(__UpperCAmelCase				)
    __lowerCamelCase													=    self.sp_model.encode(__UpperCAmelCase	,				out_type=__UpperCAmelCase				)
    __lowerCamelCase													=    []
    for piece in pieces:
       if len(__UpperCAmelCase				) > 1 and piece[-1] == str(''','''				) and piece[-2].isdigit():
          __lowerCamelCase													=    self.sp_model.EncodeAsPieces(piece[:-1].replace(__UpperCAmelCase	,				''''''				)				)
          if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
             if len(cur_pieces[0]				) == 1:
                __lowerCamelCase													=    cur_pieces[1:]
             else:
                __lowerCamelCase													=    cur_pieces[0][1:]
          cur_pieces.append(piece[-1]				)
          new_pieces.extend(__UpperCAmelCase				)
       else:
          new_pieces.append(__UpperCAmelCase				)

    return new_pieces




 def        lowerCamelCase      (						self	,				__UpperCAmelCase				):



    '''simple docstring'''
    return self.sp_model.PieceToId(__UpperCAmelCase				)




 def        lowerCamelCase      (						self	,				__UpperCAmelCase				):



    '''simple docstring'''
    return self.sp_model.IdToPiece(__UpperCAmelCase				)




 def        lowerCamelCase      (						self	,				__UpperCAmelCase				):



    '''simple docstring'''
    __lowerCamelCase													=    []
    __lowerCamelCase													=    ''''''
    __lowerCamelCase													=    False
    for token in tokens:
       # make sure that special tokens are not decoded using sentencepiece model
       if token in self.all_special_tokens:
          if not prev_is_special:
             out_string += " "
          out_string += self.sp_model.decode(__UpperCAmelCase				) + token
          __lowerCamelCase													=    True
          __lowerCamelCase													=    []
       else:
          current_sub_tokens.append(__UpperCAmelCase				)
          __lowerCamelCase													=    False
    out_string += self.sp_model.decode(__UpperCAmelCase				)
    return out_string.strip()




 def        lowerCamelCase      (						self	,				__UpperCAmelCase	,				__UpperCAmelCase = None				):



    '''simple docstring'''
    __lowerCamelCase													=    [self.sep_token_id]
    __lowerCamelCase													=    [self.cls_token_id]
    if token_ids_a is None:
       return cls + token_ids_a + sep
    return cls + token_ids_a + sep + token_ids_a + sep




 def        lowerCamelCase      (						self	,				__UpperCAmelCase	,				__UpperCAmelCase = None	,				__UpperCAmelCase = False				):



    '''simple docstring'''

    if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=__UpperCAmelCase	,				token_ids_a=__UpperCAmelCase	,				already_has_special_tokens=__UpperCAmelCase				)

    if token_ids_a is not None:
       return [1] + ([0] * len(__UpperCAmelCase				)) + [1] + ([0] * len(__UpperCAmelCase				)) + [1]
    return [1] + ([0] * len(__UpperCAmelCase				)) + [1]




 def        lowerCamelCase      (						self	,				__UpperCAmelCase	,				__UpperCAmelCase = None				):



    '''simple docstring'''
    __lowerCamelCase													=    [self.sep_token_id]
    __lowerCamelCase													=    [self.cls_token_id]

    if token_ids_a is None:
       return len(cls + token_ids_a + sep				) * [0]
    return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]




 def        lowerCamelCase      (						self	,				__UpperCAmelCase	,				__UpperCAmelCase = None				):



    '''simple docstring'''
    if not os.path.isdir(__UpperCAmelCase				):
       logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""				)
       return
    __lowerCamelCase													=    os.path.join(
        __UpperCAmelCase	,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)

    if os.path.abspath(self.vocab_file				) != os.path.abspath(__UpperCAmelCase				) and os.path.isfile(self.vocab_file				):
       copyfile(self.vocab_file	,				__UpperCAmelCase				)
    elif not os.path.isfile(self.vocab_file				):
       with open(__UpperCAmelCase	,				'''wb'''				) as fi:
          __lowerCamelCase													=    self.sp_model.serialized_model_proto()
          fi.write(__UpperCAmelCase				)

    return (out_vocab_file,)






