



import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


a_						=					logging.get_logger(__name__)

a_						=					{'''vocab_file''': '''spiece.model'''}

a_						=					{
    '''vocab_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model''',
    }
}

a_						=					{
    '''xlnet-base-cased''': None,
    '''xlnet-large-cased''': None,
}

# Segments (not really needed)
a_						=					0
a_						=					1
a_						=					2
a_						=					3
a_						=					4



class  __lowerCAmelCase     (	__a   ):
 lowerCAmelCase__       					=  VOCAB_FILES_NAMES
 lowerCAmelCase__       					=  PRETRAINED_VOCAB_FILES_MAP
 lowerCAmelCase__       					=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 lowerCAmelCase__       					=  '''left'''




 def __init__(						self	,				__UpperCAmelCase	,				__UpperCAmelCase=False	,				__UpperCAmelCase=True	,				__UpperCAmelCase=False	,				__UpperCAmelCase="<s>"	,				__UpperCAmelCase="</s>"	,				__UpperCAmelCase="<unk>"	,				__UpperCAmelCase="<sep>"	,				__UpperCAmelCase="<pad>"	,				__UpperCAmelCase="<cls>"	,				__UpperCAmelCase="<mask>"	,				__UpperCAmelCase=["<eop>", "<eod>"]	,				__UpperCAmelCase = None	,				**__UpperCAmelCase	,				):



    '''simple docstring'''
    # Mask token behave like a normal word, i.e. include the space before it
    __lowerCamelCase													=    AddedToken(snake_case__	,				lstrip=snake_case__	,				rstrip=snake_case__				) if isinstance(snake_case__	,				snake_case__				) else mask_token

    __lowerCamelCase													=    {} if sp_model_kwargs is None else sp_model_kwargs

    super().__init__(
        do_lower_case=snake_case__	,				remove_space=snake_case__	,				keep_accents=snake_case__	,				bos_token=snake_case__	,				eos_token=snake_case__	,				unk_token=snake_case__	,				sep_token=snake_case__	,				pad_token=snake_case__	,				cls_token=snake_case__	,				mask_token=snake_case__	,				additional_special_tokens=snake_case__	,				sp_model_kwargs=self.sp_model_kwargs	,				**snake_case__	,				)

    __lowerCamelCase													=    3

    __lowerCamelCase													=    do_lower_case
    __lowerCamelCase													=    remove_space
    __lowerCamelCase													=    keep_accents
    __lowerCamelCase													=    vocab_file

    __lowerCamelCase													=    spm.SentencePieceProcessor(**self.sp_model_kwargs				)
    self.sp_model.Load(snake_case__				)




 @property
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    return len(self.sp_model				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    {self.convert_ids_to_tokens(snake_case__				): i for i in range(self.vocab_size				)}
    vocab.update(self.added_tokens_encoder				)
    return vocab




 def __getstate__(						self				):



    '''simple docstring'''
    __lowerCamelCase													=    self.__dict__.copy()
    __lowerCamelCase													=    None
    return state




 def __setstate__(						self	,				__UpperCAmelCase				):



    '''simple docstring'''
    __lowerCamelCase													=    d

    # for backward compatibility
    if not hasattr(self	,				'''sp_model_kwargs'''				):
       __lowerCamelCase													=    {}

    __lowerCamelCase													=    spm.SentencePieceProcessor(**self.sp_model_kwargs				)
    self.sp_model.Load(self.vocab_file				)




 def        lowerCamelCase      (						self	,				__UpperCAmelCase				):



    '''simple docstring'''
    if self.remove_space:
       __lowerCamelCase													=    ''' '''.join(inputs.strip().split()				)
    else:
       __lowerCamelCase													=    inputs
    __lowerCamelCase													=    outputs.replace('''``'''	,				'''\"'''				).replace('''\'\''''	,				'''\"'''				)

    if not self.keep_accents:
       __lowerCamelCase													=    unicodedata.normalize('''NFKD'''	,				snake_case__				)
       __lowerCamelCase													=    ''''''.join([c for c in outputs if not unicodedata.combining(snake_case__				)]				)
    if self.do_lower_case:
       __lowerCamelCase													=    outputs.lower()

    return outputs




 def        lowerCamelCase      (						self	,				__UpperCAmelCase				):



    '''simple docstring'''
    __lowerCamelCase													=    self.preprocess_text(snake_case__				)
    __lowerCamelCase													=    self.sp_model.encode(snake_case__	,				out_type=snake_case__				)
    __lowerCamelCase													=    []
    for piece in pieces:
       if len(snake_case__				) > 1 and piece[-1] == str(''','''				) and piece[-2].isdigit():
          __lowerCamelCase													=    self.sp_model.EncodeAsPieces(piece[:-1].replace(snake_case__	,				''''''				)				)
          if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
             if len(cur_pieces[0]				) == 1:
                __lowerCamelCase													=    cur_pieces[1:]
             else:
                __lowerCamelCase													=    cur_pieces[0][1:]
          cur_pieces.append(piece[-1]				)
          new_pieces.extend(snake_case__				)
       else:
          new_pieces.append(snake_case__				)

    return new_pieces




 def        lowerCamelCase      (						self	,				__UpperCAmelCase				):



    '''simple docstring'''
    return self.sp_model.PieceToId(snake_case__				)




 def        lowerCamelCase      (						self	,				__UpperCAmelCase				):



    '''simple docstring'''
    return self.sp_model.IdToPiece(snake_case__				)




 def        lowerCamelCase      (						self	,				__UpperCAmelCase				):



    '''simple docstring'''
    __lowerCamelCase													=    ''''''.join(snake_case__				).replace(snake_case__	,				''' '''				).strip()
    return out_string




 def        lowerCamelCase      (						self	,				__UpperCAmelCase	,				__UpperCAmelCase = False	,				__UpperCAmelCase = None	,				__UpperCAmelCase = True	,				**__UpperCAmelCase	,				):



    '''simple docstring'''
    __lowerCamelCase													=    kwargs.pop('''use_source_tokenizer'''	,				snake_case__				)

    __lowerCamelCase													=    self.convert_ids_to_tokens(snake_case__	,				skip_special_tokens=snake_case__				)

    # To avoid mixing byte-level and unicode for byte-level BPT
    # we need to build string separately for added tokens and byte-level tokens
    # cf. https://github.com/huggingface/transformers/issues/1133
    __lowerCamelCase													=    []
    __lowerCamelCase													=    []
    for token in filtered_tokens:
       if skip_special_tokens and token in self.all_special_ids:
          continue
       if token in self.added_tokens_encoder:
          if current_sub_text:
             sub_texts.append(self.convert_tokens_to_string(snake_case__				)				)
             __lowerCamelCase													=    []
          sub_texts.append(snake_case__				)
       else:
          current_sub_text.append(snake_case__				)
    if current_sub_text:
       sub_texts.append(self.convert_tokens_to_string(snake_case__				)				)

    # Mimic the behavior of the Rust tokenizer:
    # By default, there are no spaces between special tokens
    __lowerCamelCase													=    ''''''.join(snake_case__				)

    __lowerCamelCase													=    (
        clean_up_tokenization_spaces
        if clean_up_tokenization_spaces is not None
        else self.clean_up_tokenization_spaces
    )
    if clean_up_tokenization_spaces:
       __lowerCamelCase													=    self.clean_up_tokenization(snake_case__				)
       return clean_text
    else:
       return text




 def        lowerCamelCase      (						self	,				__UpperCAmelCase	,				__UpperCAmelCase = None				):



    '''simple docstring'''
    __lowerCamelCase													=    [self.sep_token_id]
    __lowerCamelCase													=    [self.cls_token_id]
    if token_ids_a is None:
       return token_ids_a + sep + cls
    return token_ids_a + sep + token_ids_a + sep + cls




 def        lowerCamelCase      (						self	,				__UpperCAmelCase	,				__UpperCAmelCase = None	,				__UpperCAmelCase = False				):



    '''simple docstring'''

    if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=snake_case__	,				token_ids_a=snake_case__	,				already_has_special_tokens=snake_case__				)

    if token_ids_a is not None:
       return ([0] * len(snake_case__				)) + [1] + ([0] * len(snake_case__				)) + [1, 1]
    return ([0] * len(snake_case__				)) + [1, 1]




 def        lowerCamelCase      (						self	,				__UpperCAmelCase	,				__UpperCAmelCase = None				):



    '''simple docstring'''
    __lowerCamelCase													=    [self.sep_token_id]
    __lowerCamelCase													=    [2]

    if token_ids_a is None:
       return len(token_ids_a + sep				) * [0] + cls_segment_id
    return len(token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1] + cls_segment_id




 def        lowerCamelCase      (						self	,				__UpperCAmelCase	,				__UpperCAmelCase = None				):



    '''simple docstring'''
    if not os.path.isdir(snake_case__				):
       logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""				)
       return
    __lowerCamelCase													=    os.path.join(
        snake_case__	,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)

    if os.path.abspath(self.vocab_file				) != os.path.abspath(snake_case__				) and os.path.isfile(self.vocab_file				):
       copyfile(self.vocab_file	,				snake_case__				)
    elif not os.path.isfile(self.vocab_file				):
       with open(snake_case__	,				'''wb'''				) as fi:
          __lowerCamelCase													=    self.sp_model.serialized_model_proto()
          fi.write(snake_case__				)

    return (out_vocab_file,)











import inspect
import re

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_config_docstrings.py
a_			=					"""src/transformers"""


# This is to make sure the transformers module imported is the one in the repo.
a_			=					direct_transformers_import(PATH_TO_TRANSFORMERS)

a_			=					transformers.models.auto.configuration_auto.CONFIG_MAPPING

# Regex pattern used to find the checkpoint mentioned in the docstring of `config_class`.
# For example, `[bert-base-uncased](https://huggingface.co/bert-base-uncased)`
a_			=					re.compile(R"""\[(.+?)\]\((https://huggingface\.co/.+?)\)""")


a_			=					{
    """DecisionTransformerConfig""",
    """EncoderDecoderConfig""",
    """MusicgenConfig""",
    """RagConfig""",
    """SpeechEncoderDecoderConfig""",
    """TimmBackboneConfig""",
    """VisionEncoderDecoderConfig""",
    """VisionTextDualEncoderConfig""",
    """LlamaConfig""",
}





def 		a__			(		_UpperCamelCase			: Union[str, Any]							):
   __lowerCamelCase													=    None

   # source code of `config_class`
   __lowerCamelCase													=    inspect.getsource(_UpperCamelCase							)
   __lowerCamelCase													=    _re_checkpoint.findall(_UpperCamelCase							)

   # Each `checkpoint` is a tuple of a checkpoint name and a checkpoint link.
   # For example, `('bert-base-uncased', 'https://huggingface.co/bert-base-uncased')`
   for ckpt_name, ckpt_link in checkpoints:
      # allow the link to end with `/`
      if ckpt_link.endswith('''/'''							):
         __lowerCamelCase													=    ckpt_link[:-1]

      # verify the checkpoint name corresponds to the checkpoint link
      __lowerCamelCase													=    F"""https://huggingface.co/{ckpt_name}"""
      if ckpt_link == ckpt_link_from_name:
         __lowerCamelCase													=    ckpt_name
         break

   return checkpoint





def 		a__			(		):
   __lowerCamelCase													=    []

   for config_class in list(CONFIG_MAPPING.values()							):
      # Skip deprecated models
      if "models.deprecated" in config_class.__module__:
         continue
      __lowerCamelCase													=    get_checkpoint_from_config_class(_UpperCamelCase							)

      __lowerCamelCase													=    config_class.__name__
      if checkpoint is None and name not in CONFIG_CLASSES_TO_IGNORE_FOR_DOCSTRING_CHECKPOINT_CHECK:
         configs_without_checkpoint.append(_UpperCamelCase							)

   if len(_UpperCamelCase							) > 0:
      __lowerCamelCase													=    '''\n'''.join(sorted(_UpperCamelCase							)							)
      raise ValueError(F"""The following configurations don't contain any valid checkpoint:\n{message}"""							)


if __name__ == "__main__":
  check_config_docstrings_have_checkpoints()






