






import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


snake_case__      :	int       	=      logging.get_logger(__name__)
snake_case__      :	Any       	=      {"vocab_file": "spiece.model"}

snake_case__      :	List[Any]       	=      {
    "vocab_file": {
        "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/spiece.model",
        "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/spiece.model",
        "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model",
        "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model",
        "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/spiece.model",
        "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/spiece.model",
        "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model",
        "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model",
    }
}

snake_case__      :	Optional[Any]       	=      {
    "albert-base-v1": 5_1_2,
    "albert-large-v1": 5_1_2,
    "albert-xlarge-v1": 5_1_2,
    "albert-xxlarge-v1": 5_1_2,
    "albert-base-v2": 5_1_2,
    "albert-large-v2": 5_1_2,
    "albert-xlarge-v2": 5_1_2,
    "albert-xxlarge-v2": 5_1_2,
}

snake_case__      :	Union[str, Any]       	=      "▁"


class 						_a       (  __lowerCAmelCase    ):






		"""simple docstring"""





		A_						=   VOCAB_FILES_NAMES
		A_						=   PRETRAINED_VOCAB_FILES_MAP
		A_						=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES




		def __init__(     self		,		_UpperCAmelCase		,		_UpperCAmelCase=True		,		_UpperCAmelCase=True		,		_UpperCAmelCase=False		,		_UpperCAmelCase="[CLS]"		,		_UpperCAmelCase="[SEP]"		,		_UpperCAmelCase="<unk>"		,		_UpperCAmelCase="[SEP]"		,		_UpperCAmelCase="<pad>"		,		_UpperCAmelCase="[CLS]"		,		_UpperCAmelCase="[MASK]"		,		_UpperCAmelCase = None		,		**_UpperCAmelCase		,		)			-> None:
					UpperCamelCase_									=     (
					    AddedToken(lowerCamelCase__		,		lstrip=lowerCamelCase__		,		rstrip=lowerCamelCase__		,		normalized=lowerCamelCase__ )
					    if isinstance(lowerCamelCase__		,		lowerCamelCase__ )
					    else mask_token
					)

					UpperCamelCase_									=     {} if sp_model_kwargs is None else sp_model_kwargs

					super().__init__(
					    do_lower_case=lowerCamelCase__		,		remove_space=lowerCamelCase__		,		keep_accents=lowerCamelCase__		,		bos_token=lowerCamelCase__		,		eos_token=lowerCamelCase__		,		unk_token=lowerCamelCase__		,		sep_token=lowerCamelCase__		,		pad_token=lowerCamelCase__		,		cls_token=lowerCamelCase__		,		mask_token=lowerCamelCase__		,		sp_model_kwargs=self.sp_model_kwargs		,		**lowerCamelCase__		,		)

					UpperCamelCase_									=     do_lower_case
					UpperCamelCase_									=     remove_space
					UpperCamelCase_									=     keep_accents
					UpperCamelCase_									=     vocab_file

					UpperCamelCase_									=     spm.SentencePieceProcessor(**self.sp_model_kwargs )
					self.sp_model.Load(lowerCamelCase__ )




		@property
		def    _UpperCAmelCase     (     self )			-> int:
					return len(self.sp_model )




		def    _UpperCAmelCase     (     self )			-> List[Any]:
					UpperCamelCase_									=     {self.convert_ids_to_tokens(lowerCamelCase__ ): i for i in range(self.vocab_size )}
					vocab.update(self.added_tokens_encoder )
					return vocab




		def __getstate__(     self )			-> Any:
					UpperCamelCase_									=     self.__dict__.copy()
					UpperCamelCase_									=     None
					return state




		def __setstate__(     self		,		_UpperCAmelCase )			-> str:
					UpperCamelCase_									=     d

					# for backward compatibility
					if not hasattr(self		,		'sp_model_kwargs' ):
								UpperCamelCase_									=     {}

					UpperCamelCase_									=     spm.SentencePieceProcessor(**self.sp_model_kwargs )
					self.sp_model.Load(self.vocab_file )




		def    _UpperCAmelCase     (     self		,		_UpperCAmelCase )			-> Dict:






					if self.remove_space:
								UpperCamelCase_									=     ''' '''.join(inputs.strip().split() )
					else:
								UpperCamelCase_									=     inputs
					UpperCamelCase_									=     outputs.replace('``'		,		'"' ).replace('\'\''		,		'"' )

					if not self.keep_accents:
								UpperCamelCase_									=     unicodedata.normalize('NFKD'		,		lowerCamelCase__ )
								UpperCamelCase_									=     ''''''.join([c for c in outputs if not unicodedata.combining(lowerCamelCase__ )] )
					if self.do_lower_case:
								UpperCamelCase_									=     outputs.lower()

					return outputs




		def    _UpperCAmelCase     (     self		,		_UpperCAmelCase )			-> List[str]:
					UpperCamelCase_									=     self.preprocess_text(lowerCamelCase__ )
					UpperCamelCase_									=     self.sp_model.encode(lowerCamelCase__		,		out_type=lowerCamelCase__ )
					UpperCamelCase_									=     []
					for piece in pieces:
								if len(lowerCamelCase__ ) > 1 and piece[-1] == str(',' ) and piece[-2].isdigit():
											UpperCamelCase_									=     self.sp_model.EncodeAsPieces(piece[:-1].replace(lowerCamelCase__		,		'' ) )
											if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
														if len(cur_pieces[0] ) == 1:
																	UpperCamelCase_									=     cur_pieces[1:]
														else:
																	UpperCamelCase_									=     cur_pieces[0][1:]
											cur_pieces.append(piece[-1] )
											new_pieces.extend(lowerCamelCase__ )
								else:
											new_pieces.append(lowerCamelCase__ )

					return new_pieces




		def    _UpperCAmelCase     (     self		,		_UpperCAmelCase )			-> int:
					return self.sp_model.PieceToId(lowerCamelCase__ )




		def    _UpperCAmelCase     (     self		,		_UpperCAmelCase )			-> List[str]:
					return self.sp_model.IdToPiece(lowerCamelCase__ )




		def    _UpperCAmelCase     (     self		,		_UpperCAmelCase )			-> int:
					UpperCamelCase_									=     []
					UpperCamelCase_									=     ''''''
					UpperCamelCase_									=     False
					for token in tokens:
								# make sure that special tokens are not decoded using sentencepiece model
								if token in self.all_special_tokens:
											if not prev_is_special:
														out_string += " "
											out_string += self.sp_model.decode(lowerCamelCase__ ) + token
											UpperCamelCase_									=     True
											UpperCamelCase_									=     []
								else:
											current_sub_tokens.append(lowerCamelCase__ )
											UpperCamelCase_									=     False
					out_string += self.sp_model.decode(lowerCamelCase__ )
					return out_string.strip()




		def    _UpperCAmelCase     (     self		,		_UpperCAmelCase		,		_UpperCAmelCase = None )			-> List[int]:
					UpperCamelCase_									=     [self.sep_token_id]
					UpperCamelCase_									=     [self.cls_token_id]
					if token_ids_a is None:
								return cls + token_ids_a + sep
					return cls + token_ids_a + sep + token_ids_a + sep




		def    _UpperCAmelCase     (     self		,		_UpperCAmelCase		,		_UpperCAmelCase = None		,		_UpperCAmelCase = False )			-> List[int]:






					if already_has_special_tokens:
								return super().get_special_tokens_mask(
								    token_ids_a=lowerCamelCase__		,		token_ids_a=lowerCamelCase__		,		already_has_special_tokens=lowerCamelCase__ )

					if token_ids_a is not None:
								return [1] + ([0] * len(lowerCamelCase__ )) + [1] + ([0] * len(lowerCamelCase__ )) + [1]
					return [1] + ([0] * len(lowerCamelCase__ )) + [1]




		def    _UpperCAmelCase     (     self		,		_UpperCAmelCase		,		_UpperCAmelCase = None )			-> List[int]:
					UpperCamelCase_									=     [self.sep_token_id]
					UpperCamelCase_									=     [self.cls_token_id]

					if token_ids_a is None:
								return len(cls + token_ids_a + sep ) * [0]
					return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]




		def    _UpperCAmelCase     (     self		,		_UpperCAmelCase		,		_UpperCAmelCase = None )			-> Tuple[str]:






					if not os.path.isdir(lowerCamelCase__ ):
								logger.error(f"""Vocabulary path ({save_directory}) should be a directory""" )
								return
					UpperCamelCase_									=     os.path.join(
					    lowerCamelCase__		,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'] )

					if os.path.abspath(self.vocab_file ) != os.path.abspath(lowerCamelCase__ ) and os.path.isfile(self.vocab_file ):
								copyfile(self.vocab_file		,		lowerCamelCase__ )
					elif not os.path.isfile(self.vocab_file ):
								with open(lowerCamelCase__		,		'wb' ) as fi:
											UpperCamelCase_									=     self.sp_model.serialized_model_proto()
											fi.write(lowerCamelCase__ )

					return (out_vocab_file,)













import copy
import os
from typing import Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


snake_case__      :	Optional[int]       	=      logging.get_logger(__name__)

snake_case__      :	List[Any]       	=      {
    """microsoft/git-base""": """https://huggingface.co/microsoft/git-base/resolve/main/config.json""",
}


class 						_a       (  UpperCAmelCase__    ):






  """simple docstring"""





  A_						=   """git_vision_model"""




  def __init__(     self		,		_UpperCAmelCase=768		,		_UpperCAmelCase=3072		,		_UpperCAmelCase=12		,		_UpperCAmelCase=12		,		_UpperCAmelCase=3		,		_UpperCAmelCase=224		,		_UpperCAmelCase=16		,		_UpperCAmelCase="quick_gelu"		,		_UpperCAmelCase=1e-5		,		_UpperCAmelCase=0.0		,		_UpperCAmelCase=0.0_2		,		**_UpperCAmelCase		,		)			-> Optional[Any]:
     super().__init__(**_UpperCAmelCase )

     UpperCamelCase_									=     hidden_size
     UpperCamelCase_									=     intermediate_size
     UpperCamelCase_									=     num_hidden_layers
     UpperCamelCase_									=     num_attention_heads
     UpperCamelCase_									=     num_channels
     UpperCamelCase_									=     patch_size
     UpperCamelCase_									=     image_size
     UpperCamelCase_									=     initializer_range
     UpperCamelCase_									=     attention_dropout
     UpperCamelCase_									=     layer_norm_eps
     UpperCamelCase_									=     hidden_act




  @classmethod
  def    _UpperCAmelCase     (     cls		,		_UpperCAmelCase		,		**_UpperCAmelCase )			-> "PretrainedConfig":
     cls._set_token_in_kwargs(_UpperCAmelCase )

     UpperCamelCase_		,      UpperCamelCase_									=     cls.get_config_dict(_UpperCAmelCase		,		**_UpperCAmelCase )

     # get the vision config dict if we are loading from GITConfig
     if config_dict.get('model_type' ) == "git":
        UpperCamelCase_									=     config_dict['vision_config']

     if "model_type" in config_dict and hasattr(cls		,		'model_type' ) and config_dict["model_type"] != cls.model_type:
        logger.warning(
            f"""You are using a model of type {config_dict["model_type"]} to instantiate a model of type """
            f"""{cls.model_type}. This is not supported for all configurations of models and can yield errors.""" )

     return cls.from_dict(_UpperCAmelCase		,		**_UpperCAmelCase )




class 						_a       (  UpperCAmelCase__    ):






  """simple docstring"""





  A_						=   """git"""




  def __init__(     self		,		_UpperCAmelCase=None		,		_UpperCAmelCase=30522		,		_UpperCAmelCase=768		,		_UpperCAmelCase=6		,		_UpperCAmelCase=12		,		_UpperCAmelCase=3072		,		_UpperCAmelCase="gelu"		,		_UpperCAmelCase=0.1		,		_UpperCAmelCase=0.1		,		_UpperCAmelCase=1024		,		_UpperCAmelCase=0.0_2		,		_UpperCAmelCase=1e-12		,		_UpperCAmelCase=0		,		_UpperCAmelCase="absolute"		,		_UpperCAmelCase=True		,		_UpperCAmelCase=False		,		_UpperCAmelCase=101		,		_UpperCAmelCase=102		,		_UpperCAmelCase=None		,		**_UpperCAmelCase		,		)			-> Optional[int]:
     super().__init__(bos_token_id=_UpperCAmelCase		,		eos_token_id=_UpperCAmelCase		,		pad_token_id=_UpperCAmelCase		,		**_UpperCAmelCase )

     if vision_config is None:
        UpperCamelCase_									=     {}
        logger.info('vision_config is None. initializing the GitVisionConfig with default values.' )

     UpperCamelCase_									=     GitVisionConfig(**_UpperCAmelCase )
     UpperCamelCase_									=     vocab_size
     UpperCamelCase_									=     hidden_size
     UpperCamelCase_									=     num_hidden_layers
     UpperCamelCase_									=     num_attention_heads
     UpperCamelCase_									=     hidden_act
     UpperCamelCase_									=     intermediate_size
     UpperCamelCase_									=     hidden_dropout_prob
     UpperCamelCase_									=     attention_probs_dropout_prob
     UpperCamelCase_									=     max_position_embeddings
     UpperCamelCase_									=     initializer_range
     UpperCamelCase_									=     layer_norm_eps
     UpperCamelCase_									=     position_embedding_type
     UpperCamelCase_									=     use_cache
     UpperCamelCase_									=     tie_word_embeddings
     UpperCamelCase_									=     num_image_with_embedding

     UpperCamelCase_									=     bos_token_id
     UpperCamelCase_									=     eos_token_id




  def    _UpperCAmelCase     (     self )			-> str:
     UpperCamelCase_									=     copy.deepcopy(self.__dict__ )
     UpperCamelCase_									=     self.vision_config.to_dict()
     UpperCamelCase_									=     self.__class__.model_type
     return output





