

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase__								=		logging.get_logger(__name__)

lowerCAmelCase__								=		{
    """google/realm-cc-news-pretrained-embedder""": (
        """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/config.json"""
    ),
    """google/realm-cc-news-pretrained-encoder""": (
        """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/config.json"""
    ),
    """google/realm-cc-news-pretrained-scorer""": (
        """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/config.json"""
    ),
    """google/realm-cc-news-pretrained-openqa""": (
        """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/config.json"""
    ),
    """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/config.json""",
    """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/config.json""",
    """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/config.json""",
    """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/config.json""",
    # See all REALM models at https://huggingface.co/models?filter=realm
}







class 	lowercase    (				_lowercase							):


		"""simple docstring"""




		a__					       =  "realm"


		def __init__(				self	,       __snake_case=3_05_22	,       __snake_case=7_68	,       __snake_case=1_28	,       __snake_case=12	,       __snake_case=12	,       __snake_case=8	,       __snake_case=30_72	,       __snake_case="gelu_new"	,       __snake_case=0.1	,       __snake_case=0.1	,       __snake_case=5_12	,       __snake_case=2	,       __snake_case=0.0_2	,       __snake_case=1e-12	,       __snake_case=2_56	,       __snake_case=10	,       __snake_case=1e-3	,       __snake_case=5	,       __snake_case=3_20	,       __snake_case=13_35_37_18	,       __snake_case=50_00	,       __snake_case=1	,       __snake_case=0	,       __snake_case=2	,       **__snake_case	,       ):
			super().__init__(pad_token_id=__snake_case	,       bos_token_id=__snake_case	,       eos_token_id=__snake_case	,       **__snake_case)

			# Common config
			_UpperCamelCase   :		Optional[Any]			=						vocab_size
			_UpperCamelCase   :		Optional[int]			=						max_position_embeddings
			_UpperCamelCase   :		Optional[Any]			=						hidden_size
			_UpperCamelCase   :		List[str]			=						retriever_proj_size
			_UpperCamelCase   :		Any			=						num_hidden_layers
			_UpperCamelCase   :		Dict			=						num_attention_heads
			_UpperCamelCase   :		List[Any]			=						num_candidates
			_UpperCamelCase   :		List[Any]			=						intermediate_size
			_UpperCamelCase   :		int			=						hidden_act
			_UpperCamelCase   :		Dict			=						hidden_dropout_prob
			_UpperCamelCase   :		int			=						attention_probs_dropout_prob
			_UpperCamelCase   :		Tuple			=						initializer_range
			_UpperCamelCase   :		List[str]			=						type_vocab_size
			_UpperCamelCase   :		int			=						layer_norm_eps

			# Reader config
			_UpperCamelCase   :		Tuple			=						span_hidden_size
			_UpperCamelCase   :		Dict			=						max_span_width
			_UpperCamelCase   :		Dict			=						reader_layer_norm_eps
			_UpperCamelCase   :		Union[str, Any]			=						reader_beam_size
			_UpperCamelCase   :		Optional[int]			=						reader_seq_len

			# Retrieval config
			_UpperCamelCase   :		Tuple			=						num_block_records
			_UpperCamelCase   :		Tuple			=						searcher_beam_size







import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase__								=		logging.get_logger(__name__)

lowerCAmelCase__								=		"""▁"""

lowerCAmelCase__								=		{"""vocab_file""": """sentencepiece.bpe.model"""}

lowerCAmelCase__								=		{
    """vocab_file""": {
        """xlm-roberta-base""": """https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model""",
        """xlm-roberta-large""": """https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model""",
        """xlm-roberta-large-finetuned-conll02-dutch""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model"""
        ),
        """xlm-roberta-large-finetuned-conll02-spanish""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model"""
        ),
        """xlm-roberta-large-finetuned-conll03-english""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model"""
        ),
        """xlm-roberta-large-finetuned-conll03-german""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model"""
        ),
    }
}

lowerCAmelCase__								=		{
    """xlm-roberta-base""": 5_1_2,
    """xlm-roberta-large""": 5_1_2,
    """xlm-roberta-large-finetuned-conll02-dutch""": 5_1_2,
    """xlm-roberta-large-finetuned-conll02-spanish""": 5_1_2,
    """xlm-roberta-large-finetuned-conll03-english""": 5_1_2,
    """xlm-roberta-large-finetuned-conll03-german""": 5_1_2,
}







class 	lowercase    (				_lowercase							):


		"""simple docstring"""




		a__					       =  VOCAB_FILES_NAMES
		a__					       =  PRETRAINED_VOCAB_FILES_MAP
		a__					       =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		a__					       =  ["input_ids", "attention_mask"]


		def __init__(				self	,       __snake_case	,       __snake_case="<s>"	,       __snake_case="</s>"	,       __snake_case="</s>"	,       __snake_case="<s>"	,       __snake_case="<unk>"	,       __snake_case="<pad>"	,       __snake_case="<mask>"	,       __snake_case = None	,       **__snake_case	,       ):
			# Mask token behave like a normal word, i.e. include the space before it
			_UpperCamelCase   :		Dict			=						AddedToken(__snake_case	,       lstrip=__snake_case	,       rstrip=__snake_case) if isinstance(__snake_case	,       __snake_case) else mask_token

			_UpperCamelCase   :		str			=						{} if sp_model_kwargs is None else sp_model_kwargs

			super().__init__(
			    bos_token=__snake_case	,       eos_token=__snake_case	,       unk_token=__snake_case	,       sep_token=__snake_case	,       cls_token=__snake_case	,       pad_token=__snake_case	,       mask_token=__snake_case	,       sp_model_kwargs=self.sp_model_kwargs	,       **__snake_case	,       )

			_UpperCamelCase   :		Union[str, Any]			=						spm.SentencePieceProcessor(**self.sp_model_kwargs)
			self.sp_model.Load(str(__snake_case))
			_UpperCamelCase   :		Dict			=						vocab_file

			# Original fairseq vocab and spm vocab must be "aligned":
			# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
			# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
			# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
			# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

			# Mimic fairseq token-to-id alignment for the first 4 token
			_UpperCamelCase   :		int			=						{'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

			# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
			_UpperCamelCase   :		List[Any]			=						1

			_UpperCamelCase   :		Any			=						len(self.sp_model) + self.fairseq_offset
			_UpperCamelCase   :		List[str]			=						{v: k for k, v in self.fairseq_tokens_to_ids.items()}


		def __getstate__(				self):
			_UpperCamelCase   :		List[Any]			=						self.__dict__.copy()
			_UpperCamelCase   :		Optional[Any]			=						None
			_UpperCamelCase   :		Any			=						self.sp_model.serialized_model_proto()
			return state


		def __setstate__(				self	,       __snake_case):
			_UpperCamelCase   :		int			=						d

			# for backward compatibility
			if not hasattr(self	,       'sp_model_kwargs'):
				_UpperCamelCase   :		Tuple			=						{}

			_UpperCamelCase   :		Optional[int]			=						spm.SentencePieceProcessor(**self.sp_model_kwargs)
			self.sp_model.LoadFromSerializedProto(self.sp_model_proto)


		def 		A__					(				self	,       __snake_case	,       __snake_case = None):

			if token_ids_a is None:
				return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
			_UpperCamelCase   :		Tuple			=						[self.cls_token_id]
			_UpperCamelCase   :		int			=						[self.sep_token_id]
			return cls + token_ids_a + sep + sep + token_ids_a + sep


		def 		A__					(				self	,       __snake_case	,       __snake_case = None	,       __snake_case = False):

			if already_has_special_tokens:
				return super().get_special_tokens_mask(
				    token_ids_a=__snake_case	,       token_ids_a=__snake_case	,       already_has_special_tokens=__snake_case)

			if token_ids_a is None:
				return [1] + ([0] * len(__snake_case)) + [1]
			return [1] + ([0] * len(__snake_case)) + [1, 1] + ([0] * len(__snake_case)) + [1]


		def 		A__					(				self	,       __snake_case	,       __snake_case = None):
			_UpperCamelCase   :		Optional[Any]			=						[self.sep_token_id]
			_UpperCamelCase   :		Optional[Any]			=						[self.cls_token_id]

			if token_ids_a is None:
				return len(cls + token_ids_a + sep) * [0]
			return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]


		@property
		def 		A__					(				self):
			return len(self.sp_model) + self.fairseq_offset + 1  # Add the <mask> token


		def 		A__					(				self):
			_UpperCamelCase   :		List[str]			=						{self.convert_ids_to_tokens(__snake_case): i for i in range(self.vocab_size)}
			vocab.update(self.added_tokens_encoder)
			return vocab


		def 		A__					(				self	,       __snake_case):
			return self.sp_model.encode(__snake_case	,       out_type=__snake_case)


		def 		A__					(				self	,       __snake_case):
			if token in self.fairseq_tokens_to_ids:
				return self.fairseq_tokens_to_ids[token]
			_UpperCamelCase   :		str			=						self.sp_model.PieceToId(__snake_case)

			# Need to return unknown token if the SP model returned 0
			return spm_id + self.fairseq_offset if spm_id else self.unk_token_id


		def 		A__					(				self	,       __snake_case):
			if index in self.fairseq_ids_to_tokens:
				return self.fairseq_ids_to_tokens[index]
			return self.sp_model.IdToPiece(index - self.fairseq_offset)


		def 		A__					(				self	,       __snake_case):
			_UpperCamelCase   :		Optional[int]			=						''.join(__snake_case).replace(__snake_case	,       ' ').strip()
			return out_string






		def 		A__					(				self	,       __snake_case	,       __snake_case = None):
			if not os.path.isdir(__snake_case):
				logger.error(f'''Vocabulary path ({save_directory}) should be a directory''')
				return
			_UpperCamelCase   :		str			=						os.path.join(
			    __snake_case	,       (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])

			if os.path.abspath(self.vocab_file) != os.path.abspath(__snake_case) and os.path.isfile(self.vocab_file):
				copyfile(self.vocab_file	,       __snake_case)
			elif not os.path.isfile(self.vocab_file):
				with open(__snake_case	,       'wb') as fi:
					_UpperCamelCase   :		Any			=						self.sp_model.serialized_model_proto()
					fi.write(__snake_case)

			return (out_vocab_file,)




