



import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


a_			=					logging.get_logger(__name__)

a_			=					{"""vocab_file""": """sentencepiece.bpe.model"""}

a_			=					{
    """vocab_file""": {
        """camembert-base""": """https://huggingface.co/camembert-base/resolve/main/sentencepiece.bpe.model""",
    }
}

a_			=					{
    """camembert-base""": 512,
}

a_			=					"""▁"""



class  __lowerCAmelCase     (	lowerCAmelCase__   ):
 lowerCAmelCase__       					=  VOCAB_FILES_NAMES
 lowerCAmelCase__       					=  PRETRAINED_VOCAB_FILES_MAP
 lowerCAmelCase__       					=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 lowerCAmelCase__       					=  ["""input_ids""", """attention_mask"""]




 def __init__(						self	,				__UpperCAmelCase	,				__UpperCAmelCase="<s>"	,				__UpperCAmelCase="</s>"	,				__UpperCAmelCase="</s>"	,				__UpperCAmelCase="<s>"	,				__UpperCAmelCase="<unk>"	,				__UpperCAmelCase="<pad>"	,				__UpperCAmelCase="<mask>"	,				__UpperCAmelCase=["<s>NOTUSED", "</s>NOTUSED"]	,				__UpperCAmelCase = None	,				**__UpperCAmelCase	,				):



    '''simple docstring'''
    # Mask token behave like a normal word, i.e. include the space before it
    __lowerCamelCase													=    AddedToken(__UpperCAmelCase	,				lstrip=__UpperCAmelCase	,				rstrip=__UpperCAmelCase				) if isinstance(__UpperCAmelCase	,				__UpperCAmelCase				) else mask_token

    __lowerCamelCase													=    {} if sp_model_kwargs is None else sp_model_kwargs

    super().__init__(
        bos_token=__UpperCAmelCase	,				eos_token=__UpperCAmelCase	,				unk_token=__UpperCAmelCase	,				sep_token=__UpperCAmelCase	,				cls_token=__UpperCAmelCase	,				pad_token=__UpperCAmelCase	,				mask_token=__UpperCAmelCase	,				additional_special_tokens=__UpperCAmelCase	,				sp_model_kwargs=self.sp_model_kwargs	,				**__UpperCAmelCase	,				)
    __lowerCamelCase													=    spm.SentencePieceProcessor(**self.sp_model_kwargs				)
    self.sp_model.Load(str(__UpperCAmelCase				)				)
    __lowerCamelCase													=    vocab_file
    # HACK: These tokens were added by fairseq but don't seem to be actually used when duplicated in the actual
    # sentencepiece vocabulary (this is the case for <s> and </s>
    __lowerCamelCase													=    {'''<s>NOTUSED''': 0, '''<pad>''': 1, '''</s>NOTUSED''': 2, '''<unk>''': 3}
    __lowerCamelCase													=    len(self.fairseq_tokens_to_ids				)
    __lowerCamelCase													=    len(self.sp_model				) + len(self.fairseq_tokens_to_ids				)
    __lowerCamelCase													=    {v: k for k, v in self.fairseq_tokens_to_ids.items()}




 def        lowerCamelCase      (						self	,				__UpperCAmelCase	,				__UpperCAmelCase = None				):



    '''simple docstring'''

    if token_ids_a is None:
       return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
    __lowerCamelCase													=    [self.cls_token_id]
    __lowerCamelCase													=    [self.sep_token_id]
    return cls + token_ids_a + sep + sep + token_ids_a + sep




 def        lowerCamelCase      (						self	,				__UpperCAmelCase	,				__UpperCAmelCase = None	,				__UpperCAmelCase = False				):



    '''simple docstring'''
    if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=__UpperCAmelCase	,				token_ids_a=__UpperCAmelCase	,				already_has_special_tokens=__UpperCAmelCase				)

    if token_ids_a is None:
       return [1] + ([0] * len(__UpperCAmelCase				)) + [1]
    return [1] + ([0] * len(__UpperCAmelCase				)) + [1, 1] + ([0] * len(__UpperCAmelCase				)) + [1]




 def        lowerCamelCase      (						self	,				__UpperCAmelCase	,				__UpperCAmelCase = None				):



    '''simple docstring'''
    __lowerCamelCase													=    [self.sep_token_id]
    __lowerCamelCase													=    [self.cls_token_id]

    if token_ids_a is None:
       return len(cls + token_ids_a + sep				) * [0]
    return len(cls + token_ids_a + sep + sep + token_ids_a + sep				) * [0]




 @property
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    return len(self.fairseq_tokens_to_ids				) + len(self.sp_model				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    {self.convert_ids_to_tokens(__UpperCAmelCase				): i for i in range(self.vocab_size				)}
    vocab.update(self.added_tokens_encoder				)
    return vocab




 def        lowerCamelCase      (						self	,				__UpperCAmelCase				):



    '''simple docstring'''
    return self.sp_model.encode(__UpperCAmelCase	,				out_type=__UpperCAmelCase				)




 def        lowerCamelCase      (						self	,				__UpperCAmelCase				):



    '''simple docstring'''
    if token in self.fairseq_tokens_to_ids:
       return self.fairseq_tokens_to_ids[token]
    elif self.sp_model.PieceToId(__UpperCAmelCase				) == 0:
       # Convert sentence piece unk token to fairseq unk token index
       return self.unk_token_id
    return self.fairseq_offset + self.sp_model.PieceToId(__UpperCAmelCase				)




 def        lowerCamelCase      (						self	,				__UpperCAmelCase				):



    '''simple docstring'''
    if index in self.fairseq_ids_to_tokens:
       return self.fairseq_ids_to_tokens[index]
    return self.sp_model.IdToPiece(index - self.fairseq_offset				)




 def        lowerCamelCase      (						self	,				__UpperCAmelCase				):



    '''simple docstring'''
    __lowerCamelCase													=    []
    __lowerCamelCase													=    ''''''
    __lowerCamelCase													=    False
    for token in tokens:
       # make sure that special tokens are not decoded using sentencepiece model
       if token in self.all_special_tokens:
          if not prev_is_special:
             out_string += " "
          out_string += self.sp_model.decode(__UpperCAmelCase				) + token
          __lowerCamelCase													=    True
          __lowerCamelCase													=    []
       else:
          current_sub_tokens.append(__UpperCAmelCase				)
          __lowerCamelCase													=    False
    out_string += self.sp_model.decode(__UpperCAmelCase				)
    return out_string.strip()




 def __getstate__(						self				):



    '''simple docstring'''
    __lowerCamelCase													=    self.__dict__.copy()
    __lowerCamelCase													=    None
    return state




 def __setstate__(						self	,				__UpperCAmelCase				):



    '''simple docstring'''
    __lowerCamelCase													=    d

    # for backward compatibility
    if not hasattr(self	,				'''sp_model_kwargs'''				):
       __lowerCamelCase													=    {}

    __lowerCamelCase													=    spm.SentencePieceProcessor(**self.sp_model_kwargs				)
    self.sp_model.Load(self.vocab_file				)




 def        lowerCamelCase      (						self	,				__UpperCAmelCase	,				__UpperCAmelCase = None				):



    '''simple docstring'''
    if not os.path.isdir(__UpperCAmelCase				):
       logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""				)
       return
    __lowerCamelCase													=    os.path.join(
        __UpperCAmelCase	,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)

    if os.path.abspath(self.vocab_file				) != os.path.abspath(__UpperCAmelCase				) and os.path.isfile(self.vocab_file				):
       copyfile(self.vocab_file	,				__UpperCAmelCase				)
    elif not os.path.isfile(self.vocab_file				):
       with open(__UpperCAmelCase	,				'''wb'''				) as fi:
          __lowerCamelCase													=    self.sp_model.serialized_model_proto()
          fi.write(__UpperCAmelCase				)

    return (out_vocab_file,)











from ...configuration_utils import PretrainedConfig
from ...utils import logging


a_			=					logging.get_logger(__name__)

a_			=					{
    """unc-nlp/lxmert-base-uncased""": """https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/config.json""",
}



class  __lowerCAmelCase     (	lowerCAmelCase__   ):
 lowerCAmelCase__       					=  """lxmert"""
 lowerCAmelCase__       					=  {}




 def __init__(						self	,				__UpperCAmelCase=30522	,				__UpperCAmelCase=768	,				__UpperCAmelCase=12	,				__UpperCAmelCase=9500	,				__UpperCAmelCase=1600	,				__UpperCAmelCase=400	,				__UpperCAmelCase=3072	,				__UpperCAmelCase="gelu"	,				__UpperCAmelCase=0.1	,				__UpperCAmelCase=0.1	,				__UpperCAmelCase=512	,				__UpperCAmelCase=2	,				__UpperCAmelCase=0.02	,				__UpperCAmelCase=1E-1_2	,				__UpperCAmelCase=9	,				__UpperCAmelCase=5	,				__UpperCAmelCase=5	,				__UpperCAmelCase=2048	,				__UpperCAmelCase=4	,				__UpperCAmelCase=6.67	,				__UpperCAmelCase=True	,				__UpperCAmelCase=True	,				__UpperCAmelCase=True	,				__UpperCAmelCase=True	,				__UpperCAmelCase=True	,				__UpperCAmelCase=True	,				__UpperCAmelCase=True	,				**__UpperCAmelCase	,				):



    '''simple docstring'''
    __lowerCamelCase													=    vocab_size
    __lowerCamelCase													=    hidden_size
    __lowerCamelCase													=    num_attention_heads
    __lowerCamelCase													=    hidden_act
    __lowerCamelCase													=    intermediate_size
    __lowerCamelCase													=    hidden_dropout_prob
    __lowerCamelCase													=    attention_probs_dropout_prob
    __lowerCamelCase													=    max_position_embeddings
    __lowerCamelCase													=    type_vocab_size
    __lowerCamelCase													=    initializer_range
    __lowerCamelCase													=    layer_norm_eps
    __lowerCamelCase													=    num_qa_labels
    __lowerCamelCase													=    num_object_labels
    __lowerCamelCase													=    num_attr_labels
    __lowerCamelCase													=    l_layers
    __lowerCamelCase													=    x_layers
    __lowerCamelCase													=    r_layers
    __lowerCamelCase													=    visual_feat_dim
    __lowerCamelCase													=    visual_pos_dim
    __lowerCamelCase													=    visual_loss_normalizer
    __lowerCamelCase													=    task_matched
    __lowerCamelCase													=    task_mask_lm
    __lowerCamelCase													=    task_obj_predict
    __lowerCamelCase													=    task_qa
    __lowerCamelCase													=    visual_obj_loss
    __lowerCamelCase													=    visual_attr_loss
    __lowerCamelCase													=    visual_feat_loss
    __lowerCamelCase													=    {'''vision''': r_layers, '''cross_encoder''': x_layers, '''language''': l_layers}
    super().__init__(**__UpperCAmelCase				)






