


import os
import re
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__a     =			logging.get_logger(__name__)

__a     =			{
    'vocab_file': 'vocab.txt',
    'merges_file': 'bpe.codes',
}

__a     =			{
    'vocab_file': {
        'vinai/phobert-base': 'https://huggingface.co/vinai/phobert-base/resolve/main/vocab.txt',
        'vinai/phobert-large': 'https://huggingface.co/vinai/phobert-large/resolve/main/vocab.txt',
    },
    'merges_file': {
        'vinai/phobert-base': 'https://huggingface.co/vinai/phobert-base/resolve/main/bpe.codes',
        'vinai/phobert-large': 'https://huggingface.co/vinai/phobert-large/resolve/main/bpe.codes',
    },
}

__a     =			{
    'vinai/phobert-base': 256,
    'vinai/phobert-large': 256,
}
def   _UpperCamelCase				(						lowerCAmelCase_   )						->Optional[int]:
   UpperCAmelCase							=     set()
   UpperCAmelCase							=     word[0]
   for char in word[1:]:
      pairs.add((prev_char, char)   )
      UpperCAmelCase							=     char

   UpperCAmelCase							=     set(_lowerCAmelCase   )
   return pairs







class   __lowercase				(  __snake_case  ):
      UpperCamelCase								=       VOCAB_FILES_NAMES
      UpperCamelCase								=       PRETRAINED_VOCAB_FILES_MAP
      UpperCamelCase								=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES





      def __init__(		self						:					List[str]					,       __lowerCamelCase						:					List[str]					,       __lowerCamelCase						:					Optional[Any]					,       __lowerCamelCase						:					Any="<s>"					,       __lowerCamelCase						:					Union[str, Any]="</s>"					,       __lowerCamelCase						:					Union[str, Any]="</s>"					,       __lowerCamelCase						:					Dict="<s>"					,       __lowerCamelCase						:					Tuple="<unk>"					,       __lowerCamelCase						:					Any="<pad>"					,       __lowerCamelCase						:					List[str]="<mask>"					,       **__lowerCamelCase						:					str					,       )    ->   Optional[int]:






         """simple docstring"""







         super().__init__(
             bos_token=__A					,       eos_token=__A					,       unk_token=__A					,       sep_token=__A					,       cls_token=__A					,       pad_token=__A					,       mask_token=__A					,       **__A					,       )

         UpperCAmelCase							=     vocab_file
         UpperCAmelCase							=     merges_file

         UpperCAmelCase							=     {}
         UpperCAmelCase							=     0
         UpperCAmelCase							=     1
         UpperCAmelCase							=     2
         UpperCAmelCase							=     3

         self.add_from_file(__A       )

         UpperCAmelCase							=     {v: k for k, v in self.encoder.items()}

         with open(__A					,       encoding="""utf-8"""       ) as merges_handle:
            UpperCAmelCase							=     merges_handle.read().split("""\n"""       )[:-1]
         UpperCAmelCase							=     [tuple(merge.split()[:-1]       ) for merge in merges]
         UpperCAmelCase							=     dict(zip(__A					,       range(len(__A       )       )       )       )
         UpperCAmelCase							=     {}





      def        _lowercase				(		self						:					Optional[int]					,       __lowerCamelCase						:					List[int]					,       __lowerCamelCase						:					Optional[List[int]] = None       )    ->   int:






         """simple docstring"""







         if token_ids_a is None:
            return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
         UpperCAmelCase							=     [self.cls_token_id]
         UpperCAmelCase							=     [self.sep_token_id]
         return cls + token_ids_a + sep + sep + token_ids_a + sep





      def        _lowercase				(		self						:					Union[str, Any]					,       __lowerCamelCase						:					List[int]					,       __lowerCamelCase						:					Optional[List[int]] = None					,       __lowerCamelCase						:					bool = False       )    ->   Any:






         """simple docstring"""







         if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_a=__A					,       token_ids_a=__A					,       already_has_special_tokens=__A       )

         if token_ids_a is None:
            return [1] + ([0] * len(__A       )) + [1]
         return [1] + ([0] * len(__A       )) + [1, 1] + ([0] * len(__A       )) + [1]





      def        _lowercase				(		self						:					int					,       __lowerCamelCase						:					List[int]					,       __lowerCamelCase						:					Optional[List[int]] = None       )    ->   str:






         """simple docstring"""







         UpperCAmelCase							=     [self.sep_token_id]
         UpperCAmelCase							=     [self.cls_token_id]

         if token_ids_a is None:
            return len(cls + token_ids_a + sep       ) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]





      @property
      def        _lowercase				(		self						:					int       )    ->   List[str]:






         """simple docstring"""







         return len(self.encoder       )





      def        _lowercase				(		self						:					List[Any]       )    ->   Any:






         """simple docstring"""







         return dict(self.encoder					,       **self.added_tokens_encoder       )





      def        _lowercase				(		self						:					Any					,       __lowerCamelCase						:					Optional[Any]       )    ->   Optional[Any]:






         """simple docstring"""







         if token in self.cache:
            return self.cache[token]
         UpperCAmelCase							=     tuple(__A       )
         UpperCAmelCase							=     tuple(list(word[:-1]       ) + [word[-1] + """</w>"""]       )
         UpperCAmelCase							=     get_pairs(__A       )

         if not pairs:
            return token

         while True:
            UpperCAmelCase							=     min(__A					,       key=lambda __lowerCamelCase       : self.bpe_ranks.get(__A					,       float("""inf"""       )       )       )
            if bigram not in self.bpe_ranks:
               break
            UpperCAmelCase							=     bigram
            UpperCAmelCase							=     []
            UpperCAmelCase							=     0
            while i < len(__A       ):
               try:
                  UpperCAmelCase							=     word.index(__A					,       __A       )
               except ValueError:
                  new_word.extend(word[i:]       )
                  break
               else:
                  new_word.extend(word[i:j]       )
                  UpperCAmelCase							=     j

               if word[i] == first and i < len(__A       ) - 1 and word[i + 1] == second:
                  new_word.append(first + second       )
                  i += 2
               else:
                  new_word.append(word[i]       )
                  i += 1
            UpperCAmelCase							=     tuple(__A       )
            UpperCAmelCase							=     new_word
            if len(__A       ) == 1:
               break
            else:
               UpperCAmelCase							=     get_pairs(__A       )
         UpperCAmelCase							=     "@@ ".join(__A       )
         UpperCAmelCase							=     word[:-4]
         UpperCAmelCase							=     word
         return word





      def        _lowercase				(		self						:					str					,       __lowerCamelCase						:					Optional[Any]       )    ->   Any:






         """simple docstring"""







         UpperCAmelCase							=     []

         UpperCAmelCase							=     re.findall(r"""\S+\n?"""					,       __A       )

         for token in words:
            split_tokens.extend(list(self.bpe(__A       ).split(""" """       )       )       )
         return split_tokens





      def        _lowercase				(		self						:					str					,       __lowerCamelCase						:					Union[str, Any]       )    ->   Union[str, Any]:






         """simple docstring"""







         return self.encoder.get(__A					,       self.encoder.get(self.unk_token       )       )





      def        _lowercase				(		self						:					List[str]					,       __lowerCamelCase						:					Optional[Any]       )    ->   List[str]:






         """simple docstring"""







         return self.decoder.get(__A					,       self.unk_token       )





      def        _lowercase				(		self						:					Optional[int]					,       __lowerCamelCase						:					List[Any]       )    ->   Tuple:






         """simple docstring"""







         UpperCAmelCase							=     " ".join(__A       ).replace("""@@ """					,       """"""       ).strip()
         return out_string





      def        _lowercase				(		self						:					Optional[int]					,       __lowerCamelCase						:					str					,       __lowerCamelCase						:					Optional[str] = None       )    ->   List[Any]:






         """simple docstring"""







         if not os.path.isdir(__A       ):
            logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""       )
            return
         UpperCAmelCase							=     os.path.join(
             __A					,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]       )
         UpperCAmelCase							=     os.path.join(
             __A					,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]       )

         if os.path.abspath(self.vocab_file       ) != os.path.abspath(__A       ):
            copyfile(self.vocab_file					,       __A       )

         if os.path.abspath(self.merges_file       ) != os.path.abspath(__A       ):
            copyfile(self.merges_file					,       __A       )

         return out_vocab_file, out_merge_file





      def        _lowercase				(		self						:					Union[str, Any]					,       __lowerCamelCase						:					Optional[Any]       )    ->   Dict:






         """simple docstring"""







         if isinstance(__A					,       __A       ):
            try:
               with open(__A					,       """r"""					,       encoding="""utf-8"""       ) as fd:
                  self.add_from_file(__A       )
            except FileNotFoundError as fnfe:
               raise fnfe
            except UnicodeError:
               raise Exception(F"""Incorrect encoding detected in {f}, please rebuild the dataset"""       )
            return

         UpperCAmelCase							=     f.readlines()
         for lineTmp in lines:
            UpperCAmelCase							=     lineTmp.strip()
            UpperCAmelCase							=     line.rfind(""" """       )
            if idx == -1:
               raise ValueError("""Incorrect dictionary format, expected '<token> <cnt>'"""       )
            UpperCAmelCase							=     line[:idx]
            UpperCAmelCase							=     len(self.encoder       )








from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


__a     =			{"""configuration_ibert""": ["""IBERT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """IBertConfig""", """IBertOnnxConfig"""]}

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __a     =			[
     """IBERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
     """IBertForMaskedLM""",
     """IBertForMultipleChoice""",
     """IBertForQuestionAnswering""",
     """IBertForSequenceClassification""",
     """IBertForTokenClassification""",
     """IBertModel""",
     """IBertPreTrainedModel""",
 ]

if TYPE_CHECKING:
 from .configuration_ibert import IBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, IBertConfig, IBertOnnxConfig

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_ibert import (
      IBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
      IBertForMaskedLM,
      IBertForMultipleChoice,
      IBertForQuestionAnswering,
      IBertForSequenceClassification,
      IBertForTokenClassification,
      IBertModel,
      IBertPreTrainedModel,
  )

else:
 import sys

 __a     =			_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)



