


'''simple docstring'''







import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
       from transformers.pipelines.conversational import Conversation


lowerCAmelCase    :     Optional[int]					=						logging.get_logger(__name__)

lowerCAmelCase    :     Union[str, Any]					=						{"""vocab_file""": """vocab.txt""", """emoji_file""": """emoji.json"""}

lowerCAmelCase    :     Tuple					=						{
    """vocab_file""": {
        """abeja/gpt-neox-japanese-2.7b""": """https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt""",
    },
    """emoji_file""": {
        """abeja/gpt-neox-japanese-2.7b""": """https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json""",
    },
}

lowerCAmelCase    :     Union[str, Any]					=						{
    """abeja/gpt-neox-japanese-2.7b""": 20_48,
}






def 			lowercase				(_A   ,						_A					):







       """simple docstring"""







       with open(_A   ,						'r'   ,						encoding='utf-8'					) as f:
              _lowerCAmelCase    :							int   			=       json.loads(f.read()					)

       _lowerCAmelCase    :							List[str]   			=       collections.OrderedDict()
       _lowerCAmelCase    :							List[str]   			=       collections.OrderedDict()
       _lowerCAmelCase    :							Dict   			=       collections.OrderedDict()
       with open(_A   ,						'r'   ,						encoding='utf-8'					) as f:
              _lowerCAmelCase    :							Any   			=       f.readlines()
       _lowerCAmelCase    :							List[str]   			=       [[t.rstrip('\n'					)] if (t == ',' or ',' not in t) else t.rstrip('\n'					).split(','					) for t in token]
       for idx, b in enumerate(_A					):
              _lowerCAmelCase    :							int   			=       b
              _lowerCAmelCase    :							Union[str, Any]   			=       idx
              for wd in b:
                     _lowerCAmelCase    :							Any   			=       idx

       return vocab, raw_vocab, ids_to_tokens, emoji
class 	UpperCamelCase__					( SCREAMING_SNAKE_CASE_	):



     """simple docstring"""






     __magic_name__    			=    VOCAB_FILES_NAMES
     __magic_name__    			=    PRETRAINED_VOCAB_FILES_MAP
     __magic_name__    			=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __magic_name__    			=    ["input_ids", "attention_mask"]







     def __init__( self      ,	snake_case__      ,	snake_case__      ,	snake_case__="<|endoftext|>"      ,	snake_case__="<|endoftext|>"      ,	snake_case__="<|startoftext|>"      ,	snake_case__="<|endoftext|>"      ,	snake_case__=False      ,	**snake_case__      ,	):

            '''simple docstring'''

            super().__init__(
                unk_token=snake_case__      ,	pad_token=snake_case__      ,	bos_token=snake_case__      ,	eos_token=snake_case__      ,	do_clean_text=snake_case__      ,	**snake_case__      ,	)
            if not os.path.isfile(snake_case__   ):
                   raise ValueError(
                       F'Can\'t find a vocabulary file at path \'{vocab_file}\'. To load the vocabulary from a Google pretrained'
                       ' model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'   )
            if not os.path.isfile(snake_case__   ):
                   raise ValueError(
                       F'Can\'t find a emoji file at path \'{emoji_file}\'. To load the emoji information from a Google'
                       ' pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'   )
            _lowerCAmelCase    :							Optional[Any]   			=       do_clean_text
            _lowerCAmelCase			,     _lowerCAmelCase			,     _lowerCAmelCase			,     _lowerCAmelCase    :							List[Any]   			=       load_vocab_and_emoji(snake_case__      ,	snake_case__   )
            _lowerCAmelCase    :							Optional[int]   			=       SubWordJapaneseTokenizer(
                vocab=self.vocab      ,	ids_to_tokens=self.ids_to_tokens      ,	emoji=self.emoji   )







     @property
     def a	( self   ):

            '''simple docstring'''

            return len(self.raw_vocab   )







     def a	( self   ):

            '''simple docstring'''

            return dict(self.raw_vocab      ,	**self.added_tokens_encoder   )







     def a	( self      ,	snake_case__   ):

            '''simple docstring'''

            return self.subword_tokenizer.tokenize(snake_case__      ,	clean=self.do_clean_text   )







     def a	( self      ,	snake_case__   ):

            '''simple docstring'''

            return self.vocab.get(snake_case__      ,	self.vocab.get(self.unk_token   )   )







     def a	( self      ,	snake_case__   ):

            '''simple docstring'''

            return self.subword_tokenizer.convert_id_to_token(snake_case__   )







     def a	( self      ,	snake_case__   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Any   			=       ''.join(snake_case__   ).strip()
            return out_string







     def a	( self      ,	snake_case__   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Tuple   			=       []
            for is_user, text in conversation.iter_texts():
                   input_ids.extend(self.encode(snake_case__      ,	add_special_tokens=snake_case__   ) + [self.eos_token_id]   )

            if len(snake_case__   ) > self.model_max_length:
                   _lowerCAmelCase    :							List[str]   			=       input_ids[-self.model_max_length :]
            return input_ids







     def a	( self      ,	snake_case__      ,	snake_case__ = None   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Tuple   			=       0
            if os.path.isdir(snake_case__   ):
                   _lowerCAmelCase    :							Optional[Any]   			=       os.path.join(
                       snake_case__      ,	(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']   )
                   _lowerCAmelCase    :							Tuple   			=       os.path.join(
                       snake_case__      ,	(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['emoji_file']   )
            else:
                   _lowerCAmelCase    :							str   			=       (
                       (filename_prefix + '-' if filename_prefix else '') + save_directory + VOCAB_FILES_NAMES['vocab_file']
                   )
                   _lowerCAmelCase    :							Dict   			=       (
                       (filename_prefix + '-' if filename_prefix else '') + save_directory + VOCAB_FILES_NAMES['emoji_file']
                   )
            with open(snake_case__      ,	'w'      ,	encoding='utf-8'   ) as writer:
                   for token_index, token in self.ids_to_tokens.items():
                          if index != token_index:
                                 logger.warning(
                                     F'Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'
                                     ' Please check that the vocabulary is not corrupted!'   )
                                 _lowerCAmelCase    :							Optional[Any]   			=       token_index
                          writer.write(','.join(snake_case__   ) + '\n'   )
                          index += 1
            with open(snake_case__      ,	'w'      ,	encoding='utf-8'   ) as writer:
                   json.dump(self.emoji      ,	snake_case__   )
            return vocab_file, emoji_file
class 	UpperCamelCase__					( SCREAMING_SNAKE_CASE_	):



     """simple docstring"""






     def __init__( self      ,	snake_case__      ,	snake_case__      ,	snake_case__   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Optional[int]   			=       vocab  # same as swe
            _lowerCAmelCase    :							Tuple   			=       ids_to_tokens  # same as bpe
            _lowerCAmelCase    :							List[str]   			=       emoji
            _lowerCAmelCase    :							Union[str, Any]   			=       np.max([len(snake_case__   ) for w in self.vocab.keys()]   )
            _lowerCAmelCase    :							int   			=       re.compile(R'(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)'   )
            _lowerCAmelCase    :							Optional[int]   			=       re.compile(R'[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*'   )
            _lowerCAmelCase    :							Any   			=       re.compile(R'[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}'   )
            _lowerCAmelCase    :							Optional[Any]   			=       re.compile(
                R'([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'   )
            _lowerCAmelCase    :							List[Any]   			=       re.compile(
                R'(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'   )
            _lowerCAmelCase    :							Optional[Any]   			=       re.compile(
                R'((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*'   )
            _lowerCAmelCase    :							Dict   			=       '─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿'
            _lowerCAmelCase    :							List[Any]   			=       '▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟'
            _lowerCAmelCase    :							Optional[Any]   			=       str.maketrans({k: '<BLOCK>' for k in keisen + blocks}   )







     def __len__( self   ):

            '''simple docstring'''

            return len(self.ids_to_tokens   )







     def a	( self      ,	snake_case__   ):

            '''simple docstring'''

            _lowerCAmelCase    :							int   			=       self.content_repattera.sub('<URL>'      ,	snake_case__   )
            _lowerCAmelCase    :							str   			=       self.content_repattera.sub('<EMAIL>'      ,	snake_case__   )
            _lowerCAmelCase    :							int   			=       self.content_repattera.sub('<TEL>'      ,	snake_case__   )
            _lowerCAmelCase    :							str   			=       self.content_repattera.sub('<DATE>'      ,	snake_case__   )
            _lowerCAmelCase    :							int   			=       self.content_repattera.sub('<DATE>'      ,	snake_case__   )
            _lowerCAmelCase    :							List[Any]   			=       self.content_repattera.sub('<PRICE>'      ,	snake_case__   )
            _lowerCAmelCase    :							Dict   			=       content.translate(self.content_transa   )
            while "<BLOCK><BLOCK>" in content:
                   _lowerCAmelCase    :							Dict   			=       content.replace('<BLOCK><BLOCK>'      ,	'<BLOCK>'   )
            return content







     def a	( self      ,	snake_case__      ,	snake_case__=False   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Any   			=       text.replace(' '      ,	'<SP>'   )
            _lowerCAmelCase    :							Any   			=       text.replace('　'      ,	'<SP>'   )
            _lowerCAmelCase    :							Dict   			=       text.replace('\r\n'      ,	'<BR>'   )
            _lowerCAmelCase    :							int   			=       text.replace('\n'      ,	'<BR>'   )
            _lowerCAmelCase    :							Tuple   			=       text.replace('\r'      ,	'<BR>'   )
            _lowerCAmelCase    :							int   			=       text.replace('\t'      ,	'<TAB>'   )
            _lowerCAmelCase    :							List[Any]   			=       text.replace('—'      ,	'ー'   )
            _lowerCAmelCase    :							List[Any]   			=       text.replace('−'      ,	'ー'   )
            for k, v in self.emoji["emoji"].items():
                   if k in text:
                          _lowerCAmelCase    :							Union[str, Any]   			=       text.replace(snake_case__      ,	snake_case__   )
            if clean:
                   _lowerCAmelCase    :							Optional[int]   			=       self.clean_text(snake_case__   )

            def check_simbol(snake_case__   ):
                   _lowerCAmelCase    :							int   			=       x.encode()
                   if len(snake_case__   ) == 1 and len(snake_case__   ) == 2:
                          _lowerCAmelCase    :							str   			=       (int(e[0]   ) << 8) + int(e[1]   )
                          if (
                              (c >= 0xC2_A1 and c <= 0xC2_BF)
                              or (c >= 0xC7_80 and c <= 0xC7_83)
                              or (c >= 0xCA_B9 and c <= 0xCB_BF)
                              or (c >= 0xCC_80 and c <= 0xCD_A2)
                          ):
                                 return True
                   return False

            def checkuae(snake_case__   ):
                   _lowerCAmelCase    :							Union[str, Any]   			=       x.encode()
                   if len(snake_case__   ) == 1 and len(snake_case__   ) == 3:
                          _lowerCAmelCase    :							Dict   			=       (int(e[0]   ) << 16) + (int(e[1]   ) << 8) + int(e[2]   )
                          if c >= 0xE2_80_80 and c <= 0xE2_B0_7F:
                                 return True
                   return False

            _lowerCAmelCase    :							Optional[Any]   			=       0
            _lowerCAmelCase    :							int   			=       []
            while pos < len(snake_case__   ):
                   _lowerCAmelCase    :							List[Any]   			=       min(len(snake_case__   )      ,	pos + self.maxlen + 1   ) if text[pos] == '<' else pos + 3
                   _lowerCAmelCase    :							str   			=       []  # (token_id, token, pos)
                   for e in range(snake_case__      ,	snake_case__      ,	-1   ):
                          _lowerCAmelCase    :							Tuple   			=       text[pos:e]
                          if wd in self.vocab:
                                 if wd[0] == "<" and len(snake_case__   ) > 2:
                                        _lowerCAmelCase    :							List[Any]   			=       [(self.vocab[wd], wd, e)]
                                        break
                                 else:
                                        candidates.append((self.vocab[wd], wd, e)   )
                   if len(snake_case__   ) > 0:
                          # the smallest token_id is adopted
                          _lowerCAmelCase			,     _lowerCAmelCase			,     _lowerCAmelCase    :							List[str]   			=       sorted(snake_case__      ,	key=lambda snake_case__   : x[0]   )[0]
                          result.append(snake_case__   )
                          _lowerCAmelCase    :							int   			=       e
                   else:
                          _lowerCAmelCase    :							List[Any]   			=       pos + 1
                          _lowerCAmelCase    :							List[str]   			=       text[pos:end]
                          if check_simbol(snake_case__   ):
                                 result.append('<KIGOU>'   )
                          elif checkuae(snake_case__   ):
                                 result.append('<U2000U2BFF>'   )
                          else:
                                 for i in wd.encode('utf-8'   ):
                                        result.append('<|byte%d|>' % i   )
                          _lowerCAmelCase    :							List[Any]   			=       end
            return result







     def a	( self      ,	snake_case__      ,	snake_case__="\n"   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Union[str, Any]   			=       []
            _lowerCAmelCase    :							List[Any]   			=       []
            _lowerCAmelCase    :							int   			=       self.ids_to_tokens[index][0]
            if word[:6] == "<|byte" and word[-2:] == "|>":
                   byte_tokens.append(int(word[6:-2]   )   )
            else:
                   if len(snake_case__   ) > 0:
                          words.append(bytearray(snake_case__   ).decode('utf-8'      ,	errors='replace'   )   )
                          _lowerCAmelCase    :							Tuple   			=       []
                   if word[:7] == "<|emoji" and word[-2:] == "|>":
                          words.append(self.emoji['emoji_inv'][word]   )
                   elif word == "<SP>":
                          words.append(' '   )
                   elif word == "<BR>":
                          words.append(snake_case__   )
                   elif word == "<TAB>":
                          words.append('\t'   )
                   elif word == "<BLOCK>":
                          words.append('▀'   )
                   elif word == "<KIGOU>":
                          words.append('ǀ'   )
                   elif word == "<U2000U2BFF>":
                          words.append('‖'   )
                   else:
                          words.append(snake_case__   )
            if len(snake_case__   ) > 0:
                   words.append(bytearray(snake_case__   ).decode('utf-8'      ,	errors='replace'   )   )
            _lowerCAmelCase    :							Dict   			=       ''.join(snake_case__   )
            return text





'''simple docstring'''







from .configuration_bert_masked import MaskedBertConfig
from .modeling_bert_masked import (
    MaskedBertForMultipleChoice,
    MaskedBertForQuestionAnswering,
    MaskedBertForSequenceClassification,
    MaskedBertForTokenClassification,
    MaskedBertModel,
)
from .modules import *

