

"""simple docstring"""




import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
						from transformers.pipelines.conversational import Conversation


lowerCAmelCase__							=				logging.get_logger(__name__)

lowerCAmelCase__							=				{'''vocab_file''': '''vocab.txt''', '''emoji_file''': '''emoji.json'''}

lowerCAmelCase__							=				{
    '''vocab_file''': {
        '''abeja/gpt-neox-japanese-2.7b''': '''https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt''',
    },
    '''emoji_file''': {
        '''abeja/gpt-neox-japanese-2.7b''': '''https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json''',
    },
}

lowerCAmelCase__							=				{
    '''abeja/gpt-neox-japanese-2.7b''': 2048,
}



def        _lowerCamelCase ( __a,							__a    ):

		with open(_A,							'''r''',							encoding='''utf-8'''    ) as f:
				SCREAMING_SNAKE_CASE_						 = json.loads(f.read()    )

		SCREAMING_SNAKE_CASE_						 = collections.OrderedDict()
		SCREAMING_SNAKE_CASE_						 = collections.OrderedDict()
		SCREAMING_SNAKE_CASE_						 = collections.OrderedDict()
		with open(_A,							'''r''',							encoding='''utf-8'''    ) as f:
				SCREAMING_SNAKE_CASE_						 = f.readlines()
		SCREAMING_SNAKE_CASE_						 = [[t.rstrip('''\n'''    )] if (t == ''',''' or ''',''' not in t) else t.rstrip('''\n'''    ).split(''','''    ) for t in token]
		for idx, b in enumerate(_A    ):
				SCREAMING_SNAKE_CASE_						 = b
				SCREAMING_SNAKE_CASE_						 = idx
				for wd in b:
						SCREAMING_SNAKE_CASE_						 = idx

		return vocab, raw_vocab, ids_to_tokens, emoji





class  snake_case     (							_UpperCAmelCase   ):
							UpperCAmelCase__						     =      VOCAB_FILES_NAMES
							UpperCAmelCase__						     =      PRETRAINED_VOCAB_FILES_MAP
							UpperCAmelCase__						     =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCAmelCase__						     =      ['input_ids', 'attention_mask']

							def __init__(self       ,       SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_="<|endoftext|>"       ,       SCREAMING_SNAKE_CASE_="<|endoftext|>"       ,       SCREAMING_SNAKE_CASE_="<|startoftext|>"       ,       SCREAMING_SNAKE_CASE_="<|endoftext|>"       ,       SCREAMING_SNAKE_CASE_=False       ,       **SCREAMING_SNAKE_CASE_       ,       ):

									"""simple docstring"""







									super().__init__(
									    unk_token=__UpperCamelCase       ,       pad_token=__UpperCamelCase       ,       bos_token=__UpperCamelCase       ,       eos_token=__UpperCamelCase       ,       do_clean_text=__UpperCamelCase       ,       **__UpperCamelCase       ,       )
									if not os.path.isfile(__UpperCamelCase    ):
											raise ValueError(
											    f'Can\'t find a vocabulary file at path \'{vocab_file}\'. To load the vocabulary from a Google pretrained'
											    ''' model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'''    )
									if not os.path.isfile(__UpperCamelCase    ):
											raise ValueError(
											    f'Can\'t find a emoji file at path \'{emoji_file}\'. To load the emoji information from a Google'
											    ''' pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'''    )
									SCREAMING_SNAKE_CASE_						 = do_clean_text
									SCREAMING_SNAKE_CASE_ ,SCREAMING_SNAKE_CASE_ ,SCREAMING_SNAKE_CASE_ ,SCREAMING_SNAKE_CASE_						 = load_vocab_and_emoji(__UpperCamelCase       ,       __UpperCamelCase    )
									SCREAMING_SNAKE_CASE_						 = SubWordJapaneseTokenizer(
									    vocab=self.vocab       ,       ids_to_tokens=self.ids_to_tokens       ,       emoji=self.emoji    )

							@property
							def _lowercase					(self    ):

									"""simple docstring"""







									return len(self.raw_vocab    )

							def _lowercase					(self    ):

									"""simple docstring"""







									return dict(self.raw_vocab       ,       **self.added_tokens_encoder    )

							def _lowercase					(self       ,       SCREAMING_SNAKE_CASE_    ):

									"""simple docstring"""







									return self.subword_tokenizer.tokenize(__UpperCamelCase       ,       clean=self.do_clean_text    )

							def _lowercase					(self       ,       SCREAMING_SNAKE_CASE_    ):

									"""simple docstring"""







									return self.vocab.get(__UpperCamelCase       ,       self.vocab.get(self.unk_token    )    )

							def _lowercase					(self       ,       SCREAMING_SNAKE_CASE_    ):

									"""simple docstring"""







									return self.subword_tokenizer.convert_id_to_token(__UpperCamelCase    )

							def _lowercase					(self       ,       SCREAMING_SNAKE_CASE_    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = ''''''.join(__UpperCamelCase    ).strip()
									return out_string

							def _lowercase					(self       ,       SCREAMING_SNAKE_CASE_    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = []
									for is_user, text in conversation.iter_texts():
											input_ids.extend(self.encode(__UpperCamelCase       ,       add_special_tokens=__UpperCamelCase    ) + [self.eos_token_id]    )

									if len(__UpperCamelCase    ) > self.model_max_length:
											SCREAMING_SNAKE_CASE_						 = input_ids[-self.model_max_length :]
									return input_ids







							def _lowercase					(self       ,       SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_ = None    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = 0
									if os.path.isdir(__UpperCamelCase    ):
											SCREAMING_SNAKE_CASE_						 = os.path.join(
											    __UpperCamelCase       ,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']    )
											SCREAMING_SNAKE_CASE_						 = os.path.join(
											    __UpperCamelCase       ,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''emoji_file''']    )
									else:
											SCREAMING_SNAKE_CASE_						 = (
											    (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory + VOCAB_FILES_NAMES['''vocab_file''']
											)
											SCREAMING_SNAKE_CASE_						 = (
											    (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory + VOCAB_FILES_NAMES['''emoji_file''']
											)
									with open(__UpperCamelCase       ,       '''w'''       ,       encoding='''utf-8'''    ) as writer:
											for token_index, token in self.ids_to_tokens.items():
													if index != token_index:
															logger.warning(
															    f'Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'
															    ''' Please check that the vocabulary is not corrupted!'''    )
															SCREAMING_SNAKE_CASE_						 = token_index
													writer.write(''','''.join(__UpperCamelCase    ) + '''\n'''    )
													index += 1
									with open(__UpperCamelCase       ,       '''w'''       ,       encoding='''utf-8'''    ) as writer:
											json.dump(self.emoji       ,       __UpperCamelCase    )
									return vocab_file, emoji_file







class  snake_case     (							_UpperCAmelCase   ):

							def __init__(self       ,       SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = vocab  # same as swe
									SCREAMING_SNAKE_CASE_						 = ids_to_tokens  # same as bpe
									SCREAMING_SNAKE_CASE_						 = emoji
									SCREAMING_SNAKE_CASE_						 = np.max([len(__UpperCamelCase    ) for w in self.vocab.keys()]    )
									SCREAMING_SNAKE_CASE_						 = re.compile(R'''(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)'''    )
									SCREAMING_SNAKE_CASE_						 = re.compile(R'''[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*'''    )
									SCREAMING_SNAKE_CASE_						 = re.compile(R'''[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}'''    )
									SCREAMING_SNAKE_CASE_						 = re.compile(
									    R'''([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'''    )
									SCREAMING_SNAKE_CASE_						 = re.compile(
									    R'''(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'''    )
									SCREAMING_SNAKE_CASE_						 = re.compile(
									    R'''((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*'''    )
									SCREAMING_SNAKE_CASE_						 = '''─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿'''
									SCREAMING_SNAKE_CASE_						 = '''▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟'''
									SCREAMING_SNAKE_CASE_						 = str.maketrans({k: '''<BLOCK>''' for k in keisen + blocks}    )

							def __len__(self    ):

									"""simple docstring"""







									return len(self.ids_to_tokens    )

							def _lowercase					(self       ,       SCREAMING_SNAKE_CASE_    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = self.content_repattera.sub('''<URL>'''       ,       __UpperCamelCase    )
									SCREAMING_SNAKE_CASE_						 = self.content_repattera.sub('''<EMAIL>'''       ,       __UpperCamelCase    )
									SCREAMING_SNAKE_CASE_						 = self.content_repattera.sub('''<TEL>'''       ,       __UpperCamelCase    )
									SCREAMING_SNAKE_CASE_						 = self.content_repattera.sub('''<DATE>'''       ,       __UpperCamelCase    )
									SCREAMING_SNAKE_CASE_						 = self.content_repattera.sub('''<DATE>'''       ,       __UpperCamelCase    )
									SCREAMING_SNAKE_CASE_						 = self.content_repattera.sub('''<PRICE>'''       ,       __UpperCamelCase    )
									SCREAMING_SNAKE_CASE_						 = content.translate(self.content_transa    )
									while "<BLOCK><BLOCK>" in content:
											SCREAMING_SNAKE_CASE_						 = content.replace('''<BLOCK><BLOCK>'''       ,       '''<BLOCK>'''    )
									return content

							def _lowercase					(self       ,       SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_=False    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = text.replace(''' '''       ,       '''<SP>'''    )
									SCREAMING_SNAKE_CASE_						 = text.replace('''　'''       ,       '''<SP>'''    )
									SCREAMING_SNAKE_CASE_						 = text.replace('''\r\n'''       ,       '''<BR>'''    )
									SCREAMING_SNAKE_CASE_						 = text.replace('''\n'''       ,       '''<BR>'''    )
									SCREAMING_SNAKE_CASE_						 = text.replace('''\r'''       ,       '''<BR>'''    )
									SCREAMING_SNAKE_CASE_						 = text.replace('''\t'''       ,       '''<TAB>'''    )
									SCREAMING_SNAKE_CASE_						 = text.replace('''—'''       ,       '''ー'''    )
									SCREAMING_SNAKE_CASE_						 = text.replace('''−'''       ,       '''ー'''    )
									for k, v in self.emoji["emoji"].items():
											if k in text:
													SCREAMING_SNAKE_CASE_						 = text.replace(__UpperCamelCase       ,       __UpperCamelCase    )
									if clean:
											SCREAMING_SNAKE_CASE_						 = self.clean_text(__UpperCamelCase    )

									def check_simbol(SCREAMING_SNAKE_CASE_    ):
											SCREAMING_SNAKE_CASE_						 = x.encode()
											if len(__UpperCamelCase    ) == 1 and len(__UpperCamelCase    ) == 2:
													SCREAMING_SNAKE_CASE_						 = (int(e[0]    ) << 8) + int(e[1]    )
													if (
													    (c >= 0XC2A1 and c <= 0XC2BF)
													    or (c >= 0XC780 and c <= 0XC783)
													    or (c >= 0XCAB9 and c <= 0XCBBF)
													    or (c >= 0XCC80 and c <= 0XCDA2)
													):
															return True
											return False

									def checkuae(SCREAMING_SNAKE_CASE_    ):
											SCREAMING_SNAKE_CASE_						 = x.encode()
											if len(__UpperCamelCase    ) == 1 and len(__UpperCamelCase    ) == 3:
													SCREAMING_SNAKE_CASE_						 = (int(e[0]    ) << 16) + (int(e[1]    ) << 8) + int(e[2]    )
													if c >= 0XE28080 and c <= 0XE2B07F:
															return True
											return False

									SCREAMING_SNAKE_CASE_						 = 0
									SCREAMING_SNAKE_CASE_						 = []
									while pos < len(__UpperCamelCase    ):
											SCREAMING_SNAKE_CASE_						 = min(len(__UpperCamelCase    )       ,       pos + self.maxlen + 1    ) if text[pos] == '''<''' else pos + 3
											SCREAMING_SNAKE_CASE_						 = []  # (token_id, token, pos)
											for e in range(__UpperCamelCase       ,       __UpperCamelCase       ,       -1    ):
													SCREAMING_SNAKE_CASE_						 = text[pos:e]
													if wd in self.vocab:
															if wd[0] == "<" and len(__UpperCamelCase    ) > 2:
																	SCREAMING_SNAKE_CASE_						 = [(self.vocab[wd], wd, e)]
																	break
															else:
																	candidates.append((self.vocab[wd], wd, e)    )
											if len(__UpperCamelCase    ) > 0:
													# the smallest token_id is adopted
													SCREAMING_SNAKE_CASE_ ,SCREAMING_SNAKE_CASE_ ,SCREAMING_SNAKE_CASE_						 = sorted(__UpperCamelCase       ,       key=lambda SCREAMING_SNAKE_CASE_    : x[0]    )[0]
													result.append(__UpperCamelCase    )
													SCREAMING_SNAKE_CASE_						 = e
											else:
													SCREAMING_SNAKE_CASE_						 = pos + 1
													SCREAMING_SNAKE_CASE_						 = text[pos:end]
													if check_simbol(__UpperCamelCase    ):
															result.append('''<KIGOU>'''    )
													elif checkuae(__UpperCamelCase    ):
															result.append('''<U2000U2BFF>'''    )
													else:
															for i in wd.encode('''utf-8'''    ):
																	result.append('''<|byte%d|>''' % i    )
													SCREAMING_SNAKE_CASE_						 = end
									return result







							def _lowercase					(self       ,       SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_="\n"    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = []
									SCREAMING_SNAKE_CASE_						 = []
									SCREAMING_SNAKE_CASE_						 = self.ids_to_tokens[index][0]
									if word[:6] == "<|byte" and word[-2:] == "|>":
											byte_tokens.append(int(word[6:-2]    )    )
									else:
											if len(__UpperCamelCase    ) > 0:
													words.append(bytearray(__UpperCamelCase    ).decode('''utf-8'''       ,       errors='''replace'''    )    )
													SCREAMING_SNAKE_CASE_						 = []
											if word[:7] == "<|emoji" and word[-2:] == "|>":
													words.append(self.emoji['''emoji_inv'''][word]    )
											elif word == "<SP>":
													words.append(''' '''    )
											elif word == "<BR>":
													words.append(__UpperCamelCase    )
											elif word == "<TAB>":
													words.append('''\t'''    )
											elif word == "<BLOCK>":
													words.append('''▀'''    )
											elif word == "<KIGOU>":
													words.append('''ǀ'''    )
											elif word == "<U2000U2BFF>":
													words.append('''‖'''    )
											else:
													words.append(__UpperCamelCase    )
									if len(__UpperCamelCase    ) > 0:
											words.append(bytearray(__UpperCamelCase    ).decode('''utf-8'''       ,       errors='''replace'''    )    )
									SCREAMING_SNAKE_CASE_						 = ''''''.join(__UpperCamelCase    )
									return text

"""simple docstring"""




import argparse

from transformers import BigBirdConfig, BigBirdForPreTraining, BigBirdForQuestionAnswering, load_tf_weights_in_big_bird
from transformers.utils import logging


logging.set_verbosity_info()



def        _lowerCamelCase ( __a,							__a,							__a,							__a    ):
		# Initialise PyTorch model
		SCREAMING_SNAKE_CASE_						 = BigBirdConfig.from_json_file(__a    )
		print(F'Building PyTorch model from configuration: {config}'    )

		if is_trivia_qa:
				SCREAMING_SNAKE_CASE_						 = BigBirdForQuestionAnswering(__a    )
		else:
				SCREAMING_SNAKE_CASE_						 = BigBirdForPreTraining(__a    )

		# Load weights from tf checkpoint
		load_tf_weights_in_big_bird(__a,							__a,							is_trivia_qa=__a    )

		# Save pytorch-model
		print(F'Save PyTorch model to {pytorch_dump_path}'    )
		model.save_pretrained(__a    )


if __name__ == "__main__":
						lowerCAmelCase__							=				argparse.ArgumentParser()
						# Required parameters
						parser.add_argument(
						    '--tf_checkpoint_path', default=None, type=str, required=True, help='Path to the TensorFlow checkpoint path.'
						)
						parser.add_argument(
						    '--big_bird_config_file',
						    default=None,
						    type=str,
						    required=True,
						    help=(
						        'The config json file corresponding to the pre-trained BERT model. \n'
						        'This specifies the model architecture.'
						    ),
						)
						parser.add_argument(
						    '--pytorch_dump_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
						)
						parser.add_argument(
						    '--is_trivia_qa', action='store_true', help='Whether to convert a model with a trivia_qa head.'
						)
						lowerCAmelCase__							=				parser.parse_args()
						convert_tf_checkpoint_to_pytorch(
						    args.tf_checkpoint_path, args.big_bird_config_file, args.pytorch_dump_path, args.is_trivia_qa
						)