



'''simple docstring'''




import json
import os
from functools import lru_cache
from typing import Dict, List, Optional, Tuple, Union

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...tokenization_utils_base import BatchEncoding, EncodedInput
from ...utils import PaddingStrategy, logging


a     :    str      						=					logging.get_logger(__name__)


a     :    List[str]      						=					{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

# See all LED models at https://huggingface.co/models?filter=LED
a     :    List[str]      						=					{
    """vocab_file""": {
        """allenai/led-base-16384""": """https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """allenai/led-base-16384""": """https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """allenai/led-base-16384""": """https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json""",
    },
}

a     :    List[Any]      						=					{
    """allenai/led-base-16384""": 1_6_3_8_4,
}



@lru_cache()
# Copied from transformers.models.bart.tokenization_bart.bytes_to_unicode
def    __lowerCamelCase						(      )   ->    List[str]:
							UpperCAmelCase  :	Dict							  =	(
							    list(range(ord("""!"""	)			,       ord("""~"""	) + 1	)	) + list(range(ord("""¡"""	)			,       ord("""¬"""	) + 1	)	) + list(range(ord("""®"""	)			,       ord("""ÿ"""	) + 1	)	)
							)
							UpperCAmelCase  :	Tuple							  =	bs[:]
							UpperCAmelCase  :	List[str]							  =	0
							for b in range(2**8	):
														if b not in bs:
																					bs.append(_lowercase	)
																					cs.append(2**8 + n	)
																					n += 1
							UpperCAmelCase  :	int							  =	[chr(_lowercase	) for n in cs]
							return dict(zip(_lowercase			,       _lowercase	)	)



def    __lowerCamelCase						(      _lowercase	)   ->    Optional[Any]:
							UpperCAmelCase  :	List[Any]							  =	set()
							UpperCAmelCase  :	List[str]							  =	word[0]
							for char in word[1:]:
														pairs.add((prev_char, char)	)
														UpperCAmelCase  :	Any							  =	char
							return pairs


class     UpperCamelCase_				(							__magic_name__    ):
	lowercase    =				VOCAB_FILES_NAMES
	lowercase    =				PRETRAINED_VOCAB_FILES_MAP
	lowercase    =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowercase    =				['input_ids', 'attention_mask']





	def __init__(						self	,	A	,	A	,	A="replace"	,	A="<s>"	,	A="</s>"	,	A="</s>"	,	A="<s>"	,	A="<unk>"	,	A="<pad>"	,	A="<mask>"	,	A=False	,	**A	,	)    ->    Optional[int]:
								UpperCAmelCase  :	Any							  =	AddedToken(A	,	lstrip=A	,	rstrip=A					) if isinstance(A	,	A					) else bos_token
								UpperCAmelCase  :	Optional[Any]							  =	AddedToken(A	,	lstrip=A	,	rstrip=A					) if isinstance(A	,	A					) else eos_token
								UpperCAmelCase  :	Dict							  =	AddedToken(A	,	lstrip=A	,	rstrip=A					) if isinstance(A	,	A					) else sep_token
								UpperCAmelCase  :	Dict							  =	AddedToken(A	,	lstrip=A	,	rstrip=A					) if isinstance(A	,	A					) else cls_token
								UpperCAmelCase  :	List[str]							  =	AddedToken(A	,	lstrip=A	,	rstrip=A					) if isinstance(A	,	A					) else unk_token
								UpperCAmelCase  :	Tuple							  =	AddedToken(A	,	lstrip=A	,	rstrip=A					) if isinstance(A	,	A					) else pad_token

								# Mask token behave like a normal word, i.e. include the space before it
								UpperCAmelCase  :	Dict							  =	AddedToken(A	,	lstrip=A	,	rstrip=A					) if isinstance(A	,	A					) else mask_token

								super().__init__(
								    errors=A	,	bos_token=A	,	eos_token=A	,	unk_token=A	,	sep_token=A	,	cls_token=A	,	pad_token=A	,	mask_token=A	,	add_prefix_space=A	,	**A	,	)

								with open(A	,	encoding="""utf-8"""					) as vocab_handle:
															UpperCAmelCase  :	Union[str, Any]							  =	json.load(A					)
								UpperCAmelCase  :	str							  =	{v: k for k, v in self.encoder.items()}
								UpperCAmelCase  :	Optional[int]							  =	errors  # how to handle errors in decoding
								UpperCAmelCase  :	List[str]							  =	bytes_to_unicode()
								UpperCAmelCase  :	Any							  =	{v: k for k, v in self.byte_encoder.items()}
								with open(A	,	encoding="""utf-8"""					) as merges_handle:
															UpperCAmelCase  :	Dict							  =	merges_handle.read().split("""\n"""					)[1:-1]
								UpperCAmelCase  :	Union[str, Any]							  =	[tuple(merge.split()					) for merge in bpe_merges]
								UpperCAmelCase  :	Optional[int]							  =	dict(zip(A	,	range(len(A					)					)					)					)
								UpperCAmelCase  :	Union[str, Any]							  =	{}
								UpperCAmelCase  :	str							  =	add_prefix_space

								# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
								UpperCAmelCase  :	Optional[Any]							  =	re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""					)





	@property
	# Copied from transformers.models.bart.tokenization_bart.BartTokenizer.vocab_size
	def 				_lowercase(						self					)    ->    int:
								return len(self.encoder					)





	def 				_lowercase(						self					)    ->    Union[str, Any]:
								return dict(self.encoder	,	**self.added_tokens_encoder					)





	def 				_lowercase(						self	,	A					)    ->    int:
								if token in self.cache:
															return self.cache[token]
								UpperCAmelCase  :	List[Any]							  =	tuple(A					)
								UpperCAmelCase  :	List[Any]							  =	get_pairs(A					)

								if not pairs:
															return token

								while True:
															UpperCAmelCase  :	Dict							  =	min(A	,	key=lambda A					: self.bpe_ranks.get(A	,	float("""inf"""					)					)					)
															if bigram not in self.bpe_ranks:
																						break
															UpperCAmelCase  ,  UpperCAmelCase  :	int							  =	bigram
															UpperCAmelCase  :	Tuple							  =	[]
															UpperCAmelCase  :	Dict							  =	0
															while i < len(A					):
																						try:
																													UpperCAmelCase  :	List[Any]							  =	word.index(A	,	A					)
																						except ValueError:
																													new_word.extend(word[i:]					)
																													break
																						else:
																													new_word.extend(word[i:j]					)
																													UpperCAmelCase  :	Union[str, Any]							  =	j

																						if word[i] == first and i < len(A					) - 1 and word[i + 1] == second:
																													new_word.append(first + second					)
																													i += 2
																						else:
																													new_word.append(word[i]					)
																													i += 1
															UpperCAmelCase  :	Dict							  =	tuple(A					)
															UpperCAmelCase  :	Any							  =	new_word
															if len(A					) == 1:
																						break
															else:
																						UpperCAmelCase  :	Optional[int]							  =	get_pairs(A					)
								UpperCAmelCase  :	Optional[int]							  =	""" """.join(A					)
								UpperCAmelCase  :	str							  =	word
								return word





	def 				_lowercase(						self	,	A					)    ->    List[Any]:
								UpperCAmelCase  :	int							  =	[]
								for token in re.findall(self.pat	,	A					):
															UpperCAmelCase  :	int							  =	"""""".join(
															    self.byte_encoder[b] for b in token.encode("""utf-8"""					)					)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
															bpe_tokens.extend(bpe_token for bpe_token in self.bpe(A					).split(""" """					)					)
								return bpe_tokens





	def 				_lowercase(						self	,	A					)    ->    Dict:
								return self.encoder.get(A	,	self.encoder.get(self.unk_token					)					)





	def 				_lowercase(						self	,	A					)    ->    Dict:
								return self.decoder.get(A					)





	def 				_lowercase(						self	,	A					)    ->    Any:
								UpperCAmelCase  :	Union[str, Any]							  =	"""""".join(A					)
								UpperCAmelCase  :	Any							  =	bytearray([self.byte_decoder[c] for c in text]					).decode("""utf-8"""	,	errors=self.errors					)
								return text





	def 				_lowercase(						self	,	A	,	A = None					)    ->    Tuple[str]:
								if not os.path.isdir(A					):
															logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''					)
															return
								UpperCAmelCase  :	Dict							  =	os.path.join(
								    A	,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]					)
								UpperCAmelCase  :	Dict							  =	os.path.join(
								    A	,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]					)

								with open(A	,	"""w"""	,	encoding="""utf-8"""					) as f:
															f.write(json.dumps(self.encoder	,	indent=2	,	sort_keys=A	,	ensure_ascii=A					) + """\n"""					)

								UpperCAmelCase  :	str							  =	0
								with open(A	,	"""w"""	,	encoding="""utf-8"""					) as writer:
															writer.write("""#version: 0.2\n"""					)
															for bpe_tokens, token_index in sorted(self.bpe_ranks.items()	,	key=lambda A					: kv[1]					):
																						if index != token_index:
																													logger.warning(
																													    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																													    """ Please check that the tokenizer is not corrupted!"""					)
																													UpperCAmelCase  :	Optional[Any]							  =	token_index
																						writer.write(""" """.join(A					) + """\n"""					)
																						index += 1

								return vocab_file, merge_file





	def 				_lowercase(						self	,	A	,	A = None					)    ->    List[int]:
								if token_ids_a is None:
															return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
								UpperCAmelCase  :	Tuple							  =	[self.cls_token_id]
								UpperCAmelCase  :	Any							  =	[self.sep_token_id]
								return cls + token_ids_a + sep + sep + token_ids_a + sep





	def 				_lowercase(						self	,	A	,	A = None	,	A = False					)    ->    List[int]:
								if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=A	,	token_ids_a=A	,	already_has_special_tokens=A					)

								if token_ids_a is None:
															return [1] + ([0] * len(A					)) + [1]
								return [1] + ([0] * len(A					)) + [1, 1] + ([0] * len(A					)) + [1]





	def 				_lowercase(						self	,	A	,	A = None					)    ->    List[int]:
								UpperCAmelCase  :	List[Any]							  =	[self.sep_token_id]
								UpperCAmelCase  :	Any							  =	[self.cls_token_id]

								if token_ids_a is None:
															return len(cls + token_ids_a + sep					) * [0]
								return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]





	def 				_lowercase(						self	,	A	,	A=False	,	**A					)    ->    str:
								UpperCAmelCase  :	Tuple							  =	kwargs.pop("""add_prefix_space"""	,	self.add_prefix_space					)
								if (is_split_into_words or add_prefix_space) and (len(A					) > 0 and not text[0].isspace()):
															UpperCAmelCase  :	List[str]							  =	""" """ + text
								return (text, kwargs)






	def 				_lowercase(						self	,	A	,	A = None	,	A = PaddingStrategy.DO_NOT_PAD	,	A = None	,	A = None	,	)    ->    dict:
								UpperCAmelCase  :	Union[str, Any]							  =	super()._pad(
								    encoded_inputs=A	,	max_length=A	,	padding_strategy=A	,	pad_to_multiple_of=A	,	return_attention_mask=A	,	)

								# Load from model defaults
								if return_attention_mask is None:
															UpperCAmelCase  :	List[Any]							  =	"""attention_mask""" in self.model_input_names

								if return_attention_mask and "global_attention_mask" in encoded_inputs:
															UpperCAmelCase  :	Optional[Any]							  =	encoded_inputs[self.model_input_names[0]]
															# `global_attention_mask` need to have the same length as other (sequential) inputs.
															UpperCAmelCase  :	Dict							  =	len(encoded_inputs["""global_attention_mask"""]					) != len(A					)

															if needs_to_be_padded:
																						UpperCAmelCase  :	Dict							  =	len(A					) - len(encoded_inputs["""global_attention_mask"""]					)

																						if self.padding_side == "right":
																													# Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
																													UpperCAmelCase  :	int							  =	(
																													    encoded_inputs["""global_attention_mask"""] + [-1] * difference
																													)
																						elif self.padding_side == "left":
																													UpperCAmelCase  :	List[Any]							  =	[-1] * difference + encoded_inputs[
																													    """global_attention_mask"""
																													]
																						else:
																													raise ValueError("""Invalid padding strategy:""" + str(self.padding_side					)					)

								return encoded_inputs






'''simple docstring'''




from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tensorflow_text_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


a     :    Optional[Any]      						=					{
    """configuration_bert""": ["""BERT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """BertConfig""", """BertOnnxConfig"""],
    """tokenization_bert""": ["""BasicTokenizer""", """BertTokenizer""", """WordpieceTokenizer"""],
}

try:
							if not is_tokenizers_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							a     :    Dict      						=					["""BertTokenizerFast"""]

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							a     :    List[Any]      						=					[
							    """BERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
							    """BertForMaskedLM""",
							    """BertForMultipleChoice""",
							    """BertForNextSentencePrediction""",
							    """BertForPreTraining""",
							    """BertForQuestionAnswering""",
							    """BertForSequenceClassification""",
							    """BertForTokenClassification""",
							    """BertLayer""",
							    """BertLMHeadModel""",
							    """BertModel""",
							    """BertPreTrainedModel""",
							    """load_tf_weights_in_bert""",
							]

try:
							if not is_tf_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							a     :    List[str]      						=					[
							    """TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
							    """TFBertEmbeddings""",
							    """TFBertForMaskedLM""",
							    """TFBertForMultipleChoice""",
							    """TFBertForNextSentencePrediction""",
							    """TFBertForPreTraining""",
							    """TFBertForQuestionAnswering""",
							    """TFBertForSequenceClassification""",
							    """TFBertForTokenClassification""",
							    """TFBertLMHeadModel""",
							    """TFBertMainLayer""",
							    """TFBertModel""",
							    """TFBertPreTrainedModel""",
							]
try:
							if not is_tensorflow_text_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							a     :    int      						=					["""TFBertTokenizer"""]

try:
							if not is_flax_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							a     :    Optional[Any]      						=					[
							    """FlaxBertForCausalLM""",
							    """FlaxBertForMaskedLM""",
							    """FlaxBertForMultipleChoice""",
							    """FlaxBertForNextSentencePrediction""",
							    """FlaxBertForPreTraining""",
							    """FlaxBertForQuestionAnswering""",
							    """FlaxBertForSequenceClassification""",
							    """FlaxBertForTokenClassification""",
							    """FlaxBertModel""",
							    """FlaxBertPreTrainedModel""",
							]

if TYPE_CHECKING:
							from .configuration_bert import BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, BertConfig, BertOnnxConfig
							from .tokenization_bert import BasicTokenizer, BertTokenizer, WordpieceTokenizer

							try:
														if not is_tokenizers_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .tokenization_bert_fast import BertTokenizerFast

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_bert import (
														    BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
														    BertForMaskedLM,
														    BertForMultipleChoice,
														    BertForNextSentencePrediction,
														    BertForPreTraining,
														    BertForQuestionAnswering,
														    BertForSequenceClassification,
														    BertForTokenClassification,
														    BertLayer,
														    BertLMHeadModel,
														    BertModel,
														    BertPreTrainedModel,
														    load_tf_weights_in_bert,
														)

							try:
														if not is_tf_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_tf_bert import (
														    TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
														    TFBertEmbeddings,
														    TFBertForMaskedLM,
														    TFBertForMultipleChoice,
														    TFBertForNextSentencePrediction,
														    TFBertForPreTraining,
														    TFBertForQuestionAnswering,
														    TFBertForSequenceClassification,
														    TFBertForTokenClassification,
														    TFBertLMHeadModel,
														    TFBertMainLayer,
														    TFBertModel,
														    TFBertPreTrainedModel,
														)

							try:
														if not is_tensorflow_text_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .tokenization_bert_tf import TFBertTokenizer

							try:
														if not is_flax_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_flax_bert import (
														    FlaxBertForCausalLM,
														    FlaxBertForMaskedLM,
														    FlaxBertForMultipleChoice,
														    FlaxBertForNextSentencePrediction,
														    FlaxBertForPreTraining,
														    FlaxBertForQuestionAnswering,
														    FlaxBertForSequenceClassification,
														    FlaxBertForTokenClassification,
														    FlaxBertModel,
														    FlaxBertPreTrainedModel,
														)

else:
							import sys

							a     :    Optional[int]      						=					_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)

