





"""simple docstring"""





def      _snake_case						(						_snake_case				:			list[int]			):
      if not nums:  # Makes sure that the list is not empty
            raise ValueError('''List is empty'''			)

      lowerCAmelCase					:	int       =   sum(_snake_case			) / len(_snake_case			)  # Calculate the average
      return sum(abs(x - average			) for x in nums			) / len(_snake_case			)


if __name__ == "__main__":
 import doctest

 doctest.testmod()






"""simple docstring"""





import json
import os
from functools import lru_cache
from typing import Dict, List, Optional, Tuple, Union

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...tokenization_utils_base import BatchEncoding, EncodedInput
from ...utils import PaddingStrategy, logging


snake_case__		:     Optional[Any]    				=				logging.get_logger(__name__)


snake_case__		:     Any    				=				{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt'''}

# See all LED models at https://huggingface.co/models?filter=LED
snake_case__		:     Optional[Any]    				=				{
    '''vocab_file''': {
        '''allenai/led-base-16384''': '''https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''allenai/led-base-16384''': '''https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''allenai/led-base-16384''': '''https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json''',
    },
}

snake_case__		:     List[Any]    				=				{
    '''allenai/led-base-16384''': 16_384,
}


@lru_cache()
# Copied from transformers.models.bart.tokenization_bart.bytes_to_unicode
def      _snake_case						(						):
      lowerCAmelCase					:	Optional[int]       =   (
          list(range(ord('''!'''			) ,   ord('''~'''			) + 1			)			) + list(range(ord('''¡'''			) ,   ord('''¬'''			) + 1			)			) + list(range(ord('''®'''			) ,   ord('''ÿ'''			) + 1			)			)
      )
      lowerCAmelCase					:	str       =   bs[:]
      lowerCAmelCase					:	Optional[int]       =   0
      for b in range(2**8			):
            if b not in bs:
                  bs.append(_snake_case			)
                  cs.append(2**8 + n			)
                  n += 1
      lowerCAmelCase					:	int       =   [chr(_snake_case			) for n in cs]
      return dict(zip(_snake_case ,   _snake_case			)			)


def      _snake_case						(						_snake_case				:			List[Any]			):
      lowerCAmelCase					:	List[str]       =   set()
      lowerCAmelCase					:	Any       =   word[0]
      for char in word[1:]:
            pairs.add((prev_char, char)			)
            lowerCAmelCase					:	Optional[Any]       =   char
      return pairs






class 		snake_case_(       a__			):
     __UpperCamelCase  =					VOCAB_FILES_NAMES
     __UpperCamelCase  =					PRETRAINED_VOCAB_FILES_MAP
     __UpperCamelCase  =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __UpperCamelCase  =					['''input_ids''', '''attention_mask''']






     def __init__(	self  :					Tuple					,				UpperCamelCase_  :					Optional[Any]					,				UpperCamelCase_  :					Dict					,				UpperCamelCase_  :					Tuple="replace"					,				UpperCamelCase_  :					Union[str, Any]="<s>"					,				UpperCamelCase_  :					List[str]="</s>"					,				UpperCamelCase_  :					str="</s>"					,				UpperCamelCase_  :					int="<s>"					,				UpperCamelCase_  :					int="<unk>"					,				UpperCamelCase_  :					Union[str, Any]="<pad>"					,				UpperCamelCase_  :					Tuple="<mask>"					,				UpperCamelCase_  :					Optional[int]=False					,				**UpperCamelCase_  :					Tuple					,				):
           lowerCAmelCase					:	Any       =   AddedToken(UpperCamelCase_					,				lstrip=UpperCamelCase_					,				rstrip=UpperCamelCase_				) if isinstance(UpperCamelCase_					,				UpperCamelCase_				) else bos_token
           lowerCAmelCase					:	Union[str, Any]       =   AddedToken(UpperCamelCase_					,				lstrip=UpperCamelCase_					,				rstrip=UpperCamelCase_				) if isinstance(UpperCamelCase_					,				UpperCamelCase_				) else eos_token
           lowerCAmelCase					:	Optional[int]       =   AddedToken(UpperCamelCase_					,				lstrip=UpperCamelCase_					,				rstrip=UpperCamelCase_				) if isinstance(UpperCamelCase_					,				UpperCamelCase_				) else sep_token
           lowerCAmelCase					:	int       =   AddedToken(UpperCamelCase_					,				lstrip=UpperCamelCase_					,				rstrip=UpperCamelCase_				) if isinstance(UpperCamelCase_					,				UpperCamelCase_				) else cls_token
           lowerCAmelCase					:	Tuple       =   AddedToken(UpperCamelCase_					,				lstrip=UpperCamelCase_					,				rstrip=UpperCamelCase_				) if isinstance(UpperCamelCase_					,				UpperCamelCase_				) else unk_token
           lowerCAmelCase					:	List[Any]       =   AddedToken(UpperCamelCase_					,				lstrip=UpperCamelCase_					,				rstrip=UpperCamelCase_				) if isinstance(UpperCamelCase_					,				UpperCamelCase_				) else pad_token

           # Mask token behave like a normal word, i.e. include the space before it
           lowerCAmelCase					:	Tuple       =   AddedToken(UpperCamelCase_					,				lstrip=UpperCamelCase_					,				rstrip=UpperCamelCase_				) if isinstance(UpperCamelCase_					,				UpperCamelCase_				) else mask_token

           super().__init__(
               errors=UpperCamelCase_					,				bos_token=UpperCamelCase_					,				eos_token=UpperCamelCase_					,				unk_token=UpperCamelCase_					,				sep_token=UpperCamelCase_					,				cls_token=UpperCamelCase_					,				pad_token=UpperCamelCase_					,				mask_token=UpperCamelCase_					,				add_prefix_space=UpperCamelCase_					,				**UpperCamelCase_					,				)

           with open(UpperCamelCase_					,				encoding='''utf-8'''				) as vocab_handle:
                 lowerCAmelCase					:	Any       =   json.load(UpperCamelCase_				)
           lowerCAmelCase					:	Dict       =   {v: k for k, v in self.encoder.items()}
           lowerCAmelCase					:	Optional[int]       =   errors  # how to handle errors in decoding
           lowerCAmelCase					:	List[Any]       =   bytes_to_unicode()
           lowerCAmelCase					:	Optional[Any]       =   {v: k for k, v in self.byte_encoder.items()}
           with open(UpperCamelCase_					,				encoding='''utf-8'''				) as merges_handle:
                 lowerCAmelCase					:	Optional[int]       =   merges_handle.read().split('''\n'''				)[1:-1]
           lowerCAmelCase					:	Optional[int]       =   [tuple(merge.split()				) for merge in bpe_merges]
           lowerCAmelCase					:	Optional[int]       =   dict(zip(UpperCamelCase_					,				range(len(UpperCamelCase_				)				)				)				)
           lowerCAmelCase					:	List[Any]       =   {}
           lowerCAmelCase					:	Optional[Any]       =   add_prefix_space

           # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
           lowerCAmelCase					:	Dict       =   re.compile(r'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''				)






     @property
     # Copied from transformers.models.bart.tokenization_bart.BartTokenizer.vocab_size
     def      lowerCamelCase__     (	self  :					Union[str, Any]				):
           return len(self.encoder				)






     def      lowerCamelCase__     (	self  :					Union[str, Any]				):
           return dict(self.encoder					,				**self.added_tokens_encoder				)






     def      lowerCamelCase__     (	self  :					Any					,				UpperCamelCase_  :					int				):
           if token in self.cache:
                 return self.cache[token]
           lowerCAmelCase					:	List[str]       =   tuple(UpperCamelCase_				)
           lowerCAmelCase					:	Optional[Any]       =   get_pairs(UpperCamelCase_				)

           if not pairs:
                 return token

           while True:
                 lowerCAmelCase					:	List[Any]       =   min(UpperCamelCase_					,				key=lambda UpperCamelCase_				: self.bpe_ranks.get(UpperCamelCase_					,				float('''inf'''				)				)				)
                 if bigram not in self.bpe_ranks:
                       break
                 lowerCAmelCase,      lowerCAmelCase					:	Any       =   bigram
                 lowerCAmelCase					:	Tuple       =   []
                 lowerCAmelCase					:	Any       =   0
                 while i < len(UpperCamelCase_				):
                       try:
                             lowerCAmelCase					:	int       =   word.index(UpperCamelCase_					,				UpperCamelCase_				)
                       except ValueError:
                             new_word.extend(word[i:]				)
                             break
                       else:
                             new_word.extend(word[i:j]				)
                             lowerCAmelCase					:	int       =   j

                       if word[i] == first and i < len(UpperCamelCase_				) - 1 and word[i + 1] == second:
                             new_word.append(first + second				)
                             i += 2
                       else:
                             new_word.append(word[i]				)
                             i += 1
                 lowerCAmelCase					:	Tuple       =   tuple(UpperCamelCase_				)
                 lowerCAmelCase					:	Tuple       =   new_word
                 if len(UpperCamelCase_				) == 1:
                       break
                 else:
                       lowerCAmelCase					:	Optional[Any]       =   get_pairs(UpperCamelCase_				)
           lowerCAmelCase					:	Union[str, Any]       =   ''' '''.join(UpperCamelCase_				)
           lowerCAmelCase					:	List[str]       =   word
           return word






     def      lowerCamelCase__     (	self  :					Optional[Any]					,				UpperCamelCase_  :					Tuple				):
           lowerCAmelCase					:	Dict       =   []
           for token in re.findall(self.pat					,				UpperCamelCase_				):
                 lowerCAmelCase					:	Union[str, Any]       =   ''''''.join(
                     self.byte_encoder[b] for b in token.encode('''utf-8'''				)				)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
                 bpe_tokens.extend(bpe_token for bpe_token in self.bpe(UpperCamelCase_				).split(''' '''				)				)
           return bpe_tokens






     def      lowerCamelCase__     (	self  :					int					,				UpperCamelCase_  :					str				):
           return self.encoder.get(UpperCamelCase_					,				self.encoder.get(self.unk_token				)				)






     def      lowerCamelCase__     (	self  :					Any					,				UpperCamelCase_  :					Union[str, Any]				):
           return self.decoder.get(UpperCamelCase_				)






     def      lowerCamelCase__     (	self  :					Any					,				UpperCamelCase_  :					List[str]				):
           lowerCAmelCase					:	Optional[int]       =   ''''''.join(UpperCamelCase_				)
           lowerCAmelCase					:	Optional[int]       =   bytearray([self.byte_decoder[c] for c in text]				).decode('''utf-8'''					,				errors=self.errors				)
           return text






     def      lowerCamelCase__     (	self  :					str					,				UpperCamelCase_  :					str					,				UpperCamelCase_  :					Optional[str] = None				):
           if not os.path.isdir(UpperCamelCase_				):
                 logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''				)
                 return
           lowerCAmelCase					:	int       =   os.path.join(
               UpperCamelCase_					,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)
           lowerCAmelCase					:	Optional[Any]       =   os.path.join(
               UpperCamelCase_					,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']				)

           with open(UpperCamelCase_					,				'''w'''					,				encoding='''utf-8'''				) as f:
                 f.write(json.dumps(self.encoder					,				indent=2					,				sort_keys=UpperCamelCase_					,				ensure_ascii=UpperCamelCase_				) + '''\n'''				)

           lowerCAmelCase					:	Optional[int]       =   0
           with open(UpperCamelCase_					,				'''w'''					,				encoding='''utf-8'''				) as writer:
                 writer.write('''#version: 0.2\n'''				)
                 for bpe_tokens, token_index in sorted(self.bpe_ranks.items()					,				key=lambda UpperCamelCase_				: kv[1]				):
                       if index != token_index:
                             logger.warning(
                                 F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                                 ''' Please check that the tokenizer is not corrupted!'''				)
                             lowerCAmelCase					:	Tuple       =   token_index
                       writer.write(''' '''.join(UpperCamelCase_				) + '''\n'''				)
                       index += 1

           return vocab_file, merge_file






     def      lowerCamelCase__     (	self  :					Optional[Any]					,				UpperCamelCase_  :					List[int]					,				UpperCamelCase_  :					Optional[List[int]] = None				):
           if token_ids_a is None:
                 return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
           lowerCAmelCase					:	Any       =   [self.cls_token_id]
           lowerCAmelCase					:	str       =   [self.sep_token_id]
           return cls + token_ids_a + sep + sep + token_ids_a + sep






     def      lowerCamelCase__     (	self  :					Any					,				UpperCamelCase_  :					List[int]					,				UpperCamelCase_  :					Optional[List[int]] = None					,				UpperCamelCase_  :					bool = False				):
           if already_has_special_tokens:
                 return super().get_special_tokens_mask(
                     token_ids_a=UpperCamelCase_					,				token_ids_a=UpperCamelCase_					,				already_has_special_tokens=UpperCamelCase_				)

           if token_ids_a is None:
                 return [1] + ([0] * len(UpperCamelCase_				)) + [1]
           return [1] + ([0] * len(UpperCamelCase_				)) + [1, 1] + ([0] * len(UpperCamelCase_				)) + [1]






     def      lowerCamelCase__     (	self  :					List[str]					,				UpperCamelCase_  :					List[int]					,				UpperCamelCase_  :					Optional[List[int]] = None				):
           lowerCAmelCase					:	Optional[Any]       =   [self.sep_token_id]
           lowerCAmelCase					:	Optional[int]       =   [self.cls_token_id]

           if token_ids_a is None:
                 return len(cls + token_ids_a + sep				) * [0]
           return len(cls + token_ids_a + sep + sep + token_ids_a + sep				) * [0]






     def      lowerCamelCase__     (	self  :					int					,				UpperCamelCase_  :					Any					,				UpperCamelCase_  :					Dict=False					,				**UpperCamelCase_  :					Tuple				):
           lowerCAmelCase					:	Union[str, Any]       =   kwargs.pop('''add_prefix_space'''					,				self.add_prefix_space				)
           if (is_split_into_words or add_prefix_space) and (len(UpperCamelCase_				) > 0 and not text[0].isspace()):
                 lowerCAmelCase					:	List[Any]       =   ''' ''' + text
           return (text, kwargs)






     def      lowerCamelCase__     (	self  :					str					,				UpperCamelCase_  :					Union[Dict[str, EncodedInput], BatchEncoding]					,				UpperCamelCase_  :					Optional[int] = None					,				UpperCamelCase_  :					PaddingStrategy = PaddingStrategy.DO_NOT_PAD					,				UpperCamelCase_  :					Optional[int] = None					,				UpperCamelCase_  :					Optional[bool] = None					,				):
           lowerCAmelCase					:	Dict       =   super()._pad(
               encoded_inputs=UpperCamelCase_					,				max_length=UpperCamelCase_					,				padding_strategy=UpperCamelCase_					,				pad_to_multiple_of=UpperCamelCase_					,				return_attention_mask=UpperCamelCase_					,				)

           # Load from model defaults
           if return_attention_mask is None:
                 lowerCAmelCase					:	Tuple       =   '''attention_mask''' in self.model_input_names

           if return_attention_mask and "global_attention_mask" in encoded_inputs:
                 lowerCAmelCase					:	Dict       =   encoded_inputs[self.model_input_names[0]]
                 # `global_attention_mask` need to have the same length as other (sequential) inputs.
                 lowerCAmelCase					:	List[Any]       =   len(encoded_inputs['''global_attention_mask''']				) != len(UpperCamelCase_				)

                 if needs_to_be_padded:
                       lowerCAmelCase					:	int       =   len(UpperCamelCase_				) - len(encoded_inputs['''global_attention_mask''']				)

                       if self.padding_side == "right":
                             # Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
                             lowerCAmelCase					:	Dict       =   (
                                 encoded_inputs['''global_attention_mask'''] + [-1] * difference
                             )
                       elif self.padding_side == "left":
                             lowerCAmelCase					:	int       =   [-1] * difference + encoded_inputs[
                                 '''global_attention_mask'''
                             ]
                       else:
                             raise ValueError('''Invalid padding strategy:''' + str(self.padding_side				)				)

           return encoded_inputs


