





'''simple docstring'''







import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
						if is_torch_available():
												import torch
						if is_tf_available():
												import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


snake_case_     :       Optional[Any]						=  logging.get_logger(__name__)

snake_case_     :       List[str]						=  {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

snake_case_     :       str						=  {
    "vocab_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json",
    },
    "merges_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "Salesforce/codegen-350M-mono": (
            "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json"
        ),
    },
}

snake_case_     :       List[Any]						=  {
    "Salesforce/codegen-350M-mono": 20_48,
}


class 					__a      (__A     ):
							__a					:   Optional[int]									=       VOCAB_FILES_NAMES
							__a					:   Optional[int]									=       PRETRAINED_VOCAB_FILES_MAP
							__a					:   str									=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__a					:   int									=       ["input_ids", "attention_mask"]
							__a					:   List[Any]									=       CodeGenTokenizer

							def __init__(							self							:     Any       ,     __magic_name__							:     Union[str, Any]=None       ,     __magic_name__							:     Any=None       ,     __magic_name__							:     Union[str, Any]=None       ,     __magic_name__							:     str="<|endoftext|>"       ,     __magic_name__							:     Dict="<|endoftext|>"       ,     __magic_name__							:     Any="<|endoftext|>"       ,     __magic_name__							:     Optional[int]=False       ,     **__magic_name__							:     Union[str, Any]       ,     )					->       List[Any]:







													"""simple docstring"""
													super().__init__(
													    __magic_name__       ,     __magic_name__       ,     tokenizer_file=__magic_name__       ,     unk_token=__magic_name__       ,     bos_token=__magic_name__       ,     eos_token=__magic_name__       ,     add_prefix_space=__magic_name__       ,     **__magic_name__       ,     )

													if kwargs.pop('''add_bos_token'''       ,     __magic_name__   ):
																			UpperCAmelCase_			:      List[str]     						=  kwargs.pop('''name_or_path'''       ,     ''''''   )
																			raise ValueError(
																			    '''Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token.'''
																			    '''Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n'''
																			    F"""`CodeGenTokenizer.from_pretrained(\'{model_id}\')`\nor\n"""
																			    F"""`AutoTokenizer.from_pretrained(\'{model_id}\', use_fast=False)`\n"""
																			    '''This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005.'''
																			    ''' so that the fast tokenizer works correctly.'''   )

													UpperCAmelCase_			:      List[str]     						=  json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()   )
													if pre_tok_state.get('''add_prefix_space'''       ,     __magic_name__   ) != add_prefix_space:
																			UpperCAmelCase_			:      List[Any]     						=  getattr(__magic_name__       ,     pre_tok_state.pop('''type'''   )   )
																			UpperCAmelCase_			:      Tuple     						=  add_prefix_space
																			UpperCAmelCase_			:      Union[str, Any]     						=  pre_tok_class(**__magic_name__   )

													UpperCAmelCase_			:      str     						=  add_prefix_space

							def     UpperCAmelCase__	(							self							:     Tuple       ,     *__magic_name__							:     Optional[int]       ,     **__magic_name__							:     str   )					->       BatchEncoding:







													"""simple docstring"""
													UpperCAmelCase_			:      List[str]     						=  kwargs.get('''is_split_into_words'''       ,     __magic_name__   )
													assert self.add_prefix_space or not is_split_into_words, (
													    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
													    "to use it with pretokenized inputs."
													)

													return super()._batch_encode_plus(*__magic_name__       ,     **__magic_name__   )

							def     UpperCAmelCase__	(							self							:     Optional[int]       ,     *__magic_name__							:     Any       ,     **__magic_name__							:     str   )					->       BatchEncoding:







													"""simple docstring"""
													UpperCAmelCase_			:      str     						=  kwargs.get('''is_split_into_words'''       ,     __magic_name__   )

													assert self.add_prefix_space or not is_split_into_words, (
													    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
													    "to use it with pretokenized inputs."
													)

													return super()._encode_plus(*__magic_name__       ,     **__magic_name__   )

							def     UpperCAmelCase__	(							self							:     int       ,     __magic_name__							:     str       ,     __magic_name__							:     Optional[str] = None   )					->       Tuple[str]:







													"""simple docstring"""
													UpperCAmelCase_			:      Union[str, Any]     						=  self._tokenizer.model.save(__magic_name__       ,     name=__magic_name__   )
													return tuple(__magic_name__   )

							def     UpperCAmelCase__	(							self							:     Optional[int]       ,     __magic_name__							:     Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]       ,     __magic_name__							:     bool = False       ,     __magic_name__							:     bool = None       ,     __magic_name__							:     Optional[List[str]] = None       ,     **__magic_name__							:     Dict       ,     )					->       str:







													"""simple docstring"""
													UpperCAmelCase_			:      str     						=  super().decode(
													    token_ids=__magic_name__       ,     skip_special_tokens=__magic_name__       ,     clean_up_tokenization_spaces=__magic_name__       ,     **__magic_name__       ,     )

													if truncate_before_pattern is not None and len(__magic_name__   ) > 0:
																			UpperCAmelCase_			:      List[str]     						=  self.truncate(__magic_name__       ,     __magic_name__   )

													return decoded_text



							def     UpperCAmelCase__	(							self							:     int       ,     __magic_name__							:     Tuple       ,     __magic_name__							:     List[Any]   )					->       Optional[int]:







													"""simple docstring"""

													def find_re(__magic_name__							:     Dict       ,     __magic_name__							:     Optional[int]       ,     __magic_name__							:     int   ):
																			UpperCAmelCase_			:      Union[str, Any]     						=  pattern.search(__magic_name__       ,     __magic_name__   )
																			return m.start() if m else -1

													UpperCAmelCase_			:      str     						=  [re.compile(__magic_name__       ,     re.MULTILINE   ) for pattern in truncate_before_pattern]

													UpperCAmelCase_			:      Any     						=  list(re.finditer('''^print'''       ,     __magic_name__       ,     re.MULTILINE   )   )

													if len(__magic_name__   ) > 1:
																			UpperCAmelCase_			:      List[str]     						=  completion[: prints[1].start()]

													UpperCAmelCase_			:      Optional[int]     						=  list(re.finditer('''^def'''       ,     __magic_name__       ,     re.MULTILINE   )   )

													if len(__magic_name__   ) > 1:
																			UpperCAmelCase_			:      Optional[Any]     						=  completion[: defs[1].start()]

													UpperCAmelCase_			:      Any     						=  0

													UpperCAmelCase_			:      int     						=  [
													    pos for pos in [find_re(__magic_name__       ,     __magic_name__       ,     __magic_name__   ) for terminal in terminals] if pos != -1
													]

													if len(__magic_name__   ) > 0:
																			return completion[: min(__magic_name__   )]
													else:
																			return completion












'''simple docstring'''







import unittest

import torch
from torch import nn

from diffusers.models.activations import get_activation


class 					__a      (unittest.TestCase     ):

							def     UpperCAmelCase__	(							self							:     Dict   )					->       Dict:







													"""simple docstring"""
													UpperCAmelCase_			:      Dict     						=  get_activation('''swish'''   )

													self.assertIsInstance(__magic_name__       ,     nn.SiLU   )

													self.assertEqual(act(torch.tensor(-1_00       ,     dtype=torch.floataa   )   ).item()       ,     0   )
													self.assertNotEqual(act(torch.tensor(-1       ,     dtype=torch.floataa   )   ).item()       ,     0   )
													self.assertEqual(act(torch.tensor(0       ,     dtype=torch.floataa   )   ).item()       ,     0   )
													self.assertEqual(act(torch.tensor(20       ,     dtype=torch.floataa   )   ).item()       ,     20   )

							def     UpperCAmelCase__	(							self							:     Tuple   )					->       Tuple:







													"""simple docstring"""
													UpperCAmelCase_			:      Union[str, Any]     						=  get_activation('''silu'''   )

													self.assertIsInstance(__magic_name__       ,     nn.SiLU   )

													self.assertEqual(act(torch.tensor(-1_00       ,     dtype=torch.floataa   )   ).item()       ,     0   )
													self.assertNotEqual(act(torch.tensor(-1       ,     dtype=torch.floataa   )   ).item()       ,     0   )
													self.assertEqual(act(torch.tensor(0       ,     dtype=torch.floataa   )   ).item()       ,     0   )
													self.assertEqual(act(torch.tensor(20       ,     dtype=torch.floataa   )   ).item()       ,     20   )

							def     UpperCAmelCase__	(							self							:     Tuple   )					->       Optional[int]:







													"""simple docstring"""
													UpperCAmelCase_			:      Optional[int]     						=  get_activation('''mish'''   )

													self.assertIsInstance(__magic_name__       ,     nn.Mish   )

													self.assertEqual(act(torch.tensor(-2_00       ,     dtype=torch.floataa   )   ).item()       ,     0   )
													self.assertNotEqual(act(torch.tensor(-1       ,     dtype=torch.floataa   )   ).item()       ,     0   )
													self.assertEqual(act(torch.tensor(0       ,     dtype=torch.floataa   )   ).item()       ,     0   )
													self.assertEqual(act(torch.tensor(20       ,     dtype=torch.floataa   )   ).item()       ,     20   )



							def     UpperCAmelCase__	(							self							:     str   )					->       Optional[Any]:







													"""simple docstring"""
													UpperCAmelCase_			:      List[Any]     						=  get_activation('''gelu'''   )

													self.assertIsInstance(__magic_name__       ,     nn.GELU   )

													self.assertEqual(act(torch.tensor(-1_00       ,     dtype=torch.floataa   )   ).item()       ,     0   )
													self.assertNotEqual(act(torch.tensor(-1       ,     dtype=torch.floataa   )   ).item()       ,     0   )
													self.assertEqual(act(torch.tensor(0       ,     dtype=torch.floataa   )   ).item()       ,     0   )
													self.assertEqual(act(torch.tensor(20       ,     dtype=torch.floataa   )   ).item()       ,     20   )





