





import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class   __lowerCAmelCase			(					_UpperCamelCase		,unittest.TestCase  ):
   _UpperCamelCase :						Optional[Any]     		=				CodeGenTokenizer
   _UpperCamelCase :						str     		=				CodeGenTokenizerFast
   _UpperCamelCase :						Any     		=				True
   _UpperCamelCase :						int     		=				{"""add_prefix_space""": True}
   _UpperCamelCase :						List[Any]     		=				False





   def 	_snake_case ( self      )				->  int:
      """simple docstring"""





      super().setUp()

      # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
      a__	:						str      		=  [
          "l",
          "o",
          "w",
          "e",
          "r",
          "s",
          "t",
          "i",
          "d",
          "n",
          "\u0120",
          "\u0120l",
          "\u0120n",
          "\u0120lo",
          "\u0120low",
          "er",
          "\u0120lowest",
          "\u0120newer",
          "\u0120wider",
          "<unk>",
          "<|endoftext|>",
      ]
      a__	:						List[Any]      		=  dict(zip(snake_case					,			range(len(snake_case      )      )      )      )
      a__	:						Tuple      		=  ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
      a__	:						Optional[int]      		=  {"unk_token": "<unk>"}

      a__	:						str      		=  os.path.join(self.tmpdirname					,			VOCAB_FILES_NAMES["vocab_file"]      )
      a__	:						Dict      		=  os.path.join(self.tmpdirname					,			VOCAB_FILES_NAMES["merges_file"]      )
      with open(self.vocab_file					,			"w"					,			encoding="utf-8"      ) as fp:
         fp.write(json.dumps(snake_case      ) + "\n"      )
      with open(self.merges_file					,			"w"					,			encoding="utf-8"      ) as fp:
         fp.write("\n".join(snake_case      )      )





   def 	_snake_case ( self					,			**snake_case      )				->  Optional[Any]:
      """simple docstring"""





      kwargs.update(self.special_tokens_map      )
      return CodeGenTokenizer.from_pretrained(self.tmpdirname					,			**snake_case      )





   def 	_snake_case ( self					,			**snake_case      )				->  str:
      """simple docstring"""





      kwargs.update(self.special_tokens_map      )
      return CodeGenTokenizerFast.from_pretrained(self.tmpdirname					,			**snake_case      )





   def 	_snake_case ( self					,			snake_case      )				->  Optional[int]:
      """simple docstring"""





      a__	:						Dict      		=  "lower newer"
      a__	:						Optional[Any]      		=  "lower newer"
      return input_text, output_text





   def 	_snake_case ( self      )				->  Union[str, Any]:
      """simple docstring"""





      a__	:						Tuple      		=  CodeGenTokenizer(self.vocab_file					,			self.merges_file					,			**self.special_tokens_map      )
      a__	:						Union[str, Any]      		=  "lower newer"
      a__	:						Dict      		=  ["\u0120low", "er", "\u0120", "n", "e", "w", "er"]
      a__	:						Dict      		=  tokenizer.tokenize(snake_case					,			add_prefix_space=snake_case      )
      self.assertListEqual(snake_case					,			snake_case      )

      a__	:						Optional[Any]      		=  tokens + [tokenizer.unk_token]
      a__	:						Optional[Any]      		=  [14, 15, 10, 9, 3, 2, 15, 19]
      self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case      )					,			snake_case      )





   def 	_snake_case ( self      )				->  List[Any]:
      """simple docstring"""





      if not self.test_rust_tokenizer:
         return

      a__	:						Optional[Any]      		=  self.get_tokenizer()
      a__	:						Any      		=  self.get_rust_tokenizer(add_prefix_space=snake_case      )

      a__	:						str      		=  "lower newer"

      # Testing tokenization
      a__	:						List[Any]      		=  tokenizer.tokenize(snake_case					,			add_prefix_space=snake_case      )
      a__	:						Optional[Any]      		=  rust_tokenizer.tokenize(snake_case      )
      self.assertListEqual(snake_case					,			snake_case      )

      # Testing conversion to ids without special tokens
      a__	:						Union[str, Any]      		=  tokenizer.encode(snake_case					,			add_special_tokens=snake_case					,			add_prefix_space=snake_case      )
      a__	:						Dict      		=  rust_tokenizer.encode(snake_case					,			add_special_tokens=snake_case      )
      self.assertListEqual(snake_case					,			snake_case      )

      # Testing conversion to ids with special tokens
      a__	:						Dict      		=  self.get_rust_tokenizer(add_prefix_space=snake_case      )
      a__	:						str      		=  tokenizer.encode(snake_case					,			add_prefix_space=snake_case      )
      a__	:						str      		=  rust_tokenizer.encode(snake_case      )
      self.assertListEqual(snake_case					,			snake_case      )

      # Testing the unknown token
      a__	:						int      		=  tokens + [rust_tokenizer.unk_token]
      a__	:						Any      		=  [14, 15, 10, 9, 3, 2, 15, 19]
      self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(snake_case      )					,			snake_case      )





   def 	_snake_case ( self					,			*snake_case					,			**snake_case      )				->  List[Any]:
      """simple docstring"""





      pass





   def 	_snake_case ( self					,			snake_case=15      )				->  Union[str, Any]:
      """simple docstring"""





      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""      ):
            a__	:						Any      		=  self.rust_tokenizer_class.from_pretrained(snake_case					,			**snake_case      )

            # Simple input
            a__	:						Any      		=  "This is a simple input"
            a__	:						int      		=  ["This is a simple input 1", "This is a simple input 2"]
            a__	:						List[str]      		=  ("This is a simple input", "This is a pair")
            a__	:						List[Any]      		=  [
                ("This is a simple input 1", "This is a simple input 2"),
                ("This is a simple pair 1", "This is a simple pair 2"),
            ]

            # Simple input tests
            self.assertRaises(snake_case					,			tokenizer_r.encode					,			snake_case					,			max_length=snake_case					,			padding="max_length"      )

            # Simple input
            self.assertRaises(snake_case					,			tokenizer_r.encode_plus					,			snake_case					,			max_length=snake_case					,			padding="max_length"      )

            # Simple input
            self.assertRaises(
                snake_case					,			tokenizer_r.batch_encode_plus					,			snake_case					,			max_length=snake_case					,			padding="max_length"					,			)

            # Pair input
            self.assertRaises(snake_case					,			tokenizer_r.encode					,			snake_case					,			max_length=snake_case					,			padding="max_length"      )

            # Pair input
            self.assertRaises(snake_case					,			tokenizer_r.encode_plus					,			snake_case					,			max_length=snake_case					,			padding="max_length"      )

            # Pair input
            self.assertRaises(
                snake_case					,			tokenizer_r.batch_encode_plus					,			snake_case					,			max_length=snake_case					,			padding="max_length"					,			)





   def 	_snake_case ( self      )				->  List[Any]:
      """simple docstring"""





      a__	:						str      		=  CodeGenTokenizer.from_pretrained(self.tmpdirname					,			pad_token="<pad>"      )

      # Simple input
      a__	:						Tuple      		=  "This is a simple input"
      a__	:						Any      		=  ["This is a simple input looooooooong", "This is a simple input"]
      a__	:						Optional[int]      		=  ("This is a simple input", "This is a pair")
      a__	:						Union[str, Any]      		=  [
          ("This is a simple input loooooong", "This is a simple input"),
          ("This is a simple pair loooooong", "This is a simple pair"),
      ]

      a__	:						Optional[int]      		=  tokenizer.pad_token_id

      a__	:						str      		=  tokenizer(snake_case					,			padding="max_length"					,			max_length=30					,			return_tensors="np"      )
      a__	:						List[str]      		=  tokenizer(snake_case					,			padding=snake_case					,			truncate=snake_case					,			return_tensors="np"      )
      a__	:						Optional[Any]      		=  tokenizer(*snake_case					,			padding="max_length"					,			max_length=60					,			return_tensors="np"      )
      a__	:						Optional[Any]      		=  tokenizer(snake_case					,			padding=snake_case					,			truncate=snake_case					,			return_tensors="np"      )

      # s
      # test single string max_length padding
      self.assertEqual(out_s["input_ids"].shape[-1]					,			30      )
      self.assertTrue(pad_token_id in out_s["input_ids"]      )
      self.assertTrue(0 in out_s["attention_mask"]      )

      # s2
      # test automatic padding
      self.assertEqual(out_sa["input_ids"].shape[-1]					,			33      )
      # long slice doesn't have padding
      self.assertFalse(pad_token_id in out_sa["input_ids"][0]      )
      self.assertFalse(0 in out_sa["attention_mask"][0]      )
      # short slice does have padding
      self.assertTrue(pad_token_id in out_sa["input_ids"][1]      )
      self.assertTrue(0 in out_sa["attention_mask"][1]      )

      # p
      # test single pair max_length padding
      self.assertEqual(out_p["input_ids"].shape[-1]					,			60      )
      self.assertTrue(pad_token_id in out_p["input_ids"]      )
      self.assertTrue(0 in out_p["attention_mask"]      )

      # p2
      # test automatic padding pair
      self.assertEqual(out_pa["input_ids"].shape[-1]					,			52      )
      # long slice pair doesn't have padding
      self.assertFalse(pad_token_id in out_pa["input_ids"][0]      )
      self.assertFalse(0 in out_pa["attention_mask"][0]      )
      # short slice pair does have padding
      self.assertTrue(pad_token_id in out_pa["input_ids"][1]      )
      self.assertTrue(0 in out_pa["attention_mask"][1]      )





   def 	_snake_case ( self      )				->  Dict:
      """simple docstring"""





      a__	:						Optional[Any]      		=  "$$$"
      a__	:						Dict      		=  CodeGenTokenizer.from_pretrained(self.tmpdirname					,			bos_token=snake_case					,			add_bos_token=snake_case      )

      a__	:						Optional[int]      		=  "This is a simple input"
      a__	:						Dict      		=  ["This is a simple input 1", "This is a simple input 2"]

      a__	:						Tuple      		=  tokenizer.bos_token_id

      a__	:						Any      		=  tokenizer(snake_case      )
      a__	:						Any      		=  tokenizer(snake_case      )

      self.assertEqual(out_s.input_ids[0]					,			snake_case      )
      self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids      )      )

      a__	:						int      		=  tokenizer.decode(out_s.input_ids      )
      a__	:						Optional[int]      		=  tokenizer.batch_decode(out_sa.input_ids      )

      self.assertEqual(decode_s.split()[0]					,			snake_case      )
      self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa      )      )





   @slow
   def 	_snake_case ( self      )				->  Optional[int]:
      """simple docstring"""





      a__	:						Union[str, Any]      		=  CodeGenTokenizer.from_pretrained("Salesforce/codegen-350M-mono"      )

      a__	:						Dict      		=  "\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"
      a__	:						Dict      		=  "\nif len_a > len_b:      result = a\nelse:      result = b"

      a__	:						List[str]      		=  tokenizer.encode(snake_case      )
      a__	:						Tuple      		=  ["^#", re.escape("<|endoftext|>"      ), "^'''", "^\"\"\"", "\n\n\n"]
      a__	:						Optional[Any]      		=  tokenizer.decode(snake_case					,			truncate_before_pattern=snake_case      )
      self.assertEqual(snake_case					,			snake_case      )





   def 	_snake_case ( self      )				->  List[str]:
      """simple docstring"""





      pass








from typing import List, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE__       :			Any			      =						logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__       :			str			      =						{
    """huggingface/informer-tourism-monthly""": (
        """https://huggingface.co/huggingface/informer-tourism-monthly/resolve/main/config.json"""
    ),
    # See all Informer models at https://huggingface.co/models?filter=informer
}







class   __lowerCAmelCase			(					_UpperCamelCase  ):
   _UpperCamelCase :						Optional[Any]     		=				"""informer"""
   _UpperCamelCase :						Any     		=				{
       """hidden_size""": """d_model""",
       """num_attention_heads""": """encoder_attention_heads""",
       """num_hidden_layers""": """encoder_layers""",
   }





   def __init__( self					,			snake_case = None					,			snake_case = None					,			snake_case = "student_t"					,			snake_case = "nll"					,			snake_case = 1					,			snake_case = None					,			snake_case = "mean"					,			snake_case = 0					,			snake_case = 0					,			snake_case = 0					,			snake_case = 0					,			snake_case = None					,			snake_case = None					,			snake_case = 64					,			snake_case = 32					,			snake_case = 32					,			snake_case = 2					,			snake_case = 2					,			snake_case = 2					,			snake_case = 2					,			snake_case = True					,			snake_case = "gelu"					,			snake_case = 0.05					,			snake_case = 0.1					,			snake_case = 0.1					,			snake_case = 0.1					,			snake_case = 0.1					,			snake_case = 100					,			snake_case = 0.02					,			snake_case=True					,			snake_case = "prob"					,			snake_case = 5					,			snake_case = True					,			**snake_case					,			)				->  Union[str, Any]:
      """simple docstring"""





      a__	:						Optional[Any]      		=  prediction_length
      a__	:						Optional[int]      		=  context_length or prediction_length
      a__	:						Optional[int]      		=  distribution_output
      a__	:						str      		=  loss
      a__	:						Optional[Any]      		=  input_size
      a__	:						int      		=  num_time_features
      a__	:						Optional[int]      		=  lags_sequence if lags_sequence is not None else [1, 2, 3, 4, 5, 6, 7]
      a__	:						Optional[int]      		=  scaling
      a__	:						List[str]      		=  num_dynamic_real_features
      a__	:						Optional[int]      		=  num_static_real_features
      a__	:						Optional[int]      		=  num_static_categorical_features

      # set cardinality
      if cardinality and num_static_categorical_features > 0:
         if len(snake_case      ) != num_static_categorical_features:
            raise ValueError(
                "The cardinality should be a list of the same length as `num_static_categorical_features`"      )
         a__	:						List[Any]      		=  cardinality
      else:
         a__	:						Tuple      		=  [0]

      # set embedding_dimension
      if embedding_dimension and num_static_categorical_features > 0:
         if len(snake_case      ) != num_static_categorical_features:
            raise ValueError(
                "The embedding dimension should be a list of the same length as `num_static_categorical_features`"      )
         a__	:						Tuple      		=  embedding_dimension
      else:
         a__	:						int      		=  [min(50					,			(cat + 1) // 2      ) for cat in self.cardinality]

      a__	:						Optional[Any]      		=  num_parallel_samples

      # Transformer architecture configuration
      a__	:						int      		=  input_size * len(self.lags_sequence      ) + self._number_of_features
      a__	:						Union[str, Any]      		=  d_model
      a__	:						Any      		=  encoder_attention_heads
      a__	:						Optional[Any]      		=  decoder_attention_heads
      a__	:						int      		=  encoder_ffn_dim
      a__	:						List[Any]      		=  decoder_ffn_dim
      a__	:						List[str]      		=  encoder_layers
      a__	:						Any      		=  decoder_layers

      a__	:						List[str]      		=  dropout
      a__	:						int      		=  attention_dropout
      a__	:						List[Any]      		=  activation_dropout
      a__	:						Optional[int]      		=  encoder_layerdrop
      a__	:						Tuple      		=  decoder_layerdrop

      a__	:						Any      		=  activation_function
      a__	:						Tuple      		=  init_std

      a__	:						Optional[int]      		=  use_cache

      # Informer
      a__	:						Union[str, Any]      		=  attention_type
      a__	:						List[str]      		=  sampling_factor
      a__	:						Optional[int]      		=  distil

      super().__init__(is_encoder_decoder=snake_case					,			**snake_case      )





   @property
   def 	_snake_case ( self      )				->  int:
      """simple docstring"""





      return (
          sum(self.embedding_dimension      )
          + self.num_dynamic_real_features
          + self.num_time_features
          + self.num_static_real_features
          + self.input_size * 2  # the log1p(abs(loc)) and log(scale) features
      )

