"""simple docstring"""




import itertools
import random
import unittest

import numpy as np

from transformers import BatchFeature, SpeechTaFeatureExtractor
from transformers.testing_utils import require_torch
from transformers.utils.import_utils import is_torch_available

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_torch_available():
     import torch


_a   :	Any    					=     random.Random()





def 							SCREAMING_SNAKE_CASE  (    _lowerCamelCase		: Tuple    ,_lowerCamelCase		: Any=1.0    ,_lowerCamelCase		: List[Any]=None    ,_lowerCamelCase		: Dict=None			) ->   int:
  if rng is None:
    _lowerCAmelCase		: List[str]						    =  global_rng

  _lowerCAmelCase		: str						    =  []
  for batch_idx in range(shape[0]			):
    values.append([]			)
    for _ in range(shape[1]			):
      values[-1].append(rng.random() * scale			)

  return values
@require_torch
class 		__A							(				unittest.TestCase	):






     def __init__(	self       ,  a__       ,  a__=7       ,  a__=400       ,  a__=2000       ,  a__=1       ,  a__=0.0       ,  a__=16000       ,  a__=True       ,  a__=80       ,  a__=16       ,  a__=64       ,  a__="hann_window"       ,  a__=80       ,  a__=7600       ,  a__=1e-10       ,  a__=True       ,  ):
       _lowerCAmelCase		: Any						    =  parent
       _lowerCAmelCase		: Any						    =  batch_size
       _lowerCAmelCase		: Union[str, Any]						    =  min_seq_length
       _lowerCAmelCase		: Any						    =  max_seq_length
       _lowerCAmelCase		: List[Any]						    =  (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
       _lowerCAmelCase		: Optional[Any]						    =  feature_size
       _lowerCAmelCase		: int						    =  padding_value
       _lowerCAmelCase		: List[Any]						    =  sampling_rate
       _lowerCAmelCase		: Tuple						    =  do_normalize
       _lowerCAmelCase		: Dict						    =  num_mel_bins
       _lowerCAmelCase		: Dict						    =  hop_length
       _lowerCAmelCase		: Any						    =  win_length
       _lowerCAmelCase		: Optional[int]						    =  win_function
       _lowerCAmelCase		: List[Any]						    =  fmin
       _lowerCAmelCase		: List[str]						    =  fmax
       _lowerCAmelCase		: List[str]						    =  mel_floor
       _lowerCAmelCase		: Any						    =  return_attention_mask






     def 					__A						(	self						):
       return {
           "feature_size": self.feature_size,
           "padding_value": self.padding_value,
           "sampling_rate": self.sampling_rate,
           "do_normalize": self.do_normalize,
           "num_mel_bins": self.num_mel_bins,
           "hop_length": self.hop_length,
           "win_length": self.win_length,
           "win_function": self.win_function,
           "fmin": self.fmin,
           "fmax": self.fmax,
           "mel_floor": self.mel_floor,
           "return_attention_mask": self.return_attention_mask,
       }






     def 					__A						(	self       ,  a__=False       ,  a__=False						):
       def _flatten(a__						):
         return list(itertools.chain(*a__						)						)

       if equal_length:
         _lowerCAmelCase		: Dict						    =  floats_list((self.batch_size, self.max_seq_length)						)
       else:
         # make sure that inputs increase in size
         _lowerCAmelCase		: Optional[int]						    =  [
             _flatten(floats_list((x, self.feature_size)						)						)
             for x in range(self.min_seq_length       ,  self.max_seq_length       ,  self.seq_length_diff						)
         ]

       if numpify:
         _lowerCAmelCase		: List[str]						    =  [np.asarray(a__						) for x in speech_inputs]

       return speech_inputs






     def 					__A						(	self       ,  a__=False       ,  a__=False						):
       if equal_length:
         _lowerCAmelCase		: str						    =  [floats_list((self.max_seq_length, self.num_mel_bins)						) for _ in range(self.batch_size						)]
       else:
         # make sure that inputs increase in size
         _lowerCAmelCase		: List[str]						    =  [
             floats_list((x, self.num_mel_bins)						)
             for x in range(self.min_seq_length       ,  self.max_seq_length       ,  self.seq_length_diff						)
         ]

       if numpify:
         _lowerCAmelCase		: Union[str, Any]						    =  [np.asarray(a__						) for x in speech_inputs]

       return speech_inputs







@require_torch
class 		__A							(				SCREAMING_SNAKE_CASE_					,			unittest.TestCase	):
     _UpperCamelCase							:    Optional[int]	      =	SpeechTaFeatureExtractor






     def 					__A						(	self						):
       _lowerCAmelCase		: Any						    =  SpeechTaFeatureExtractionTester(self						)






     def 					__A						(	self       ,  a__						):
       self.assertTrue(np.all(np.mean(a__       ,  axis=0						) < 1e-3						)						)
       self.assertTrue(np.all(np.abs(np.var(a__       ,  axis=0						) - 1						) < 1e-3						)						)






     def 					__A						(	self						):
       # Tests that all call wrap to encode_plus and batch_encode_plus
       _lowerCAmelCase		: Optional[Any]						    =  self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()						)
       # create three inputs of length 800, 1000, and 1200
       _lowerCAmelCase		: List[str]						    =  [floats_list((1, x)						)[0] for x in range(800       ,  1400       ,  200						)]
       _lowerCAmelCase		: List[Any]						    =  [np.asarray(a__						) for speech_input in speech_inputs]

       # Test not batched input
       _lowerCAmelCase		: Union[str, Any]						    =  feat_extract(speech_inputs[0]       ,  return_tensors="""np"""						).input_values
       _lowerCAmelCase		: List[Any]						    =  feat_extract(np_speech_inputs[0]       ,  return_tensors="""np"""						).input_values
       self.assertTrue(np.allclose(a__       ,  a__       ,  atol=1e-3						)						)

       # Test batched
       _lowerCAmelCase		: Union[str, Any]						    =  feat_extract(a__       ,  return_tensors="""np"""						).input_values
       _lowerCAmelCase		: Tuple						    =  feat_extract(a__       ,  return_tensors="""np"""						).input_values
       for enc_seq_a, enc_seq_a in zip(a__       ,  a__						):
         self.assertTrue(np.allclose(a__       ,  a__       ,  atol=1e-3						)						)






     def 					__A						(	self						):
       _lowerCAmelCase		: int						    =  self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()						)
       _lowerCAmelCase		: List[Any]						    =  [floats_list((1, x)						)[0] for x in range(800       ,  1400       ,  200						)]

       _lowerCAmelCase		: Any						    =  ["""longest""", """max_length""", """do_not_pad"""]
       _lowerCAmelCase		: str						    =  [None, 1600, None]
       for max_length, padding in zip(a__       ,  a__						):
         _lowerCAmelCase		: List[Any]						    =  feat_extract(a__       ,  padding=a__       ,  max_length=a__       ,  return_tensors="""np"""						)
         _lowerCAmelCase		: Optional[Any]						    =  processed.input_values

         self._check_zero_mean_unit_variance(input_values[0][:800]						)
         self.assertTrue(input_values[0][800:].sum() < 1e-6						)
         self._check_zero_mean_unit_variance(input_values[1][:1000]						)
         self.assertTrue(input_values[0][1000:].sum() < 1e-6						)
         self._check_zero_mean_unit_variance(input_values[2][:1200]						)






     def 					__A						(	self						):
       _lowerCAmelCase		: Tuple						    =  self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()						)
       _lowerCAmelCase		: Union[str, Any]						    =  range(800       ,  1400       ,  200						)
       _lowerCAmelCase		: int						    =  [floats_list((1, x)						)[0] for x in lengths]

       _lowerCAmelCase		: int						    =  ["""longest""", """max_length""", """do_not_pad"""]
       _lowerCAmelCase		: Optional[int]						    =  [None, 1600, None]

       for max_length, padding in zip(a__       ,  a__						):
         _lowerCAmelCase		: Any						    =  feat_extract(a__       ,  max_length=a__       ,  padding=a__						)
         _lowerCAmelCase		: Union[str, Any]						    =  processed.input_values

         self._check_zero_mean_unit_variance(input_values[0][:800]						)
         self._check_zero_mean_unit_variance(input_values[1][:1000]						)
         self._check_zero_mean_unit_variance(input_values[2][:1200]						)






     def 					__A						(	self						):
       _lowerCAmelCase		: int						    =  self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()						)
       _lowerCAmelCase		: str						    =  [floats_list((1, x)						)[0] for x in range(800       ,  1400       ,  200						)]
       _lowerCAmelCase		: int						    =  feat_extract(
           a__       ,  truncation=a__       ,  max_length=1000       ,  padding="""max_length"""       ,  return_tensors="""np"""						)
       _lowerCAmelCase		: int						    =  processed.input_values

       self._check_zero_mean_unit_variance(input_values[0, :800]						)
       self._check_zero_mean_unit_variance(input_values[1]						)
       self._check_zero_mean_unit_variance(input_values[2]						)






     def 					__A						(	self						):
       _lowerCAmelCase		: int						    =  self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()						)
       _lowerCAmelCase		: Union[str, Any]						    =  [floats_list((1, x)						)[0] for x in range(800       ,  1400       ,  200						)]
       _lowerCAmelCase		: Optional[Any]						    =  feat_extract(
           a__       ,  truncation=a__       ,  max_length=1000       ,  padding="""longest"""       ,  return_tensors="""np"""						)
       _lowerCAmelCase		: int						    =  processed.input_values

       self._check_zero_mean_unit_variance(input_values[0, :800]						)
       self._check_zero_mean_unit_variance(input_values[1, :1000]						)
       self._check_zero_mean_unit_variance(input_values[2]						)

       # make sure that if max_length < longest -> then pad to max_length
       self.assertTrue(input_values.shape == (3, 1000)						)

       _lowerCAmelCase		: Optional[Any]						    =  [floats_list((1, x)						)[0] for x in range(800       ,  1400       ,  200						)]
       _lowerCAmelCase		: List[Any]						    =  feat_extract(
           a__       ,  truncation=a__       ,  max_length=2000       ,  padding="""longest"""       ,  return_tensors="""np"""						)
       _lowerCAmelCase		: List[Any]						    =  processed.input_values

       self._check_zero_mean_unit_variance(input_values[0, :800]						)
       self._check_zero_mean_unit_variance(input_values[1, :1000]						)
       self._check_zero_mean_unit_variance(input_values[2]						)

       # make sure that if max_length > longest -> then pad to longest
       self.assertTrue(input_values.shape == (3, 1200)						)






     def 					__A						(	self						):
       _lowerCAmelCase		: int						    =  self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()						)
       _lowerCAmelCase		: Optional[int]						    =  np.random.rand(100						).astype(np.floataa						)
       _lowerCAmelCase		: Union[str, Any]						    =  np_speech_inputs.tolist()

       for inputs in [py_speech_inputs, np_speech_inputs]:
         _lowerCAmelCase		: Optional[Any]						    =  feature_extractor.pad([{"""input_values""": inputs}]       ,  return_tensors="""np"""						)
         self.assertTrue(np_processed.input_values.dtype == np.floataa						)
         _lowerCAmelCase		: List[Any]						    =  feature_extractor.pad([{"""input_values""": inputs}]       ,  return_tensors="""pt"""						)
         self.assertTrue(pt_processed.input_values.dtype == torch.floataa						)






     def 					__A						(	self						):
       # Tests that all call wrap to encode_plus and batch_encode_plus
       _lowerCAmelCase		: Tuple						    =  self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()						)
       # create three inputs of length 800, 1000, and 1200
       _lowerCAmelCase		: Optional[int]						    =  [floats_list((1, x)						)[0] for x in range(800       ,  1400       ,  200						)]
       _lowerCAmelCase		: Tuple						    =  [np.asarray(a__						) for speech_input in speech_inputs]

       # Test feature size
       _lowerCAmelCase		: List[str]						    =  feature_extractor(audio_target=a__       ,  padding=a__       ,  return_tensors="""np"""						).input_values
       self.assertTrue(input_values.ndim == 3						)
       self.assertTrue(input_values.shape[-1] == feature_extractor.num_mel_bins						)

       # Test not batched input
       _lowerCAmelCase		: str						    =  feature_extractor(speech_inputs[0]       ,  return_tensors="""np"""						).input_values
       _lowerCAmelCase		: Union[str, Any]						    =  feature_extractor(np_speech_inputs[0]       ,  return_tensors="""np"""						).input_values
       self.assertTrue(np.allclose(a__       ,  a__       ,  atol=1e-3						)						)

       # Test batched
       _lowerCAmelCase		: List[Any]						    =  feature_extractor(a__       ,  return_tensors="""np"""						).input_values
       _lowerCAmelCase		: Any						    =  feature_extractor(a__       ,  return_tensors="""np"""						).input_values
       for enc_seq_a, enc_seq_a in zip(a__       ,  a__						):
         self.assertTrue(np.allclose(a__       ,  a__       ,  atol=1e-3						)						)

       # Test 2-D numpy arrays are batched.
       _lowerCAmelCase		: Dict						    =  [floats_list((1, x)						)[0] for x in (800, 800, 800)]
       _lowerCAmelCase		: str						    =  np.asarray(a__						)
       _lowerCAmelCase		: List[Any]						    =  feature_extractor(a__       ,  return_tensors="""np"""						).input_values
       _lowerCAmelCase		: List[Any]						    =  feature_extractor(a__       ,  return_tensors="""np"""						).input_values
       for enc_seq_a, enc_seq_a in zip(a__       ,  a__						):
         self.assertTrue(np.allclose(a__       ,  a__       ,  atol=1e-3						)						)






     def 					__A						(	self						):
       _lowerCAmelCase		: Any						    =  self.feat_extract_tester.prepare_inputs_for_target()
       _lowerCAmelCase		: Optional[Any]						    =  self.feature_extraction_class(**self.feat_extract_dict						)
       _lowerCAmelCase		: Optional[Any]						    =  feat_extract.model_input_names[0]

       _lowerCAmelCase		: Tuple						    =  BatchFeature({input_name: speech_inputs}						)

       self.assertTrue(all(len(a__						) == len(a__						) for x, y in zip(a__       ,  processed_features[input_name]						)						)						)

       _lowerCAmelCase		: Union[str, Any]						    =  self.feat_extract_tester.prepare_inputs_for_target(equal_length=a__						)
       _lowerCAmelCase		: Any						    =  BatchFeature({input_name: speech_inputs}       ,  tensor_type="""np"""						)

       _lowerCAmelCase		: Dict						    =  processed_features[input_name]

       if len(batch_features_input.shape						) < 3:
         _lowerCAmelCase		: List[str]						    =  batch_features_input[:, :, None]

       self.assertTrue(
           batch_features_input.shape
           == (self.feat_extract_tester.batch_size, len(speech_inputs[0]						), self.feat_extract_tester.num_mel_bins)						)






     @require_torch
     def 					__A						(	self						):
       _lowerCAmelCase		: Dict						    =  self.feat_extract_tester.prepare_inputs_for_target(equal_length=a__						)
       _lowerCAmelCase		: Optional[Any]						    =  self.feature_extraction_class(**self.feat_extract_dict						)
       _lowerCAmelCase		: Tuple						    =  feat_extract.model_input_names[0]

       _lowerCAmelCase		: Any						    =  BatchFeature({input_name: speech_inputs}       ,  tensor_type="""pt"""						)

       _lowerCAmelCase		: Optional[int]						    =  processed_features[input_name]

       if len(batch_features_input.shape						) < 3:
         _lowerCAmelCase		: Union[str, Any]						    =  batch_features_input[:, :, None]

       self.assertTrue(
           batch_features_input.shape
           == (self.feat_extract_tester.batch_size, len(speech_inputs[0]						), self.feat_extract_tester.num_mel_bins)						)






     @require_torch
     def 					__A						(	self						):
       _lowerCAmelCase		: Any						    =  self.feature_extraction_class(**self.feat_extract_dict						)
       _lowerCAmelCase		: int						    =  self.feat_extract_tester.prepare_inputs_for_target()
       _lowerCAmelCase		: str						    =  feat_extract.model_input_names[0]

       _lowerCAmelCase		: List[str]						    =  BatchFeature({input_name: speech_inputs}						)

       _lowerCAmelCase		: Any						    =  feat_extract.num_mel_bins  # hack!

       _lowerCAmelCase		: Optional[int]						    =  feat_extract.pad(a__       ,  padding="""longest"""       ,  return_tensors="""np"""						)[input_name]
       _lowerCAmelCase		: str						    =  feat_extract.pad(a__       ,  padding="""longest"""       ,  return_tensors="""pt"""						)[input_name]

       self.assertTrue(abs(input_np.astype(np.floataa						).sum() - input_pt.numpy().astype(np.floataa						).sum()						) < 1e-2						)






     def 					__A						(	self						):
       _lowerCAmelCase		: List[str]						    =  self.feat_extract_dict
       _lowerCAmelCase		: Dict						    =  True
       _lowerCAmelCase		: str						    =  self.feature_extraction_class(**a__						)
       _lowerCAmelCase		: Union[str, Any]						    =  self.feat_extract_tester.prepare_inputs_for_target()
       _lowerCAmelCase		: Optional[int]						    =  [len(a__						) for x in speech_inputs]
       _lowerCAmelCase		: Tuple						    =  feat_extract.model_input_names[0]

       _lowerCAmelCase		: List[str]						    =  BatchFeature({input_name: speech_inputs}						)

       _lowerCAmelCase		: str						    =  feat_extract.num_mel_bins  # hack!

       _lowerCAmelCase		: Tuple						    =  feat_extract.pad(a__       ,  padding="""longest"""       ,  return_tensors="""np"""						)
       self.assertIn("""attention_mask"""       ,  a__						)
       self.assertListEqual(list(processed.attention_mask.shape						)       ,  list(processed[input_name].shape[:2]						)						)
       self.assertListEqual(processed.attention_mask.sum(-1						).tolist()       ,  a__						)






     def 					__A						(	self						):
       _lowerCAmelCase		: Dict						    =  self.feat_extract_dict
       _lowerCAmelCase		: List[str]						    =  True
       _lowerCAmelCase		: Dict						    =  self.feature_extraction_class(**a__						)
       _lowerCAmelCase		: Tuple						    =  self.feat_extract_tester.prepare_inputs_for_target()
       _lowerCAmelCase		: int						    =  [len(a__						) for x in speech_inputs]
       _lowerCAmelCase		: Optional[int]						    =  feat_extract.model_input_names[0]

       _lowerCAmelCase		: Union[str, Any]						    =  BatchFeature({input_name: speech_inputs}						)
       _lowerCAmelCase		: List[Any]						    =  min(a__						)

       _lowerCAmelCase		: Optional[int]						    =  feat_extract.num_mel_bins  # hack!

       _lowerCAmelCase		: Tuple						    =  feat_extract.pad(
           a__       ,  padding="""max_length"""       ,  max_length=a__       ,  truncation=a__       ,  return_tensors="""np"""						)
       self.assertIn("""attention_mask"""       ,  a__						)
       self.assertListEqual(
           list(processed_pad.attention_mask.shape						)       ,  [processed_pad[input_name].shape[0], max_length]						)
       self.assertListEqual(
           processed_pad.attention_mask[:, :max_length].sum(-1						).tolist()       ,  [max_length for x in speech_inputs]						)






     def 					__A						(	self       ,  a__						):
       from datasets import load_dataset

       _lowerCAmelCase		: str						    =  load_dataset("""hf-internal-testing/librispeech_asr_dummy"""       ,  """clean"""       ,  split="""validation"""						)
       # automatic decoding with librispeech
       _lowerCAmelCase		: Union[str, Any]						    =  ds.sort("""id"""						).select(range(a__						)						)[:num_samples]["""audio"""]

       return [x["array"] for x in speech_samples]






     def 					__A						(	self						):
       # fmt: off
       _lowerCAmelCase		: Union[str, Any]						    =  torch.tensor(
           [2.3804e-03, 2.0752e-03, 1.9836e-03, 2.1057e-03, 1.6174e-03,
            3.0518e-04, 9.1553e-05, 3.3569e-04, 9.7656e-04, 1.8311e-03,
            2.0142e-03, 2.1057e-03, 1.7395e-03, 4.5776e-04, -3.9673e-04,
            4.5776e-04, 1.0071e-03, 9.1553e-05, 4.8828e-04, 1.1597e-03,
            7.3242e-04, 9.4604e-04, 1.8005e-03, 1.8311e-03, 8.8501e-04,
            4.2725e-04, 4.8828e-04, 7.3242e-04, 1.0986e-03, 2.1057e-03]						)
       # fmt: on

       _lowerCAmelCase		: Tuple						    =  self._load_datasamples(1						)
       _lowerCAmelCase		: int						    =  SpeechTaFeatureExtractor()
       _lowerCAmelCase		: int						    =  feature_extractor(a__       ,  return_tensors="""pt"""						).input_values
       self.assertEquals(input_values.shape       ,  (1, 93680)						)
       self.assertTrue(torch.allclose(input_values[0, :30]       ,  a__       ,  atol=1e-6						)						)






     def 					__A						(	self						):
       # fmt: off
       _lowerCAmelCase		: str						    =  torch.tensor(
           [-2.6_8_7_0, -3.0_1_0_4, -3.1_3_5_6, -3.5_3_5_2, -3.0_0_4_4, -3.0_3_5_3, -3.4_7_1_9, -3.6_7_7_7,
            -3.1_5_2_0, -2.9_4_3_5, -2.6_5_5_3, -2.8_7_9_5, -2.9_9_4_4, -2.5_9_2_1, -3.0_2_7_9, -3.0_3_8_6,
            -3.0_8_6_4, -3.1_2_9_1, -3.2_3_5_3, -2.7_4_4_4, -2.6_8_3_1, -2.7_2_8_7, -3.1_7_6_1, -3.1_5_7_1,
            -3.2_7_2_6, -3.0_5_8_2, -3.1_0_0_7, -3.4_5_3_3, -3.4_6_9_5, -3.0_9_9_8]						)
       # fmt: on

       _lowerCAmelCase		: Optional[int]						    =  self._load_datasamples(1						)
       _lowerCAmelCase		: Optional[Any]						    =  SpeechTaFeatureExtractor()
       _lowerCAmelCase		: Tuple						    =  feature_extractor(audio_target=a__       ,  return_tensors="""pt"""						).input_values
       self.assertEquals(input_values.shape       ,  (1, 366, 80)						)
       self.assertTrue(torch.allclose(input_values[0, 0, :30]       ,  a__       ,  atol=1e-4						)						)





"""simple docstring"""




import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 		__A							(				SCREAMING_SNAKE_CASE_					,			unittest.TestCase	):
     _UpperCamelCase							:    Dict	      =	CodeGenTokenizer
     _UpperCamelCase							:    Dict	      =	CodeGenTokenizerFast
     _UpperCamelCase							:    Tuple	      =	True
     _UpperCamelCase							:    List[Any]	      =	{"add_prefix_space": True}
     _UpperCamelCase							:    str	      =	False






     def 					__A						(	self						):
       super().setUp()

       # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
       _lowerCAmelCase		: Union[str, Any]						    =  [
           """l""",
           """o""",
           """w""",
           """e""",
           """r""",
           """s""",
           """t""",
           """i""",
           """d""",
           """n""",
           """\u0120""",
           """\u0120l""",
           """\u0120n""",
           """\u0120lo""",
           """\u0120low""",
           """er""",
           """\u0120lowest""",
           """\u0120newer""",
           """\u0120wider""",
           """<unk>""",
           """<|endoftext|>""",
       ]
       _lowerCAmelCase		: Optional[int]						    =  dict(zip(a__       ,  range(len(a__						)						)						)						)
       _lowerCAmelCase		: str						    =  ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
       _lowerCAmelCase		: Any						    =  {"""unk_token""": """<unk>"""}

       _lowerCAmelCase		: List[str]						    =  os.path.join(self.tmpdirname       ,  VOCAB_FILES_NAMES["""vocab_file"""]						)
       _lowerCAmelCase		: List[Any]						    =  os.path.join(self.tmpdirname       ,  VOCAB_FILES_NAMES["""merges_file"""]						)
       with open(self.vocab_file       ,  """w"""       ,  encoding="""utf-8"""						) as fp:
         fp.write(json.dumps(a__						) + """\n"""						)
       with open(self.merges_file       ,  """w"""       ,  encoding="""utf-8"""						) as fp:
         fp.write("""\n""".join(a__						)						)






     def 					__A						(	self       ,  **a__						):
       kwargs.update(self.special_tokens_map						)
       return CodeGenTokenizer.from_pretrained(self.tmpdirname       ,  **a__						)






     def 					__A						(	self       ,  **a__						):
       kwargs.update(self.special_tokens_map						)
       return CodeGenTokenizerFast.from_pretrained(self.tmpdirname       ,  **a__						)






     def 					__A						(	self       ,  a__						):
       _lowerCAmelCase		: str						    =  """lower newer"""
       _lowerCAmelCase		: Tuple						    =  """lower newer"""
       return input_text, output_text






     def 					__A						(	self						):
       _lowerCAmelCase		: str						    =  CodeGenTokenizer(self.vocab_file       ,  self.merges_file       ,  **self.special_tokens_map						)
       _lowerCAmelCase		: int						    =  """lower newer"""
       _lowerCAmelCase		: List[str]						    =  ["""\u0120low""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
       _lowerCAmelCase		: Optional[int]						    =  tokenizer.tokenize(a__       ,  add_prefix_space=a__						)
       self.assertListEqual(a__       ,  a__						)

       _lowerCAmelCase		: Tuple						    =  tokens + [tokenizer.unk_token]
       _lowerCAmelCase		: List[str]						    =  [14, 15, 10, 9, 3, 2, 15, 19]
       self.assertListEqual(tokenizer.convert_tokens_to_ids(a__						)       ,  a__						)






     def 					__A						(	self						):
       if not self.test_rust_tokenizer:
         return

       _lowerCAmelCase		: Optional[int]						    =  self.get_tokenizer()
       _lowerCAmelCase		: Optional[int]						    =  self.get_rust_tokenizer(add_prefix_space=a__						)

       _lowerCAmelCase		: Any						    =  """lower newer"""

       # Testing tokenization
       _lowerCAmelCase		: Any						    =  tokenizer.tokenize(a__       ,  add_prefix_space=a__						)
       _lowerCAmelCase		: int						    =  rust_tokenizer.tokenize(a__						)
       self.assertListEqual(a__       ,  a__						)

       # Testing conversion to ids without special tokens
       _lowerCAmelCase		: Union[str, Any]						    =  tokenizer.encode(a__       ,  add_special_tokens=a__       ,  add_prefix_space=a__						)
       _lowerCAmelCase		: Dict						    =  rust_tokenizer.encode(a__       ,  add_special_tokens=a__						)
       self.assertListEqual(a__       ,  a__						)

       # Testing conversion to ids with special tokens
       _lowerCAmelCase		: int						    =  self.get_rust_tokenizer(add_prefix_space=a__						)
       _lowerCAmelCase		: Optional[int]						    =  tokenizer.encode(a__       ,  add_prefix_space=a__						)
       _lowerCAmelCase		: Any						    =  rust_tokenizer.encode(a__						)
       self.assertListEqual(a__       ,  a__						)

       # Testing the unknown token
       _lowerCAmelCase		: List[str]						    =  tokens + [rust_tokenizer.unk_token]
       _lowerCAmelCase		: Dict						    =  [14, 15, 10, 9, 3, 2, 15, 19]
       self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(a__						)       ,  a__						)






     def 					__A						(	self       ,  *a__       ,  **a__						):
       # It's very difficult to mix/test pretokenization with byte-level
       # And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
       pass






     def 					__A						(	self       ,  a__=15						):
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"						):
           _lowerCAmelCase		: List[Any]						    =  self.rust_tokenizer_class.from_pretrained(a__       ,  **a__						)

           # Simple input
           _lowerCAmelCase		: Dict						    =  """This is a simple input"""
           _lowerCAmelCase		: Optional[int]						    =  ["""This is a simple input 1""", """This is a simple input 2"""]
           _lowerCAmelCase		: Optional[int]						    =  ("""This is a simple input""", """This is a pair""")
           _lowerCAmelCase		: str						    =  [
               ("""This is a simple input 1""", """This is a simple input 2"""),
               ("""This is a simple pair 1""", """This is a simple pair 2"""),
           ]

           # Simple input tests
           self.assertRaises(a__       ,  tokenizer_r.encode       ,  a__       ,  max_length=a__       ,  padding="""max_length"""						)

           # Simple input
           self.assertRaises(a__       ,  tokenizer_r.encode_plus       ,  a__       ,  max_length=a__       ,  padding="""max_length"""						)

           # Simple input
           self.assertRaises(
               a__       ,  tokenizer_r.batch_encode_plus       ,  a__       ,  max_length=a__       ,  padding="""max_length"""       ,  )

           # Pair input
           self.assertRaises(a__       ,  tokenizer_r.encode       ,  a__       ,  max_length=a__       ,  padding="""max_length"""						)

           # Pair input
           self.assertRaises(a__       ,  tokenizer_r.encode_plus       ,  a__       ,  max_length=a__       ,  padding="""max_length"""						)

           # Pair input
           self.assertRaises(
               a__       ,  tokenizer_r.batch_encode_plus       ,  a__       ,  max_length=a__       ,  padding="""max_length"""       ,  )






     def 					__A						(	self						):
       _lowerCAmelCase		: Any						    =  CodeGenTokenizer.from_pretrained(self.tmpdirname       ,  pad_token="""<pad>"""						)

       # Simple input
       _lowerCAmelCase		: Union[str, Any]						    =  """This is a simple input"""
       _lowerCAmelCase		: Dict						    =  ["""This is a simple input looooooooong""", """This is a simple input"""]
       _lowerCAmelCase		: Any						    =  ("""This is a simple input""", """This is a pair""")
       _lowerCAmelCase		: Optional[int]						    =  [
           ("""This is a simple input loooooong""", """This is a simple input"""),
           ("""This is a simple pair loooooong""", """This is a simple pair"""),
       ]

       _lowerCAmelCase		: Optional[int]						    =  tokenizer.pad_token_id

       _lowerCAmelCase		: Any						    =  tokenizer(a__       ,  padding="""max_length"""       ,  max_length=30       ,  return_tensors="""np"""						)
       _lowerCAmelCase		: str						    =  tokenizer(a__       ,  padding=a__       ,  truncate=a__       ,  return_tensors="""np"""						)
       _lowerCAmelCase		: int						    =  tokenizer(*a__       ,  padding="""max_length"""       ,  max_length=60       ,  return_tensors="""np"""						)
       _lowerCAmelCase		: int						    =  tokenizer(a__       ,  padding=a__       ,  truncate=a__       ,  return_tensors="""np"""						)

       # s
       # test single string max_length padding
       self.assertEqual(out_s["""input_ids"""].shape[-1]       ,  30						)
       self.assertTrue(pad_token_id in out_s["""input_ids"""]						)
       self.assertTrue(0 in out_s["""attention_mask"""]						)

       # s2
       # test automatic padding
       self.assertEqual(out_sa["""input_ids"""].shape[-1]       ,  33						)
       # long slice doesn't have padding
       self.assertFalse(pad_token_id in out_sa["""input_ids"""][0]						)
       self.assertFalse(0 in out_sa["""attention_mask"""][0]						)
       # short slice does have padding
       self.assertTrue(pad_token_id in out_sa["""input_ids"""][1]						)
       self.assertTrue(0 in out_sa["""attention_mask"""][1]						)

       # p
       # test single pair max_length padding
       self.assertEqual(out_p["""input_ids"""].shape[-1]       ,  60						)
       self.assertTrue(pad_token_id in out_p["""input_ids"""]						)
       self.assertTrue(0 in out_p["""attention_mask"""]						)

       # p2
       # test automatic padding pair
       self.assertEqual(out_pa["""input_ids"""].shape[-1]       ,  52						)
       # long slice pair doesn't have padding
       self.assertFalse(pad_token_id in out_pa["""input_ids"""][0]						)
       self.assertFalse(0 in out_pa["""attention_mask"""][0]						)
       # short slice pair does have padding
       self.assertTrue(pad_token_id in out_pa["""input_ids"""][1]						)
       self.assertTrue(0 in out_pa["""attention_mask"""][1]						)






     def 					__A						(	self						):
       _lowerCAmelCase		: List[str]						    =  """$$$"""
       _lowerCAmelCase		: str						    =  CodeGenTokenizer.from_pretrained(self.tmpdirname       ,  bos_token=a__       ,  add_bos_token=a__						)

       _lowerCAmelCase		: Tuple						    =  """This is a simple input"""
       _lowerCAmelCase		: Union[str, Any]						    =  ["""This is a simple input 1""", """This is a simple input 2"""]

       _lowerCAmelCase		: List[str]						    =  tokenizer.bos_token_id

       _lowerCAmelCase		: str						    =  tokenizer(a__						)
       _lowerCAmelCase		: Union[str, Any]						    =  tokenizer(a__						)

       self.assertEqual(out_s.input_ids[0]       ,  a__						)
       self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids						)						)

       _lowerCAmelCase		: Optional[Any]						    =  tokenizer.decode(out_s.input_ids						)
       _lowerCAmelCase		: Optional[int]						    =  tokenizer.batch_decode(out_sa.input_ids						)

       self.assertEqual(decode_s.split()[0]       ,  a__						)
       self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa						)						)






     @slow
     def 					__A						(	self						):
       _lowerCAmelCase		: int						    =  CodeGenTokenizer.from_pretrained("""Salesforce/codegen-350M-mono"""						)

       _lowerCAmelCase		: Optional[int]						    =  """\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"""
       _lowerCAmelCase		: List[Any]						    =  """\nif len_a > len_b:      result = a\nelse:      result = b"""

       _lowerCAmelCase		: Tuple						    =  tokenizer.encode(a__						)
       _lowerCAmelCase		: Optional[Any]						    =  ["""^#""", re.escape("""<|endoftext|>"""						), """^'''""", """^\"\"\"""", """\n\n\n"""]
       _lowerCAmelCase		: int						    =  tokenizer.decode(a__       ,  truncate_before_pattern=a__						)
       self.assertEqual(a__       ,  a__						)






     def 					__A						(	self						):
       pass




