






import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin
class    _UpperCAmelCase      ( lowercase__	,unittest.TestCase			):

				"""simple docstring"""




				lowercase__       							= TransfoXLTokenizer
				lowercase__       							= False
				lowercase__       							= False




				def        lowercase__     (  self		:					Union[str, Any]						):

								'''simple docstring'''
								super().setUp()

								lowercase__     				=    [
								    '''<unk>''',
								    '''[CLS]''',
								    '''[SEP]''',
								    '''want''',
								    '''unwanted''',
								    '''wa''',
								    '''un''',
								    '''running''',
								    ''',''',
								    '''low''',
								    '''l''',
								]
								lowercase__     				=    os.path.join(self.tmpdirname,						VOCAB_FILES_NAMES['''vocab_file''']						)
								with open(self.vocab_file,						'''w''',						encoding='''utf-8'''						) as vocab_writer:
												vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]						)						)




				def        lowercase__     (  self		:					Union[str, Any],						**lowerCamelCase		:					Any						):

								'''simple docstring'''
								lowercase__     				=    True
								return TransfoXLTokenizer.from_pretrained(self.tmpdirname,						**__lowercase						)




				def        lowercase__     (  self		:					Tuple,						lowerCamelCase		:					Optional[int]						):

								'''simple docstring'''
								lowercase__     				=    '''<unk> UNwanted , running'''
								lowercase__     				=    '''<unk> unwanted, running'''
								return input_text, output_text




				def        lowercase__     (  self		:					str						):

								'''simple docstring'''
								lowercase__     				=    TransfoXLTokenizer(vocab_file=self.vocab_file,						lower_case=__lowercase						)

								lowercase__     				=    tokenizer.tokenize('''<unk> UNwanted , running'''						)
								self.assertListEqual(__lowercase,						['''<unk>''', '''unwanted''', ''',''', '''running''']						)

								self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowercase						),						[0, 4, 8, 7]						)




				def        lowercase__     (  self		:					Tuple						):

								'''simple docstring'''
								lowercase__     				=    TransfoXLTokenizer(lower_case=__lowercase						)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHeLLo ! how  \n Are yoU ?  '''						),						['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']						)




				def        lowercase__     (  self		:					Tuple						):

								'''simple docstring'''
								lowercase__     				=    TransfoXLTokenizer(lower_case=__lowercase						)

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHeLLo ! how  \n Are yoU ?  '''						),						['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']						)




				def        lowercase__     (  self		:					str						):

								'''simple docstring'''
								lowercase__     				=    TransfoXLTokenizer(lower_case=__lowercase						)
								lowercase__     				=    '''Hello (bracket) and side-scrolled [and] Henry\'s $5,000 with 3.34 m. What\'s up!?'''
								lowercase__     				=    [
								    '''Hello''',
								    '''(''',
								    '''bracket''',
								    ''')''',
								    '''and''',
								    '''side''',
								    '''@-@''',
								    '''scrolled''',
								    '''[''',
								    '''and''',
								    ''']''',
								    '''Henry''',
								    '''\'s''',
								    '''$''',
								    '''5''',
								    '''@,@''',
								    '''000''',
								    '''with''',
								    '''3''',
								    '''@.@''',
								    '''34''',
								    '''m''',
								    '''.''',
								    '''What''',
								    '''\'s''',
								    '''up''',
								    '''!''',
								    '''?''',
								]

								self.assertListEqual(tokenizer.tokenize(__lowercase						),						__lowercase						)

								self.assertEqual(tokenizer.convert_tokens_to_string(__lowercase						),						__lowercase						)




				def        lowercase__     (  self		:					List[str]						):

								'''simple docstring'''
								lowercase__     				=    self.get_tokenizer()
								lowercase__     				=    len(__lowercase						)

								tokenizer.add_tokens(['''new1''', '''new2''']						)
								tokenizer.move_added_token('''new1''',						1						)

								# Check that moved token is not copied (duplicate)
								self.assertEqual(len(__lowercase						),						original_len + 2						)
								# Check that token is moved to specified id
								self.assertEqual(tokenizer.encode('''new1'''						),						[1]						)
								self.assertEqual(tokenizer.decode([1]						),						'''new1'''						)












import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
							import torch

if is_vision_available():
							from PIL import Image

							from transformers import MobileNetVaImageProcessor
class    _UpperCAmelCase      ( unittest.TestCase			):

				"""simple docstring"""




				def __init__(  self		:					str,						lowerCamelCase		:					Any,						lowerCamelCase		:					Tuple=7,						lowerCamelCase		:					str=3,						lowerCamelCase		:					Tuple=18,						lowerCamelCase		:					int=30,						lowerCamelCase		:					Tuple=400,						lowerCamelCase		:					Any=True,						lowerCamelCase		:					Any=None,						lowerCamelCase		:					List[str]=True,						lowerCamelCase		:					Union[str, Any]=None,						):

								'''simple docstring'''
								lowercase__     				=    size if size is not None else {'''shortest_edge''': 20}
								lowercase__     				=    crop_size if crop_size is not None else {'''height''': 18, '''width''': 18}
								lowercase__     				=    parent
								lowercase__     				=    batch_size
								lowercase__     				=    num_channels
								lowercase__     				=    image_size
								lowercase__     				=    min_resolution
								lowercase__     				=    max_resolution
								lowercase__     				=    do_resize
								lowercase__     				=    size
								lowercase__     				=    do_center_crop
								lowercase__     				=    crop_size




				def        lowercase__     (  self		:					Any						):

								'''simple docstring'''
								return {
								    "do_resize": self.do_resize,
								    "size": self.size,
								    "do_center_crop": self.do_center_crop,
								    "crop_size": self.crop_size,
								}



@require_torch
@require_vision
class    _UpperCAmelCase      ( A__	,unittest.TestCase			):

				"""simple docstring"""




				lowercase__       							= MobileNetVaImageProcessor if is_vision_available() else None




				def        lowercase__     (  self		:					Any						):

								'''simple docstring'''
								lowercase__     				=    MobileNetVaImageProcessingTester(self						)




				@property
				def        lowercase__     (  self		:					Union[str, Any]						):

								'''simple docstring'''
								return self.image_processor_tester.prepare_image_processor_dict()




				def        lowercase__     (  self		:					Dict						):

								'''simple docstring'''
								lowercase__     				=    self.image_processing_class(**self.image_processor_dict						)
								self.assertTrue(hasattr(lowerCamelCase,						'''do_resize'''						)						)
								self.assertTrue(hasattr(lowerCamelCase,						'''size'''						)						)
								self.assertTrue(hasattr(lowerCamelCase,						'''do_center_crop'''						)						)
								self.assertTrue(hasattr(lowerCamelCase,						'''crop_size'''						)						)




				def        lowercase__     (  self		:					Any						):

								'''simple docstring'''
								lowercase__     				=    self.image_processing_class.from_dict(self.image_processor_dict						)
								self.assertEqual(image_processor.size,						{'''shortest_edge''': 20}						)
								self.assertEqual(image_processor.crop_size,						{'''height''': 18, '''width''': 18}						)

								lowercase__     				=    self.image_processing_class.from_dict(self.image_processor_dict,						size=42,						crop_size=84						)
								self.assertEqual(image_processor.size,						{'''shortest_edge''': 42}						)
								self.assertEqual(image_processor.crop_size,						{'''height''': 84, '''width''': 84}						)




				def        lowercase__     (  self		:					Optional[int]						):

								'''simple docstring'''
								pass




				def        lowercase__     (  self		:					Any						):

								'''simple docstring'''
								# Initialize image_processing
								lowercase__     				=    self.image_processing_class(**self.image_processor_dict						)
								# create random PIL images
								lowercase__     				=    prepare_image_inputs(self.image_processor_tester,						equal_resolution=lowerCamelCase						)
								for image in image_inputs:
												self.assertIsInstance(lowerCamelCase,						Image.Image						)

								# Test not batched input
								lowercase__     				=    image_processing(image_inputs[0],						return_tensors='''pt'''						).pixel_values
								self.assertEqual(
								    encoded_images.shape,						(
								        1,
								        self.image_processor_tester.num_channels,
								        self.image_processor_tester.crop_size['''height'''],
								        self.image_processor_tester.crop_size['''width'''],
								    ),						)

								# Test batched
								lowercase__     				=    image_processing(lowerCamelCase,						return_tensors='''pt'''						).pixel_values
								self.assertEqual(
								    encoded_images.shape,						(
								        self.image_processor_tester.batch_size,
								        self.image_processor_tester.num_channels,
								        self.image_processor_tester.crop_size['''height'''],
								        self.image_processor_tester.crop_size['''width'''],
								    ),						)




				def        lowercase__     (  self		:					str						):

								'''simple docstring'''
								# Initialize image_processing
								lowercase__     				=    self.image_processing_class(**self.image_processor_dict						)
								# create random numpy tensors
								lowercase__     				=    prepare_image_inputs(self.image_processor_tester,						equal_resolution=lowerCamelCase,						numpify=lowerCamelCase						)
								for image in image_inputs:
												self.assertIsInstance(lowerCamelCase,						np.ndarray						)

								# Test not batched input
								lowercase__     				=    image_processing(image_inputs[0],						return_tensors='''pt'''						).pixel_values
								self.assertEqual(
								    encoded_images.shape,						(
								        1,
								        self.image_processor_tester.num_channels,
								        self.image_processor_tester.crop_size['''height'''],
								        self.image_processor_tester.crop_size['''width'''],
								    ),						)

								# Test batched
								lowercase__     				=    image_processing(lowerCamelCase,						return_tensors='''pt'''						).pixel_values
								self.assertEqual(
								    encoded_images.shape,						(
								        self.image_processor_tester.batch_size,
								        self.image_processor_tester.num_channels,
								        self.image_processor_tester.crop_size['''height'''],
								        self.image_processor_tester.crop_size['''width'''],
								    ),						)




				def        lowercase__     (  self		:					str						):

								'''simple docstring'''
								# Initialize image_processing
								lowercase__     				=    self.image_processing_class(**self.image_processor_dict						)
								# create random PyTorch tensors
								lowercase__     				=    prepare_image_inputs(self.image_processor_tester,						equal_resolution=lowerCamelCase,						torchify=lowerCamelCase						)
								for image in image_inputs:
												self.assertIsInstance(lowerCamelCase,						torch.Tensor						)

								# Test not batched input
								lowercase__     				=    image_processing(image_inputs[0],						return_tensors='''pt'''						).pixel_values
								self.assertEqual(
								    encoded_images.shape,						(
								        1,
								        self.image_processor_tester.num_channels,
								        self.image_processor_tester.crop_size['''height'''],
								        self.image_processor_tester.crop_size['''width'''],
								    ),						)

								# Test batched
								lowercase__     				=    image_processing(lowerCamelCase,						return_tensors='''pt'''						).pixel_values
								self.assertEqual(
								    encoded_images.shape,						(
								        self.image_processor_tester.batch_size,
								        self.image_processor_tester.num_channels,
								        self.image_processor_tester.crop_size['''height'''],
								        self.image_processor_tester.crop_size['''width'''],
								    ),						)




