


import unittest

from transformers import BigBirdTokenizer, BigBirdTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


a					  =     '▁'

a					  =     get_tests_dir('fixtures/test_sentencepiece.model')







@require_sentencepiece
@require_tokenizers
class        UpperCamelCase__   (	__magic_name__							,				unittest.TestCase   ):
					__SCREAMING_SNAKE_CASE       :  str						=   BigBirdTokenizer
					__SCREAMING_SNAKE_CASE       :  Optional[Any]						=   BigBirdTokenizerFast
					__SCREAMING_SNAKE_CASE       :  List[str]						=   True
					__SCREAMING_SNAKE_CASE       :  Tuple						=   True

					def   UpperCAmelCase__       (     self       :   Dict       ):






										'''simple docstring'''






										super().setUp()

										lowercase_  							=			self.tokenizer_class(UpperCamelCase__							,					keep_accents=UpperCamelCase__       )
										tokenizer.save_pretrained(self.tmpdirname       )

					def   UpperCAmelCase__       (     self       :   Dict       ):






										'''simple docstring'''






										lowercase_  							=			"""<s>"""
										lowercase_  							=			1

										self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase__       )							,					UpperCamelCase__       )
										self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase__       )							,					UpperCamelCase__       )

					def   UpperCAmelCase__       (     self       :   List[str]       ):






										'''simple docstring'''






										lowercase_  							=			list(self.get_tokenizer().get_vocab().keys()       )

										self.assertEqual(vocab_keys[0]							,					"""<unk>"""       )
										self.assertEqual(vocab_keys[1]							,					"""<s>"""       )
										self.assertEqual(vocab_keys[-1]							,					"""[MASK]"""       )
										self.assertEqual(len(UpperCamelCase__       )							,					1_004       )

					def   UpperCAmelCase__       (     self       :   Dict       ):






										'''simple docstring'''






										self.assertEqual(self.get_tokenizer().vocab_size							,					1_000       )

					def   UpperCAmelCase__       (     self       :   Dict       ):






										'''simple docstring'''






										if not self.test_rust_tokenizer:
															return

										lowercase_  							=			self.get_tokenizer()
										lowercase_  							=			self.get_rust_tokenizer()

										lowercase_  							=			"""I was born in 92000, and this is falsé."""

										lowercase_  							=			tokenizer.tokenize(UpperCamelCase__       )
										lowercase_  							=			rust_tokenizer.tokenize(UpperCamelCase__       )
										self.assertListEqual(UpperCamelCase__							,					UpperCamelCase__       )

										lowercase_  							=			tokenizer.encode(UpperCamelCase__							,					add_special_tokens=UpperCamelCase__       )
										lowercase_  							=			rust_tokenizer.encode(UpperCamelCase__							,					add_special_tokens=UpperCamelCase__       )
										self.assertListEqual(UpperCamelCase__							,					UpperCamelCase__       )

										lowercase_  							=			self.get_rust_tokenizer()
										lowercase_  							=			tokenizer.encode(UpperCamelCase__       )
										lowercase_  							=			rust_tokenizer.encode(UpperCamelCase__       )
										self.assertListEqual(UpperCamelCase__							,					UpperCamelCase__       )

					def   UpperCAmelCase__       (     self       :   str       ):






										'''simple docstring'''






										lowercase_  							=			BigBirdTokenizer(UpperCamelCase__							,					keep_accents=UpperCamelCase__       )

										lowercase_  							=			tokenizer.tokenize("""This is a test"""       )
										self.assertListEqual(UpperCamelCase__							,					["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]       )

										self.assertListEqual(
										    tokenizer.convert_tokens_to_ids(UpperCamelCase__       )							,					[285, 46, 10, 170, 382]							,					)

										lowercase_  							=			tokenizer.tokenize("""I was born in 92000, and this is falsé."""       )
										self.assertListEqual(
										    UpperCamelCase__							,					[
										        SPIECE_UNDERLINE + """I""",
										        SPIECE_UNDERLINE + """was""",
										        SPIECE_UNDERLINE + """b""",
										        """or""",
										        """n""",
										        SPIECE_UNDERLINE + """in""",
										        SPIECE_UNDERLINE + """""",
										        """9""",
										        """2""",
										        """0""",
										        """0""",
										        """0""",
										        """,""",
										        SPIECE_UNDERLINE + """and""",
										        SPIECE_UNDERLINE + """this""",
										        SPIECE_UNDERLINE + """is""",
										        SPIECE_UNDERLINE + """f""",
										        """al""",
										        """s""",
										        """é""",
										        """.""",
										    ]							,					)
										lowercase_  							=			tokenizer.convert_tokens_to_ids(UpperCamelCase__       )
										self.assertListEqual(
										    UpperCamelCase__							,					[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]							,					)

										lowercase_  							=			tokenizer.convert_ids_to_tokens(UpperCamelCase__       )
										self.assertListEqual(
										    UpperCamelCase__							,					[
										        SPIECE_UNDERLINE + """I""",
										        SPIECE_UNDERLINE + """was""",
										        SPIECE_UNDERLINE + """b""",
										        """or""",
										        """n""",
										        SPIECE_UNDERLINE + """in""",
										        SPIECE_UNDERLINE + """""",
										        """<unk>""",
										        """2""",
										        """0""",
										        """0""",
										        """0""",
										        """,""",
										        SPIECE_UNDERLINE + """and""",
										        SPIECE_UNDERLINE + """this""",
										        SPIECE_UNDERLINE + """is""",
										        SPIECE_UNDERLINE + """f""",
										        """al""",
										        """s""",
										        """<unk>""",
										        """.""",
										    ]							,					)

					@cached_property
					def   UpperCAmelCase__       (     self       :   Any       ):






										'''simple docstring'''






										return BigBirdTokenizer.from_pretrained("""google/bigbird-roberta-base"""       )

					@slow
					def   UpperCAmelCase__       (     self       :   Optional[int]       ):






										'''simple docstring'''






										lowercase_  							=			"""Hello World!"""
										lowercase_  							=			[65, 18_536, 2_260, 101, 66]

										self.assertListEqual(UpperCamelCase__							,					self.big_tokenizer.encode(UpperCamelCase__       )       )

					@slow
					def   UpperCAmelCase__       (     self       :   str       ):






										'''simple docstring'''






										lowercase_  							=			(
										    """This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"""
										    """ add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"""
										)
										# fmt: off
										lowercase_  							=			[65, 871, 419, 358, 946, 991, 2_521, 452, 358, 1_357, 387, 7_751, 3_536, 112, 985, 456, 126, 865, 938, 5_400, 5_734, 458, 1_368, 467, 786, 2_462, 5_246, 1_159, 633, 865, 4_519, 457, 582, 852, 2_557, 427, 916, 508, 405, 34_324, 497, 391, 408, 11_342, 1_244, 385, 100, 938, 985, 456, 574, 362, 12_597, 3_200, 3_129, 1_172, 66]  # noqa: E231
										# fmt: on
										self.assertListEqual(UpperCamelCase__							,					self.big_tokenizer.encode(UpperCamelCase__       )       )

					@require_torch
					@slow
					def   UpperCAmelCase__       (     self       :   Tuple       ):






										'''simple docstring'''






										import torch

										from transformers import BigBirdConfig, BigBirdModel

										# Build sequence
										lowercase_  							=			list(self.big_tokenizer.get_vocab().keys()       )[:10]
										lowercase_  							=			""" """.join(UpperCamelCase__       )
										lowercase_  							=			self.big_tokenizer.encode_plus(UpperCamelCase__							,					return_tensors="""pt"""							,					return_token_type_ids=UpperCamelCase__       )
										lowercase_  							=			self.big_tokenizer.batch_encode_plus(
										    [sequence + """ """ + sequence]							,					return_tensors="""pt"""							,					return_token_type_ids=UpperCamelCase__       )

										lowercase_  							=			BigBirdConfig(attention_type="""original_full"""       )
										lowercase_  							=			BigBirdModel(UpperCamelCase__       )

										assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

										with torch.no_grad():
															model(**UpperCamelCase__       )
															model(**UpperCamelCase__       )

					@slow
					def   UpperCAmelCase__       (     self       :   int       ):






										'''simple docstring'''






										lowercase_  							=			BigBirdTokenizer.from_pretrained("""google/bigbird-roberta-base"""       )
										lowercase_  							=			tokenizer.decode(tokenizer("""Paris is the [MASK]."""       ).input_ids       )

										self.assertTrue(decoded_text == """[CLS] Paris is the[MASK].[SEP]"""       )







					@slow
					def   UpperCAmelCase__       (     self       :   Optional[int]       ):






										'''simple docstring'''






										lowercase_  							=			{"""input_ids""": [[65, 39_286, 458, 36_335, 2_001, 456, 13_073, 13_266, 455, 113, 7_746, 1_741, 11_157, 391, 13_073, 13_266, 455, 113, 3_967, 35_412, 113, 4_936, 109, 3_870, 2_377, 113, 30_084, 45_720, 458, 134, 17_496, 112, 503, 11_672, 113, 118, 112, 5_665, 13_347, 38_687, 112, 1_496, 31_389, 112, 3_268, 47_264, 134, 962, 112, 16_377, 8_035, 23_130, 430, 12_169, 15_518, 28_592, 458, 146, 41_697, 109, 391, 12_169, 15_518, 16_689, 458, 146, 41_358, 109, 452, 726, 4_034, 111, 763, 35_412, 5_082, 388, 1_903, 111, 9_051, 391, 2_870, 48_918, 1_900, 1_123, 550, 998, 112, 9_586, 15_985, 455, 391, 410, 22_955, 37_636, 114, 66], [65, 448, 17_496, 419, 3_663, 385, 763, 113, 27_533, 2_870, 3_283, 13_043, 1_639, 24_713, 523, 656, 24_013, 18_550, 2_521, 517, 27_014, 21_244, 420, 1_212, 1_465, 391, 927, 4_833, 388, 578, 11_786, 114, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [65, 484, 2_169, 7_687, 21_932, 18_146, 726, 363, 17_032, 3_391, 114, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
										# fmt: on

										self.tokenizer_integration_test_util(
										    expected_encoding=UpperCamelCase__							,					model_name="""google/bigbird-roberta-base"""							,					revision="""215c99f1600e06f83acce68422f2035b2b5c3510"""							,					)






from typing import Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature
from ...image_transforms import get_image_size, pad, rescale, to_channel_dimension_format
from ...image_utils import ChannelDimension, ImageInput, make_list_of_images, to_numpy_array, valid_images
from ...utils import TensorType, logging


a					  =     logging.get_logger(__name__)







class        UpperCamelCase__   (	__magic_name__   ):
					__SCREAMING_SNAKE_CASE       :  str						=   ['pixel_values']

					def __init__(     self       :   List[Any]							,					UpperCamelCase__       :   bool = True							,					UpperCamelCase__       :   Union[int, float] = 1 / 255							,					UpperCamelCase__       :   bool = True							,					UpperCamelCase__       :   int = 8							,					**UpperCamelCase__       :   Optional[Any]							,					):






										'''simple docstring'''






										super().__init__(**UpperCamelCase__       )

										lowercase_  							=			do_rescale
										lowercase_  							=			rescale_factor
										lowercase_  							=			do_pad
										lowercase_  							=			pad_size

					def   UpperCAmelCase__       (     self       :   int							,					UpperCamelCase__       :   np.ndarray							,					UpperCamelCase__       :   float							,					UpperCamelCase__       :   Optional[Union[str, ChannelDimension]] = None							,					**UpperCamelCase__       :   Optional[Any]       ):






										'''simple docstring'''






										return rescale(UpperCamelCase__							,					scale=UpperCamelCase__							,					data_format=UpperCamelCase__							,					**UpperCamelCase__       )

					def   UpperCAmelCase__       (     self       :   List[Any]							,					UpperCamelCase__       :   np.ndarray							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   Optional[Union[str, ChannelDimension]] = None       ):






										'''simple docstring'''






										lowercase_							,					lowercase_  							=			get_image_size(UpperCamelCase__       )
										lowercase_  							=			(old_height // size + 1) * size - old_height
										lowercase_  							=			(old_width // size + 1) * size - old_width

										return pad(UpperCamelCase__							,					((0, pad_height), (0, pad_width))							,					mode="""symmetric"""							,					data_format=UpperCamelCase__       )







					def   UpperCAmelCase__       (     self       :   Optional[int]							,					UpperCamelCase__       :   ImageInput							,					UpperCamelCase__       :   Optional[bool] = None							,					UpperCamelCase__       :   Optional[float] = None							,					UpperCamelCase__       :   Optional[bool] = None							,					UpperCamelCase__       :   Optional[int] = None							,					UpperCamelCase__       :   Optional[Union[str, TensorType]] = None							,					UpperCamelCase__       :   Union[str, ChannelDimension] = ChannelDimension.FIRST							,					**UpperCamelCase__       :   Dict							,					):






										'''simple docstring'''






										lowercase_  							=			do_rescale if do_rescale is not None else self.do_rescale
										lowercase_  							=			rescale_factor if rescale_factor is not None else self.rescale_factor
										lowercase_  							=			do_pad if do_pad is not None else self.do_pad
										lowercase_  							=			pad_size if pad_size is not None else self.pad_size

										lowercase_  							=			make_list_of_images(UpperCamelCase__       )

										if not valid_images(UpperCamelCase__       ):
															raise ValueError(
															    """Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, """
															    """torch.Tensor, tf.Tensor or jax.ndarray."""       )

										if do_rescale and rescale_factor is None:
															raise ValueError("""Rescale factor must be specified if do_rescale is True."""       )

										# All transformations expect numpy arrays.
										lowercase_  							=			[to_numpy_array(UpperCamelCase__       ) for image in images]

										if do_rescale:
															lowercase_  							=			[self.rescale(image=UpperCamelCase__							,					scale=UpperCamelCase__       ) for image in images]

										if do_pad:
															lowercase_  							=			[self.pad(UpperCamelCase__							,					size=UpperCamelCase__       ) for image in images]

										lowercase_  							=			[to_channel_dimension_format(UpperCamelCase__							,					UpperCamelCase__       ) for image in images]

										lowercase_  							=			{"""pixel_values""": images}
										return BatchFeature(data=UpperCamelCase__							,					tensor_type=UpperCamelCase__       )


