



import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


a_			=					get_tests_dir("""fixtures/test_sentencepiece.model""")



@require_sentencepiece
@require_tokenizers
class  __lowerCAmelCase     (	lowerCAmelCase__  ,	unittest.TestCase   ):
 lowerCAmelCase__       					=  ReformerTokenizer
 lowerCAmelCase__       					=  ReformerTokenizerFast
 lowerCAmelCase__       					=  True
 lowerCAmelCase__       					=  False
 lowerCAmelCase__       					=  True




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    super().setUp()

    __lowerCamelCase													=    ReformerTokenizer(__UpperCAmelCase	,				keep_accents=__UpperCAmelCase				)
    tokenizer.save_pretrained(self.tmpdirname				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    '''<s>'''
    __lowerCamelCase													=    1

    self.assertEqual(self.get_tokenizer()._convert_token_to_id(__UpperCAmelCase				)	,				__UpperCAmelCase				)
    self.assertEqual(self.get_tokenizer()._convert_id_to_token(__UpperCAmelCase				)	,				__UpperCAmelCase				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    list(self.get_tokenizer().get_vocab().keys()				)

    self.assertEqual(vocab_keys[0]	,				'''<unk>'''				)
    self.assertEqual(vocab_keys[1]	,				'''<s>'''				)
    self.assertEqual(vocab_keys[-1]	,				'''j'''				)
    self.assertEqual(len(__UpperCAmelCase				)	,				1000				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    self.assertEqual(self.get_tokenizer().vocab_size	,				1000				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    if not self.test_rust_tokenizer:
       return

    __lowerCamelCase													=    self.get_tokenizer()
    __lowerCamelCase													=    self.get_rust_tokenizer()

    __lowerCamelCase													=    '''I was born in 92000, and this is falsé.'''

    __lowerCamelCase													=    tokenizer.tokenize(__UpperCAmelCase				)
    __lowerCamelCase													=    rust_tokenizer.tokenize(__UpperCAmelCase				)
    self.assertListEqual(__UpperCAmelCase	,				__UpperCAmelCase				)

    __lowerCamelCase													=    tokenizer.encode(__UpperCAmelCase	,				add_special_tokens=__UpperCAmelCase				)
    __lowerCamelCase													=    rust_tokenizer.encode(__UpperCAmelCase	,				add_special_tokens=__UpperCAmelCase				)
    self.assertListEqual(__UpperCAmelCase	,				__UpperCAmelCase				)

    __lowerCamelCase													=    self.get_rust_tokenizer()
    __lowerCamelCase													=    tokenizer.encode(__UpperCAmelCase				)
    __lowerCamelCase													=    rust_tokenizer.encode(__UpperCAmelCase				)
    self.assertListEqual(__UpperCAmelCase	,				__UpperCAmelCase				)




 def        lowerCamelCase      (						self	,				__UpperCAmelCase=15				):



    '''simple docstring'''
    for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
          __lowerCamelCase													=    self.rust_tokenizer_class.from_pretrained(__UpperCAmelCase	,				**__UpperCAmelCase				)

          # Simple input
          __lowerCamelCase													=    '''This is a simple input'''
          __lowerCamelCase													=    ['''This is a simple input 1''', '''This is a simple input 2''']
          __lowerCamelCase													=    ('''This is a simple input''', '''This is a pair''')
          __lowerCamelCase													=    [
              ('''This is a simple input 1''', '''This is a simple input 2'''),
              ('''This is a simple pair 1''', '''This is a simple pair 2'''),
          ]

          # Simple input tests
          self.assertRaises(__UpperCAmelCase	,				tokenizer_r.encode	,				__UpperCAmelCase	,				max_length=__UpperCAmelCase	,				padding='''max_length'''				)

          # Simple input
          self.assertRaises(__UpperCAmelCase	,				tokenizer_r.encode_plus	,				__UpperCAmelCase	,				max_length=__UpperCAmelCase	,				padding='''max_length'''				)

          # Simple input
          self.assertRaises(
              __UpperCAmelCase	,				tokenizer_r.batch_encode_plus	,				__UpperCAmelCase	,				max_length=__UpperCAmelCase	,				padding='''max_length'''	,				)

          # Pair input
          self.assertRaises(__UpperCAmelCase	,				tokenizer_r.encode	,				__UpperCAmelCase	,				max_length=__UpperCAmelCase	,				padding='''max_length'''				)

          # Pair input
          self.assertRaises(__UpperCAmelCase	,				tokenizer_r.encode_plus	,				__UpperCAmelCase	,				max_length=__UpperCAmelCase	,				padding='''max_length'''				)

          # Pair input
          self.assertRaises(
              __UpperCAmelCase	,				tokenizer_r.batch_encode_plus	,				__UpperCAmelCase	,				max_length=__UpperCAmelCase	,				padding='''max_length'''	,				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    pass




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    ReformerTokenizer(__UpperCAmelCase	,				keep_accents=__UpperCAmelCase				)

    __lowerCamelCase													=    tokenizer.tokenize('''This is a test'''				)
    self.assertListEqual(__UpperCAmelCase	,				['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']				)

    self.assertListEqual(
        tokenizer.convert_tokens_to_ids(__UpperCAmelCase				)	,				[285, 46, 10, 170, 382]	,				)

    __lowerCamelCase													=    tokenizer.tokenize('''I was born in 92000, and this is falsé.'''				)
    self.assertListEqual(
        __UpperCAmelCase	,				[
            SPIECE_UNDERLINE + '''I''',
            SPIECE_UNDERLINE + '''was''',
            SPIECE_UNDERLINE + '''b''',
            '''or''',
            '''n''',
            SPIECE_UNDERLINE + '''in''',
            SPIECE_UNDERLINE + '''''',
            '''9''',
            '''2''',
            '''0''',
            '''0''',
            '''0''',
            ''',''',
            SPIECE_UNDERLINE + '''and''',
            SPIECE_UNDERLINE + '''this''',
            SPIECE_UNDERLINE + '''is''',
            SPIECE_UNDERLINE + '''f''',
            '''al''',
            '''s''',
            '''é''',
            '''.''',
        ]	,				)
    __lowerCamelCase													=    tokenizer.convert_tokens_to_ids(__UpperCAmelCase				)
    self.assertListEqual(
        __UpperCAmelCase	,				[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]	,				)

    __lowerCamelCase													=    tokenizer.convert_ids_to_tokens(__UpperCAmelCase				)
    self.assertListEqual(
        __UpperCAmelCase	,				[
            SPIECE_UNDERLINE + '''I''',
            SPIECE_UNDERLINE + '''was''',
            SPIECE_UNDERLINE + '''b''',
            '''or''',
            '''n''',
            SPIECE_UNDERLINE + '''in''',
            SPIECE_UNDERLINE + '''''',
            '''<unk>''',
            '''2''',
            '''0''',
            '''0''',
            '''0''',
            ''',''',
            SPIECE_UNDERLINE + '''and''',
            SPIECE_UNDERLINE + '''this''',
            SPIECE_UNDERLINE + '''is''',
            SPIECE_UNDERLINE + '''f''',
            '''al''',
            '''s''',
            '''<unk>''',
            '''.''',
        ]	,				)




 @cached_property
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    return ReformerTokenizer.from_pretrained('''google/reformer-crime-and-punishment'''				)




 @slow
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    '''Hello World!'''
    __lowerCamelCase													=    [126, 32, 262, 152, 38, 72, 287]

    self.assertListEqual(__UpperCAmelCase	,				self.big_tokenizer.encode(__UpperCAmelCase				)				)




 @slow
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    (
        '''This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'''
        ''' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'''
    )
    __lowerCamelCase													=    [
        108,
        265,
        24,
        111,
        4,
        258,
        156,
        35,
        28,
        275,
        3,
        259,
        297,
        260,
        84,
        4,
        35,
        110,
        44,
        8,
        259,
        91,
        268,
        21,
        11,
        209,
        274,
        109,
        266,
        277,
        117,
        86,
        93,
        315,
        258,
        278,
        258,
        277,
        258,
        0,
        258,
        288,
        258,
        319,
        258,
        0,
        258,
        0,
        258,
        0,
        258,
        0,
        258,
        287,
        258,
        315,
        258,
        289,
        258,
        278,
        99,
        269,
        266,
        262,
        8,
        259,
        241,
        4,
        217,
        230,
        268,
        266,
        55,
        168,
        106,
        75,
        193,
        266,
        223,
        27,
        49,
        26,
        282,
        25,
        264,
        299,
        19,
        26,
        0,
        258,
        277,
        117,
        86,
        93,
        176,
        183,
        270,
        11,
        262,
        42,
        61,
        265,
    ]

    self.assertListEqual(__UpperCAmelCase	,				self.big_tokenizer.encode(__UpperCAmelCase				)				)




 @require_torch
 @slow
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    import torch

    from transformers import ReformerConfig, ReformerModel

    # Build sequence
    __lowerCamelCase													=    list(self.big_tokenizer.get_vocab().keys()				)[:10]
    __lowerCamelCase													=    ''' '''.join(__UpperCAmelCase				)
    __lowerCamelCase													=    self.big_tokenizer.encode_plus(__UpperCAmelCase	,				return_tensors='''pt'''				)
    __lowerCamelCase													=    self.big_tokenizer.batch_encode_plus([sequence, sequence]	,				return_tensors='''pt'''				)

    __lowerCamelCase													=    ReformerConfig()
    # The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
    __lowerCamelCase													=    encoded_sequence['''input_ids'''].shape
    __lowerCamelCase													=    ReformerModel(__UpperCAmelCase				)

    # Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
    assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

    with torch.no_grad():
       model(**__UpperCAmelCase				)
       model(**__UpperCAmelCase				)




 @slow
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    # fmt: off
    __lowerCamelCase													=    {'''input_ids''': [[108, 265, 24, 111, 4, 258, 156, 7, 51, 279, 58, 7, 76, 25, 69, 278], [140, 243, 264, 134, 17, 267, 77, 263, 22, 262, 297, 258, 304, 177, 279, 266, 14, 89, 13, 35, 261, 299, 272, 137, 275, 278]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
    # fmt: on

    # This tokenizer does not know some characters like ")".
    # That is the reason why we use very simple texts here.
    # Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
    __lowerCamelCase													=    [
        '''This is a very simple sentence.''',
        '''The quick brown fox jumps over the lazy dog.''',
    ]

    self.tokenizer_integration_test_util(
        expected_encoding=__UpperCAmelCase	,				model_name='''google/reformer-crime-and-punishment'''	,				revision='''0e6c3decb8211d49bf881013425dc8b0448b3f5a'''	,				padding=__UpperCAmelCase	,				sequences=__UpperCAmelCase	,				)











import gc
import random
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTextModelWithProjection, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DiffusionPipeline,
    EulerDiscreteScheduler,
    StableDiffusionXLImgaImgPipeline,
    UNetaDConditionModel,
)
from diffusers.utils import floats_tensor, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..pipeline_params import (
    IMAGE_TO_IMAGE_IMAGE_PARAMS,
    TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS,
    TEXT_GUIDED_IMAGE_VARIATION_PARAMS,
)
from ..test_pipelines_common import PipelineLatentTesterMixin, PipelineTesterMixin


enable_full_determinism()



class  __lowerCAmelCase     (	lowerCAmelCase__  ,	lowerCAmelCase__  ,	unittest.TestCase   ):
 lowerCAmelCase__       					=  StableDiffusionXLImgaImgPipeline
 lowerCAmelCase__       					=  TEXT_GUIDED_IMAGE_VARIATION_PARAMS - {"""height""", """width"""}
 lowerCAmelCase__       					=  PipelineTesterMixin.required_optional_params - {"""latents"""}
 lowerCAmelCase__       					=  TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS
 lowerCAmelCase__       					=  IMAGE_TO_IMAGE_IMAGE_PARAMS
 lowerCAmelCase__       					=  IMAGE_TO_IMAGE_IMAGE_PARAMS




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    torch.manual_seed(0				)
    __lowerCamelCase													=    UNetaDConditionModel(
        block_out_channels=(32, 64)	,				layers_per_block=2	,				sample_size=32	,				in_channels=4	,				out_channels=4	,				down_block_types=('''DownBlock2D''', '''CrossAttnDownBlock2D''')	,				up_block_types=('''CrossAttnUpBlock2D''', '''UpBlock2D''')	,				attention_head_dim=(2, 4)	,				use_linear_projection=__UpperCAmelCase	,				addition_embed_type='''text_time'''	,				addition_time_embed_dim=8	,				transformer_layers_per_block=(1, 2)	,				projection_class_embeddings_input_dim=80	,				cross_attention_dim=64	,				)
    __lowerCamelCase													=    EulerDiscreteScheduler(
        beta_start=0.00_085	,				beta_end=0.012	,				steps_offset=1	,				beta_schedule='''scaled_linear'''	,				timestep_spacing='''leading'''	,				)
    torch.manual_seed(0				)
    __lowerCamelCase													=    AutoencoderKL(
        block_out_channels=[32, 64]	,				in_channels=3	,				out_channels=3	,				down_block_types=['''DownEncoderBlock2D''', '''DownEncoderBlock2D''']	,				up_block_types=['''UpDecoderBlock2D''', '''UpDecoderBlock2D''']	,				latent_channels=4	,				sample_size=128	,				)
    torch.manual_seed(0				)
    __lowerCamelCase													=    CLIPTextConfig(
        bos_token_id=0	,				eos_token_id=2	,				hidden_size=32	,				intermediate_size=37	,				layer_norm_eps=1E-0_5	,				num_attention_heads=4	,				num_hidden_layers=5	,				pad_token_id=1	,				vocab_size=1000	,				hidden_act='''gelu'''	,				projection_dim=32	,				)
    __lowerCamelCase													=    CLIPTextModel(__UpperCAmelCase				)
    __lowerCamelCase													=    CLIPTokenizer.from_pretrained('''hf-internal-testing/tiny-random-clip'''	,				local_files_only=__UpperCAmelCase				)

    __lowerCamelCase													=    CLIPTextModelWithProjection(__UpperCAmelCase				)
    __lowerCamelCase													=    CLIPTokenizer.from_pretrained('''hf-internal-testing/tiny-random-clip'''	,				local_files_only=__UpperCAmelCase				)

    __lowerCamelCase													=    {
        '''unet''': unet,
        '''scheduler''': scheduler,
        '''vae''': vae,
        '''text_encoder''': text_encoder,
        '''tokenizer''': tokenizer,
        '''text_encoder_2''': text_encoder_a,
        '''tokenizer_2''': tokenizer_a,
        # "safety_checker": None,
        # "feature_extractor": None,
    }
    return components




 def        lowerCamelCase      (						self	,				__UpperCAmelCase	,				__UpperCAmelCase=0				):



    '''simple docstring'''
    __lowerCamelCase													=    floats_tensor((1, 3, 32, 32)	,				rng=random.Random(__UpperCAmelCase				)				).to(__UpperCAmelCase				)
    __lowerCamelCase													=    image / 2 + 0.5
    if str(__UpperCAmelCase				).startswith('''mps'''				):
       __lowerCamelCase													=    torch.manual_seed(__UpperCAmelCase				)
    else:
       __lowerCamelCase													=    torch.Generator(device=__UpperCAmelCase				).manual_seed(__UpperCAmelCase				)
    __lowerCamelCase													=    {
        '''prompt''': '''A painting of a squirrel eating a burger''',
        '''image''': image,
        '''generator''': generator,
        '''num_inference_steps''': 2,
        '''guidance_scale''': 5.0,
        '''output_type''': '''numpy''',
        '''strength''': 0.75,
    }
    return inputs




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    '''cpu'''  # ensure determinism for the device-dependent torch.Generator
    __lowerCamelCase													=    self.get_dummy_components()
    __lowerCamelCase													=    StableDiffusionXLImgaImgPipeline(**__UpperCAmelCase				)
    __lowerCamelCase													=    sd_pipe.to(__UpperCAmelCase				)
    sd_pipe.set_progress_bar_config(disable=__UpperCAmelCase				)

    __lowerCamelCase													=    self.get_dummy_inputs(__UpperCAmelCase				)
    __lowerCamelCase													=    sd_pipe(**__UpperCAmelCase				).images
    __lowerCamelCase													=    image[0, -3:, -3:, -1]

    assert image.shape == (1, 32, 32, 3)

    __lowerCamelCase													=    np.array([0.4_656, 0.4_840, 0.4_439, 0.6_698, 0.5_574, 0.4_524, 0.5_799, 0.5_943, 0.5_165]				)

    assert np.abs(image_slice.flatten() - expected_slice				).max() < 1E-2




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    super().test_attention_slicing_forward_pass(expected_max_diff=3E-3				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    super().test_inference_batch_single_identical(expected_max_diff=3E-3				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    pass




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    self.get_dummy_components()
    __lowerCamelCase													=    StableDiffusionXLImgaImgPipeline(**__UpperCAmelCase				)
    __lowerCamelCase													=    sd_pipe.to(__UpperCAmelCase				)
    __lowerCamelCase													=    sd_pipe.to(__UpperCAmelCase				)
    sd_pipe.set_progress_bar_config(disable=__UpperCAmelCase				)

    # forward without prompt embeds
    __lowerCamelCase													=    self.get_dummy_inputs(__UpperCAmelCase				)
    __lowerCamelCase													=    3 * ['''this is a negative prompt''']
    __lowerCamelCase													=    negative_prompt
    __lowerCamelCase													=    3 * [inputs['''prompt''']]

    __lowerCamelCase													=    sd_pipe(**__UpperCAmelCase				)
    __lowerCamelCase													=    output.images[0, -3:, -3:, -1]

    # forward with prompt embeds
    __lowerCamelCase													=    self.get_dummy_inputs(__UpperCAmelCase				)
    __lowerCamelCase													=    3 * ['''this is a negative prompt''']
    __lowerCamelCase													=    3 * [inputs.pop('''prompt'''				)]

    (
        (
        __lowerCamelCase
    )							,(
        __lowerCamelCase
    )							,(
        __lowerCamelCase
    )							,(
        __lowerCamelCase
    )							,
    )													=    sd_pipe.encode_prompt(__UpperCAmelCase	,				negative_prompt=__UpperCAmelCase				)

    __lowerCamelCase													=    sd_pipe(
        **__UpperCAmelCase	,				prompt_embeds=__UpperCAmelCase	,				negative_prompt_embeds=__UpperCAmelCase	,				pooled_prompt_embeds=__UpperCAmelCase	,				negative_pooled_prompt_embeds=__UpperCAmelCase	,				)
    __lowerCamelCase													=    output.images[0, -3:, -3:, -1]

    # make sure that it's equal
    assert np.abs(image_slice_a.flatten() - image_slice_a.flatten()				).max() < 1E-4



@slow
@require_torch_gpu
class  __lowerCAmelCase     (	unittest.TestCase   ):




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    super().tearDown()
    gc.collect()
    torch.cuda.empty_cache()




 def        lowerCamelCase      (						self	,				__UpperCAmelCase	,				__UpperCAmelCase="cpu"	,				__UpperCAmelCase=torch.floataa	,				__UpperCAmelCase=0				):



    '''simple docstring'''
    __lowerCamelCase													=    torch.Generator(device=__UpperCAmelCase				).manual_seed(__UpperCAmelCase				)
    __lowerCamelCase													=    np.random.RandomState(__UpperCAmelCase				).standard_normal((1, 4, 64, 64)				)
    __lowerCamelCase													=    torch.from_numpy(__UpperCAmelCase				).to(device=__UpperCAmelCase	,				dtype=__UpperCAmelCase				)
    __lowerCamelCase													=    {
        '''prompt''': '''a photograph of an astronaut riding a horse''',
        '''latents''': latents,
        '''generator''': generator,
        '''num_inference_steps''': 3,
        '''guidance_scale''': 7.5,
        '''output_type''': '''numpy''',
    }
    return inputs




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    DiffusionPipeline.from_pretrained('''stabilityai/stable-diffusion-2-base'''				)
    pipe.to(__UpperCAmelCase				)
    pipe.set_progress_bar_config(disable=__UpperCAmelCase				)

    __lowerCamelCase													=    self.get_inputs(__UpperCAmelCase				)
    __lowerCamelCase													=    pipe(**__UpperCAmelCase				).images
    __lowerCamelCase													=    image[0, -3:, -3:, -1].flatten()

    assert image.shape == (1, 512, 512, 3)
    __lowerCamelCase													=    np.array([0.49_493, 0.47_896, 0.40_798, 0.54_214, 0.53_212, 0.48_202, 0.47_656, 0.46_329, 0.48_506]				)
    assert np.abs(image_slice - expected_slice				).max() < 7E-3






