


import gc
import random
import tempfile
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import AutoencoderKL, DDIMScheduler, LMSDiscreteScheduler, PNDMScheduler, UNetaDConditionModel
from diffusers.pipelines.stable_diffusion_safe import StableDiffusionPipelineSafe as StableDiffusionPipeline
from diffusers.utils import floats_tensor, nightly, torch_device
from diffusers.utils.testing_utils import require_torch_gpu





class  lowerCAmelCase__(    unittest.TestCase						):






    '''simple docstring'''





    def 					_lowerCamelCase (	self				:   int	):





       '''simple docstring'''
       # clean up the VRAM after each test
       super().tearDown()
       gc.collect()
       torch.cuda.empty_cache()





    @property
    def 					_lowerCamelCase (	self				:   int	):





       '''simple docstring'''
       UpperCAmelCase_							:      str						=   1
       UpperCAmelCase_							:      Any						=   3
       UpperCAmelCase_							:      List[str]						=   (32, 32)

       UpperCAmelCase_							:      Any						=   floats_tensor((batch_size, num_channels) + sizes       ,   rng=random.Random(0	)	).to(__snake_case	)
       return image





    @property
    def 					_lowerCamelCase (	self				:   Tuple	):





       '''simple docstring'''
       torch.manual_seed(0	)
       UpperCAmelCase_							:      Union[str, Any]						=   UNetaDConditionModel(
           block_out_channels=(32, 64)       ,   layers_per_block=2       ,   sample_size=32       ,   in_channels=4       ,   out_channels=4       ,   down_block_types=('''DownBlock2D''', '''CrossAttnDownBlock2D''')       ,   up_block_types=('''CrossAttnUpBlock2D''', '''UpBlock2D''')       ,   cross_attention_dim=32       ,   )
       return model





    @property
    def 					_lowerCamelCase (	self				:   Optional[int]	):





       '''simple docstring'''
       torch.manual_seed(0	)
       UpperCAmelCase_							:      str						=   AutoencoderKL(
           block_out_channels=[32, 64]       ,   in_channels=3       ,   out_channels=3       ,   down_block_types=['''DownEncoderBlock2D''', '''DownEncoderBlock2D''']       ,   up_block_types=['''UpDecoderBlock2D''', '''UpDecoderBlock2D''']       ,   latent_channels=4       ,   )
       return model





    @property
    def 					_lowerCamelCase (	self				:   Tuple	):





       '''simple docstring'''
       torch.manual_seed(0	)
       UpperCAmelCase_							:      Any						=   CLIPTextConfig(
           bos_token_id=0       ,   eos_token_id=2       ,   hidden_size=32       ,   intermediate_size=37       ,   layer_norm_eps=1E-05       ,   num_attention_heads=4       ,   num_hidden_layers=5       ,   pad_token_id=1       ,   vocab_size=1_000       ,   )
       return CLIPTextModel(__snake_case	)





    @property
    def 					_lowerCamelCase (	self				:   Any	):





       '''simple docstring'''
       def extract(*__snake_case				:   Any       ,   **__snake_case				:   List[str]	):





          class  lowerCAmelCase__:






              '''simple docstring'''





              def __init__(	self				:   Union[str, Any]	):





                 '''simple docstring'''
                 UpperCAmelCase_							:      Dict						=   torch.ones([0]	)





              def 					_lowerCamelCase (	self				:   Optional[Any]       ,   __snake_case				:   List[Any]	):





                 '''simple docstring'''
                 self.pixel_values.to(__snake_case	)
                 return self





          return Out()

       return extract





    def 					_lowerCamelCase (	self				:   Optional[Any]	):





       '''simple docstring'''
       UpperCAmelCase_							:      Tuple						=   '''cpu'''  # ensure determinism for the device-dependent torch.Generator
       UpperCAmelCase_							:      List[str]						=   self.dummy_cond_unet
       UpperCAmelCase_							:      Tuple						=   DDIMScheduler(
           beta_start=0.00_085       ,   beta_end=0.012       ,   beta_schedule='''scaled_linear'''       ,   clip_sample=__snake_case       ,   set_alpha_to_one=__snake_case       ,   )

       UpperCAmelCase_							:      int						=   self.dummy_vae
       UpperCAmelCase_							:      Dict						=   self.dummy_text_encoder
       UpperCAmelCase_							:      Any						=   CLIPTokenizer.from_pretrained('''hf-internal-testing/tiny-random-clip'''	)

       # make sure here that pndm scheduler skips prk
       UpperCAmelCase_							:      int						=   StableDiffusionPipeline(
           unet=__snake_case       ,   scheduler=__snake_case       ,   vae=__snake_case       ,   text_encoder=__snake_case       ,   tokenizer=__snake_case       ,   safety_checker=__snake_case       ,   feature_extractor=self.dummy_extractor       ,   )
       UpperCAmelCase_							:      int						=   sd_pipe.to(__snake_case	)
       sd_pipe.set_progress_bar_config(disable=__snake_case	)

       UpperCAmelCase_							:      int						=   '''A painting of a squirrel eating a burger'''

       UpperCAmelCase_							:      Optional[int]						=   torch.Generator(device=__snake_case	).manual_seed(0	)
       UpperCAmelCase_							:      Any						=   sd_pipe([prompt]       ,   generator=__snake_case       ,   guidance_scale=6.0       ,   num_inference_steps=2       ,   output_type='''np'''	)
       UpperCAmelCase_							:      Tuple						=   output.images

       UpperCAmelCase_							:      str						=   torch.Generator(device=__snake_case	).manual_seed(0	)
       UpperCAmelCase_							:      Optional[Any]						=   sd_pipe(
           [prompt]       ,   generator=__snake_case       ,   guidance_scale=6.0       ,   num_inference_steps=2       ,   output_type='''np'''       ,   return_dict=__snake_case       ,   )[0]

       UpperCAmelCase_							:      Any						=   image[0, -3:, -3:, -1]
       UpperCAmelCase_							:      int						=   image_from_tuple[0, -3:, -3:, -1]

       assert image.shape == (1, 64, 64, 3)
       UpperCAmelCase_							:      Optional[int]						=   np.array([0.5_756, 0.6_118, 0.5_005, 0.5_041, 0.5_471, 0.4_726, 0.4_976, 0.4_865, 0.4_864]	)

       assert np.abs(image_slice.flatten() - expected_slice	).max() < 1E-2
       assert np.abs(image_from_tuple_slice.flatten() - expected_slice	).max() < 1E-2





    def 					_lowerCamelCase (	self				:   Dict	):





       '''simple docstring'''
       UpperCAmelCase_							:      str						=   '''cpu'''  # ensure determinism for the device-dependent torch.Generator
       UpperCAmelCase_							:      int						=   self.dummy_cond_unet
       UpperCAmelCase_							:      int						=   PNDMScheduler(skip_prk_steps=__snake_case	)
       UpperCAmelCase_							:      List[str]						=   self.dummy_vae
       UpperCAmelCase_							:      Dict						=   self.dummy_text_encoder
       UpperCAmelCase_							:      str						=   CLIPTokenizer.from_pretrained('''hf-internal-testing/tiny-random-clip'''	)

       # make sure here that pndm scheduler skips prk
       UpperCAmelCase_							:      Optional[Any]						=   StableDiffusionPipeline(
           unet=__snake_case       ,   scheduler=__snake_case       ,   vae=__snake_case       ,   text_encoder=__snake_case       ,   tokenizer=__snake_case       ,   safety_checker=__snake_case       ,   feature_extractor=self.dummy_extractor       ,   )
       UpperCAmelCase_							:      Dict						=   sd_pipe.to(__snake_case	)
       sd_pipe.set_progress_bar_config(disable=__snake_case	)

       UpperCAmelCase_							:      Dict						=   '''A painting of a squirrel eating a burger'''
       UpperCAmelCase_							:      Optional[Any]						=   torch.Generator(device=__snake_case	).manual_seed(0	)
       UpperCAmelCase_							:      List[Any]						=   sd_pipe([prompt]       ,   generator=__snake_case       ,   guidance_scale=6.0       ,   num_inference_steps=2       ,   output_type='''np'''	)

       UpperCAmelCase_							:      Optional[int]						=   output.images

       UpperCAmelCase_							:      Tuple						=   torch.Generator(device=__snake_case	).manual_seed(0	)
       UpperCAmelCase_							:      List[Any]						=   sd_pipe(
           [prompt]       ,   generator=__snake_case       ,   guidance_scale=6.0       ,   num_inference_steps=2       ,   output_type='''np'''       ,   return_dict=__snake_case       ,   )[0]

       UpperCAmelCase_							:      int						=   image[0, -3:, -3:, -1]
       UpperCAmelCase_							:      Union[str, Any]						=   image_from_tuple[0, -3:, -3:, -1]

       assert image.shape == (1, 64, 64, 3)
       UpperCAmelCase_							:      List[Any]						=   np.array([0.5_125, 0.5_716, 0.4_828, 0.5_060, 0.5_650, 0.4_768, 0.5_185, 0.4_895, 0.4_993]	)

       assert np.abs(image_slice.flatten() - expected_slice	).max() < 1E-2
       assert np.abs(image_from_tuple_slice.flatten() - expected_slice	).max() < 1E-2





    def 					_lowerCamelCase (	self				:   str	):





       '''simple docstring'''
       UpperCAmelCase_							:      List[Any]						=   StableDiffusionPipeline.from_pretrained(
           '''hf-internal-testing/tiny-stable-diffusion-lms-pipe'''       ,   safety_checker=__snake_case	)
       assert isinstance(__snake_case       ,   __snake_case	)
       assert isinstance(pipe.scheduler       ,   __snake_case	)
       assert pipe.safety_checker is None

       UpperCAmelCase_							:      List[str]						=   pipe('''example prompt'''       ,   num_inference_steps=2	).images[0]
       assert image is not None

       # check that there's no error when saving a pipeline with one of the models being None
       with tempfile.TemporaryDirectory() as tmpdirname:
          pipe.save_pretrained(__snake_case	)
          UpperCAmelCase_							:      List[Any]						=   StableDiffusionPipeline.from_pretrained(__snake_case	)

       # sanity check that the pipeline still works
       assert pipe.safety_checker is None
       UpperCAmelCase_							:      int						=   pipe('''example prompt'''       ,   num_inference_steps=2	).images[0]
       assert image is not None





    @unittest.skipIf(torch_device != '''cuda'''       ,   '''This test requires a GPU'''	)
    def 					_lowerCamelCase (	self				:   Dict	):





       '''simple docstring'''
       UpperCAmelCase_							:      List[str]						=   self.dummy_cond_unet
       UpperCAmelCase_							:      Optional[int]						=   PNDMScheduler(skip_prk_steps=__snake_case	)
       UpperCAmelCase_							:      Tuple						=   self.dummy_vae
       UpperCAmelCase_							:      str						=   self.dummy_text_encoder
       UpperCAmelCase_							:      Optional[int]						=   CLIPTokenizer.from_pretrained('''hf-internal-testing/tiny-random-clip'''	)

       # put models in fp16
       UpperCAmelCase_							:      Tuple						=   unet.half()
       UpperCAmelCase_							:      Dict						=   vae.half()
       UpperCAmelCase_							:      Tuple						=   bert.half()

       # make sure here that pndm scheduler skips prk
       UpperCAmelCase_							:      Tuple						=   StableDiffusionPipeline(
           unet=__snake_case       ,   scheduler=__snake_case       ,   vae=__snake_case       ,   text_encoder=__snake_case       ,   tokenizer=__snake_case       ,   safety_checker=__snake_case       ,   feature_extractor=self.dummy_extractor       ,   )
       UpperCAmelCase_							:      Optional[Any]						=   sd_pipe.to(__snake_case	)
       sd_pipe.set_progress_bar_config(disable=__snake_case	)

       UpperCAmelCase_							:      Union[str, Any]						=   '''A painting of a squirrel eating a burger'''
       UpperCAmelCase_							:      Optional[Any]						=   sd_pipe([prompt]       ,   num_inference_steps=2       ,   output_type='''np'''	).images

       assert image.shape == (1, 64, 64, 3)





@nightly
@require_torch_gpu
class  lowerCAmelCase__(    unittest.TestCase						):






    '''simple docstring'''





    def 					_lowerCamelCase (	self				:   Optional[int]	):





       '''simple docstring'''
       # clean up the VRAM after each test
       super().tearDown()
       gc.collect()
       torch.cuda.empty_cache()





    def 					_lowerCamelCase (	self				:   Union[str, Any]	):





       '''simple docstring'''
       UpperCAmelCase_							:      List[str]						=   StableDiffusionPipeline.from_pretrained('''runwayml/stable-diffusion-v1-5'''       ,   safety_checker=__snake_case	)
       UpperCAmelCase_							:      List[Any]						=   LMSDiscreteScheduler.from_config(sd_pipe.scheduler.config	)
       UpperCAmelCase_							:      List[Any]						=   sd_pipe.to(__snake_case	)
       sd_pipe.set_progress_bar_config(disable=__snake_case	)

       UpperCAmelCase_							:      Any						=   (
           '''portrait of girl with smokey eyes makeup in abandoned hotel, grange clothes, redshift, wide high angle'''
           ''' coloured polaroid photograph with flash, kodak film, hyper real, stunning moody cinematography, with'''
           ''' anamorphic lenses, by maripol, fallen angels by wong kar - wai, style of suspiria and neon demon and'''
           ''' children from bahnhof zoo, detailed '''
       )
       UpperCAmelCase_							:      Optional[Any]						=   4_003_660_346
       UpperCAmelCase_							:      Optional[Any]						=   7

       # without safety guidance (sld_guidance_scale = 0)
       UpperCAmelCase_							:      List[Any]						=   torch.manual_seed(__snake_case	)
       UpperCAmelCase_							:      Optional[int]						=   sd_pipe(
           [prompt]       ,   generator=__snake_case       ,   guidance_scale=__snake_case       ,   num_inference_steps=50       ,   output_type='''np'''       ,   width=512       ,   height=512       ,   sld_guidance_scale=0       ,   )

       UpperCAmelCase_							:      str						=   output.images
       UpperCAmelCase_							:      int						=   image[0, -3:, -3:, -1]
       UpperCAmelCase_							:      str						=   [0.2_278, 0.2_231, 0.2_249, 0.2_333, 0.2_303, 0.1_885, 0.2_273, 0.2_144, 0.2_176]

       assert image.shape == (1, 512, 512, 3)

       assert np.abs(image_slice.flatten() - expected_slice	).max() < 1E-2

       # without safety guidance (strong configuration)
       UpperCAmelCase_							:      List[str]						=   torch.manual_seed(__snake_case	)
       UpperCAmelCase_							:      Optional[Any]						=   sd_pipe(
           [prompt]       ,   generator=__snake_case       ,   guidance_scale=__snake_case       ,   num_inference_steps=50       ,   output_type='''np'''       ,   width=512       ,   height=512       ,   sld_guidance_scale=2_000       ,   sld_warmup_steps=7       ,   sld_threshold=0.025       ,   sld_momentum_scale=0.5       ,   sld_mom_beta=0.7       ,   )

       UpperCAmelCase_							:      List[str]						=   output.images
       UpperCAmelCase_							:      List[Any]						=   image[0, -3:, -3:, -1]
       UpperCAmelCase_							:      Any						=   [0.2_383, 0.2_276, 0.236, 0.2_192, 0.2_186, 0.2_053, 0.1_971, 0.1_901, 0.1_719]

       assert image.shape == (1, 512, 512, 3)

       assert np.abs(image_slice.flatten() - expected_slice	).max() < 1E-2





    def 					_lowerCamelCase (	self				:   Dict	):





       '''simple docstring'''
       UpperCAmelCase_							:      Optional[int]						=   StableDiffusionPipeline.from_pretrained('''runwayml/stable-diffusion-v1-5'''       ,   safety_checker=__snake_case	)
       UpperCAmelCase_							:      str						=   LMSDiscreteScheduler.from_config(sd_pipe.scheduler.config	)
       UpperCAmelCase_							:      Optional[Any]						=   sd_pipe.to(__snake_case	)
       sd_pipe.set_progress_bar_config(disable=__snake_case	)

       UpperCAmelCase_							:      Optional[Any]						=   '''padme amidala taking a bath artwork, safe for work, no nudity'''
       UpperCAmelCase_							:      str						=   2_734_971_755
       UpperCAmelCase_							:      Tuple						=   7

       UpperCAmelCase_							:      Any						=   torch.manual_seed(__snake_case	)
       UpperCAmelCase_							:      List[Any]						=   sd_pipe(
           [prompt]       ,   generator=__snake_case       ,   guidance_scale=__snake_case       ,   num_inference_steps=50       ,   output_type='''np'''       ,   width=512       ,   height=512       ,   sld_guidance_scale=0       ,   )

       UpperCAmelCase_							:      int						=   output.images
       UpperCAmelCase_							:      Optional[Any]						=   image[0, -3:, -3:, -1]
       UpperCAmelCase_							:      Optional[Any]						=   [0.3_502, 0.3_622, 0.3_396, 0.3_642, 0.3_478, 0.3_318, 0.35, 0.3_348, 0.3_297]

       assert image.shape == (1, 512, 512, 3)

       assert np.abs(image_slice.flatten() - expected_slice	).max() < 1E-2

       UpperCAmelCase_							:      Optional[int]						=   torch.manual_seed(__snake_case	)
       UpperCAmelCase_							:      Union[str, Any]						=   sd_pipe(
           [prompt]       ,   generator=__snake_case       ,   guidance_scale=__snake_case       ,   num_inference_steps=50       ,   output_type='''np'''       ,   width=512       ,   height=512       ,   sld_guidance_scale=2_000       ,   sld_warmup_steps=7       ,   sld_threshold=0.025       ,   sld_momentum_scale=0.5       ,   sld_mom_beta=0.7       ,   )

       UpperCAmelCase_							:      Optional[int]						=   output.images
       UpperCAmelCase_							:      Dict						=   image[0, -3:, -3:, -1]
       UpperCAmelCase_							:      int						=   [0.5_531, 0.5_206, 0.4_895, 0.5_156, 0.5_182, 0.4_751, 0.4_802, 0.4_803, 0.4_443]

       assert image.shape == (1, 512, 512, 3)

       assert np.abs(image_slice.flatten() - expected_slice	).max() < 1E-2





    def 					_lowerCamelCase (	self				:   Tuple	):





       '''simple docstring'''
       UpperCAmelCase_							:      Any						=   StableDiffusionPipeline.from_pretrained('''runwayml/stable-diffusion-v1-5'''	)
       UpperCAmelCase_							:      int						=   sd_pipe.to(__snake_case	)
       sd_pipe.set_progress_bar_config(disable=__snake_case	)

       UpperCAmelCase_							:      Union[str, Any]						=   (
           '''the four horsewomen of the apocalypse, painting by tom of finland, gaston bussiere, craig mullins, j. c.'''
           ''' leyendecker'''
       )
       UpperCAmelCase_							:      Optional[Any]						=   1_044_355_234
       UpperCAmelCase_							:      int						=   12

       UpperCAmelCase_							:      List[Any]						=   torch.manual_seed(__snake_case	)
       UpperCAmelCase_							:      Optional[int]						=   sd_pipe(
           [prompt]       ,   generator=__snake_case       ,   guidance_scale=__snake_case       ,   num_inference_steps=50       ,   output_type='''np'''       ,   width=512       ,   height=512       ,   sld_guidance_scale=0       ,   )

       UpperCAmelCase_							:      Optional[int]						=   output.images
       UpperCAmelCase_							:      Optional[int]						=   image[0, -3:, -3:, -1]
       UpperCAmelCase_							:      Tuple						=   np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]	)

       assert image.shape == (1, 512, 512, 3)

       assert np.abs(image_slice.flatten() - expected_slice	).max() < 1E-7

       UpperCAmelCase_							:      Dict						=   torch.manual_seed(__snake_case	)
       UpperCAmelCase_							:      List[Any]						=   sd_pipe(
           [prompt]       ,   generator=__snake_case       ,   guidance_scale=__snake_case       ,   num_inference_steps=50       ,   output_type='''np'''       ,   width=512       ,   height=512       ,   sld_guidance_scale=2_000       ,   sld_warmup_steps=7       ,   sld_threshold=0.025       ,   sld_momentum_scale=0.5       ,   sld_mom_beta=0.7       ,   )

       UpperCAmelCase_							:      List[str]						=   output.images
       UpperCAmelCase_							:      List[str]						=   image[0, -3:, -3:, -1]
       UpperCAmelCase_							:      Dict						=   np.array([0.5_818, 0.6_285, 0.6_835, 0.6_019, 0.625, 0.6_754, 0.6_096, 0.6_334, 0.6_561]	)
       assert image.shape == (1, 512, 512, 3)

       assert np.abs(image_slice.flatten() - expected_slice	).max() < 1E-2



import os
import re
import warnings
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
       from .tokenization_ta import TaTokenizer
else:
       __UpperCamelCase				:  Optional[int]      					=							None


__UpperCamelCase				:  Union[str, Any]      					=							logging.get_logger(__name__)

__UpperCamelCase				:  List[Any]      					=							{'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

__UpperCamelCase				:  Optional[int]      					=							{
    'vocab_file': {
        't5-small': 'https://huggingface.co/t5-small/resolve/main/spiece.model',
        't5-base': 'https://huggingface.co/t5-base/resolve/main/spiece.model',
        't5-large': 'https://huggingface.co/t5-large/resolve/main/spiece.model',
        't5-3b': 'https://huggingface.co/t5-3b/resolve/main/spiece.model',
        't5-11b': 'https://huggingface.co/t5-11b/resolve/main/spiece.model',
    },
    'tokenizer_file': {
        't5-small': 'https://huggingface.co/t5-small/resolve/main/tokenizer.json',
        't5-base': 'https://huggingface.co/t5-base/resolve/main/tokenizer.json',
        't5-large': 'https://huggingface.co/t5-large/resolve/main/tokenizer.json',
        't5-3b': 'https://huggingface.co/t5-3b/resolve/main/tokenizer.json',
        't5-11b': 'https://huggingface.co/t5-11b/resolve/main/tokenizer.json',
    },
}


# TODO(PVP) - this should be removed in Transformers v5
__UpperCamelCase				:  Optional[Any]      					=							{
    't5-small': 512,
    't5-base': 512,
    't5-large': 512,
    't5-3b': 512,
    't5-11b': 512,
}





class  lowerCAmelCase__(    snake_case__						):






    '''simple docstring'''





    A_  :     Optional[int]			= VOCAB_FILES_NAMES
    A_  :     List[Any]			= PRETRAINED_VOCAB_FILES_MAP
    A_  :     int			= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    A_  :     str			= ['input_ids', 'attention_mask']
    A_  :     int			= TaTokenizer

    A_  :     List[int]			= []





    def __init__(	self				:   Union[str, Any]       ,   __snake_case				:   Tuple=None       ,   __snake_case				:   List[Any]=None       ,   __snake_case				:   int="</s>"       ,   __snake_case				:   List[Any]="<unk>"       ,   __snake_case				:   Dict="<pad>"       ,   __snake_case				:   Tuple=100       ,   __snake_case				:   int=None       ,   **__snake_case				:   Any       ,   ):





       '''simple docstring'''
       # Add extra_ids to the special token list
       if extra_ids > 0 and additional_special_tokens is None:
          UpperCAmelCase_							:      Optional[int]						=   [f'''<extra_id_{i}>''' for i in range(__snake_case	)]
       elif extra_ids > 0 and additional_special_tokens is not None:
          # Check that we have the right number of extra special tokens
          UpperCAmelCase_							:      Any						=   len(set(filter(lambda __snake_case	: bool('''extra_id_''' in str(__snake_case	)	)       ,   __snake_case	)	)	)
          if extra_tokens != extra_ids:
             raise ValueError(
                 f'''Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are'''
                 ''' provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids'''
                 ''' tokens'''	)

       super().__init__(
           __snake_case       ,   tokenizer_file=__snake_case       ,   eos_token=__snake_case       ,   unk_token=__snake_case       ,   pad_token=__snake_case       ,   extra_ids=__snake_case       ,   additional_special_tokens=__snake_case       ,   **__snake_case       ,   )

       UpperCAmelCase_							:      str						=   vocab_file
       UpperCAmelCase_							:      List[str]						=   False if not self.vocab_file else True
       UpperCAmelCase_							:      Union[str, Any]						=   extra_ids





    @staticmethod
    def 					_lowerCamelCase (	__snake_case				:   Dict       ,   __snake_case				:   List[str]       ,   __snake_case				:   Tuple	):





       '''simple docstring'''
       if pretrained_model_name_or_path in TaTokenizerFast.max_model_input_sizes:
          UpperCAmelCase_							:      str						=   TaTokenizerFast.max_model_input_sizes[pretrained_model_name_or_path]
          if init_max_model_length is not None and init_max_model_length != max_model_length:
             return init_max_model_length
          elif init_max_model_length is None:
             warnings.warn(
                 '''This tokenizer was incorrectly instantiated with a model max length of'''
                 f''' {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this'''
                 ''' behavior is kept to avoid breaking backwards compatibility when padding/encoding with'''
                 ''' `truncation is True`.\n- Be aware that you SHOULD NOT rely on'''
                 f''' {pretrained_model_name_or_path} automatically truncating your input to'''
                 f''' {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences'''
                 f''' longer than {deprecated_max_model_length} you can either instantiate this tokenizer with'''
                 ''' `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please'''
                 ''' instantiate this tokenizer with `model_max_length` set to your preferred value.'''       ,   __snake_case       ,   )

       return max_model_length





    def 					_lowerCamelCase (	self				:   Dict       ,   __snake_case				:   str       ,   __snake_case				:   Optional[str] = None	):





       '''simple docstring'''
       if not self.can_save_slow_tokenizer:
          raise ValueError(
              '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
              '''tokenizer.'''	)

       if not os.path.isdir(__snake_case	):
          logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''	)
          return
       UpperCAmelCase_							:      str						=   os.path.join(
           __snake_case       ,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']	)

       if os.path.abspath(self.vocab_file	) != os.path.abspath(__snake_case	):
          copyfile(self.vocab_file       ,   __snake_case	)
          logger.info(f'''Copy vocab file to {out_vocab_file}'''	)

       return (out_vocab_file,)





    def 					_lowerCamelCase (	self				:   List[str]       ,   __snake_case				:   List[int]       ,   __snake_case				:   Optional[List[int]] = None	):





       '''simple docstring'''
       UpperCAmelCase_							:      Optional[Any]						=   token_ids_a + [self.eos_token_id]
       if token_ids_a is None:
          return self.prefix_tokens + token_ids_a
       else:
          UpperCAmelCase_							:      int						=   token_ids_a + [self.eos_token_id]
          return self.prefix_tokens + token_ids_a + token_ids_a





    def 					_lowerCamelCase (	self				:   Dict       ,   __snake_case				:   List[int]       ,   __snake_case				:   Optional[List[int]] = None	):





       '''simple docstring'''
       UpperCAmelCase_							:      int						=   [self.eos_token_id]

       if token_ids_a is None:
          return len(token_ids_a + eos	) * [0]
       return len(token_ids_a + eos + token_ids_a + eos	) * [0]





    def 					_lowerCamelCase (	self				:   Tuple	):





       '''simple docstring'''
       return list(
           set(filter(lambda __snake_case	: bool(re.search(R'''<extra_id_\d+>'''       ,   __snake_case	)	) is not None       ,   self.additional_special_tokens	)	)	)





    def 					_lowerCamelCase (	self				:   Tuple	):





       '''simple docstring'''
       return [self.convert_tokens_to_ids(__snake_case	) for token in self.get_sentinel_tokens()]