



import tempfile
import unittest

import numpy as np

from diffusers import (
    DDIMScheduler,
    DPMSolverMultistepScheduler,
    EulerAncestralDiscreteScheduler,
    EulerDiscreteScheduler,
    LMSDiscreteScheduler,
    OnnxStableDiffusionPipeline,
    PNDMScheduler,
)
from diffusers.utils.testing_utils import is_onnx_available, nightly, require_onnxruntime, require_torch_gpu

from ..test_pipelines_onnx_common import OnnxPipelineTesterMixin


if is_onnx_available():
							import onnxruntime as ort



class       UpperCAmelCase__(							lowerCamelCase     ,							unittest.TestCase	):

			'''simple docstring'''




			A    :  List[str]       				=    "hf-internal-testing/tiny-random-OnnxStableDiffusionPipeline"


			def 			UpperCAmelCase		(						self  :			Tuple			,				lowerCAmelCase  :			Tuple=0)	->				Any:




									"""simple docstring"""

									lowercase__     =  np.random.RandomState(lowerCAmelCase)
									lowercase__     =  {
									    'prompt': 'A painting of a squirrel eating a burger',
									    'generator': generator,
									    'num_inference_steps': 2,
									    'guidance_scale': 7.5,
									    'output_type': 'numpy',
									}
									return inputs


			def 			UpperCAmelCase		(						self  :			Tuple)	->				str:




									"""simple docstring"""

									lowercase__     =  OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint			,				provider='CPUExecutionProvider')
									pipe.set_progress_bar_config(disable=lowerCAmelCase)

									lowercase__     =  self.get_dummy_inputs()
									lowercase__     =  pipe(**lowerCAmelCase).images
									lowercase__     =  image[0, -3:, -3:, -1]

									assert image.shape == (1, 1_28, 1_28, 3)
									lowercase__     =  np.array([0.6_50_72, 0.5_84_92, 0.4_82_19, 0.5_55_21, 0.5_31_80, 0.5_59_39, 0.5_06_97, 0.3_98_00, 0.4_64_55])

									assert np.abs(image_slice.flatten() - expected_slice).max() < 1E-2


			def 			UpperCAmelCase		(						self  :			Dict)	->				int:




									"""simple docstring"""

									lowercase__     =  OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint			,				provider='CPUExecutionProvider')
									lowercase__     =  PNDMScheduler.from_config(pipe.scheduler.config			,				skip_prk_steps=lowerCAmelCase)
									pipe.set_progress_bar_config(disable=lowerCAmelCase)

									lowercase__     =  self.get_dummy_inputs()
									lowercase__     =  pipe(**lowerCAmelCase).images
									lowercase__     =  image[0, -3:, -3:, -1]

									assert image.shape == (1, 1_28, 1_28, 3)
									lowercase__     =  np.array([0.6_58_63, 0.5_94_25, 0.4_93_26, 0.5_63_13, 0.5_38_75, 0.5_66_27, 0.5_10_65, 0.3_97_77, 0.4_63_30])

									assert np.abs(image_slice.flatten() - expected_slice).max() < 1E-2


			def 			UpperCAmelCase		(						self  :			int)	->				int:




									"""simple docstring"""

									lowercase__     =  OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint			,				provider='CPUExecutionProvider')
									lowercase__     =  LMSDiscreteScheduler.from_config(pipe.scheduler.config)
									pipe.set_progress_bar_config(disable=lowerCAmelCase)

									lowercase__     =  self.get_dummy_inputs()
									lowercase__     =  pipe(**lowerCAmelCase).images
									lowercase__     =  image[0, -3:, -3:, -1]

									assert image.shape == (1, 1_28, 1_28, 3)
									lowercase__     =  np.array([0.5_37_55, 0.6_07_86, 0.4_74_02, 0.4_94_88, 0.5_18_69, 0.4_98_19, 0.4_79_85, 0.3_89_57, 0.4_42_79])

									assert np.abs(image_slice.flatten() - expected_slice).max() < 1E-2


			def 			UpperCAmelCase		(						self  :			List[Any])	->				List[Any]:




									"""simple docstring"""

									lowercase__     =  OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint			,				provider='CPUExecutionProvider')
									lowercase__     =  EulerDiscreteScheduler.from_config(pipe.scheduler.config)
									pipe.set_progress_bar_config(disable=lowerCAmelCase)

									lowercase__     =  self.get_dummy_inputs()
									lowercase__     =  pipe(**lowerCAmelCase).images
									lowercase__     =  image[0, -3:, -3:, -1]

									assert image.shape == (1, 1_28, 1_28, 3)
									lowercase__     =  np.array([0.5_37_55, 0.6_07_86, 0.4_74_02, 0.4_94_88, 0.5_18_69, 0.4_98_19, 0.4_79_85, 0.3_89_57, 0.4_42_79])

									assert np.abs(image_slice.flatten() - expected_slice).max() < 1E-2


			def 			UpperCAmelCase		(						self  :			Union[str, Any])	->				str:




									"""simple docstring"""

									lowercase__     =  OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint			,				provider='CPUExecutionProvider')
									lowercase__     =  EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config)
									pipe.set_progress_bar_config(disable=lowerCAmelCase)

									lowercase__     =  self.get_dummy_inputs()
									lowercase__     =  pipe(**lowerCAmelCase).images
									lowercase__     =  image[0, -3:, -3:, -1]

									assert image.shape == (1, 1_28, 1_28, 3)
									lowercase__     =  np.array([0.5_38_17, 0.6_08_12, 0.4_73_84, 0.4_95_30, 0.5_18_94, 0.4_98_14, 0.4_79_84, 0.3_89_58, 0.4_42_71])

									assert np.abs(image_slice.flatten() - expected_slice).max() < 1E-2


			def 			UpperCAmelCase		(						self  :			List[str])	->				int:




									"""simple docstring"""

									lowercase__     =  OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint			,				provider='CPUExecutionProvider')
									lowercase__     =  DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)
									pipe.set_progress_bar_config(disable=lowerCAmelCase)

									lowercase__     =  self.get_dummy_inputs()
									lowercase__     =  pipe(**lowerCAmelCase).images
									lowercase__     =  image[0, -3:, -3:, -1]

									assert image.shape == (1, 1_28, 1_28, 3)
									lowercase__     =  np.array([0.5_38_95, 0.6_08_08, 0.4_79_33, 0.4_96_08, 0.5_18_86, 0.4_99_50, 0.4_80_53, 0.3_89_57, 0.4_42_00])

									assert np.abs(image_slice.flatten() - expected_slice).max() < 1E-2


			def 			UpperCAmelCase		(						self  :			List[Any])	->				List[Any]:




									"""simple docstring"""

									lowercase__     =  OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint			,				provider='CPUExecutionProvider')
									pipe.set_progress_bar_config(disable=lowerCAmelCase)

									lowercase__     =  self.get_dummy_inputs()
									lowercase__     =  3 * [inputs['prompt']]

									# forward
									lowercase__     =  pipe(**lowerCAmelCase)
									lowercase__     =  output.images[0, -3:, -3:, -1]

									lowercase__     =  self.get_dummy_inputs()
									lowercase__     =  3 * [inputs.pop('prompt')]

									lowercase__     =  pipe.tokenizer(
									    lowerCAmelCase			,				padding='max_length'			,				max_length=pipe.tokenizer.model_max_length			,				truncation=lowerCAmelCase			,				return_tensors='np'			,				)
									lowercase__     =  text_inputs['input_ids']

									lowercase__     =  pipe.text_encoder(input_ids=text_inputs.astype(np.intaa))[0]

									lowercase__     =  prompt_embeds

									# forward
									lowercase__     =  pipe(**lowerCAmelCase)
									lowercase__     =  output.images[0, -3:, -3:, -1]

									assert np.abs(image_slice_a.flatten() - image_slice_a.flatten()).max() < 1E-4



			def 			UpperCAmelCase		(						self  :			Optional[int])	->				Dict:




									"""simple docstring"""

									lowercase__     =  OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint			,				provider='CPUExecutionProvider')
									pipe.set_progress_bar_config(disable=lowerCAmelCase)

									lowercase__     =  self.get_dummy_inputs()
									lowercase__     =  3 * ['this is a negative prompt']
									lowercase__     =  negative_prompt
									lowercase__     =  3 * [inputs['prompt']]

									# forward
									lowercase__     =  pipe(**lowerCAmelCase)
									lowercase__     =  output.images[0, -3:, -3:, -1]

									lowercase__     =  self.get_dummy_inputs()
									lowercase__     =  3 * [inputs.pop('prompt')]

									lowercase__     =  []
									for p in [prompt, negative_prompt]:
															lowercase__     =  pipe.tokenizer(
															    lowerCAmelCase			,				padding='max_length'			,				max_length=pipe.tokenizer.model_max_length			,				truncation=lowerCAmelCase			,				return_tensors='np'			,				)
															lowercase__     =  text_inputs['input_ids']

															embeds.append(pipe.text_encoder(input_ids=text_inputs.astype(np.intaa))[0])

									lowercase__,	lowercase__     =  embeds

									# forward
									lowercase__     =  pipe(**lowerCAmelCase)
									lowercase__     =  output.images[0, -3:, -3:, -1]

									assert np.abs(image_slice_a.flatten() - image_slice_a.flatten()).max() < 1E-4



@nightly
@require_onnxruntime
@require_torch_gpu
class       UpperCAmelCase__(							unittest.TestCase	):

			'''simple docstring'''




			@property
			def 			UpperCAmelCase		(						self  :			int)	->				Dict:




									"""simple docstring"""

									return (
									    "CUDAExecutionProvider",
									    {
									        "gpu_mem_limit": "15000000000",  # 15GB
									        "arena_extend_strategy": "kSameAsRequested",
									    },
									)


			@property
			def 			UpperCAmelCase		(						self  :			Union[str, Any])	->				List[str]:




									"""simple docstring"""

									lowercase__     =  ort.SessionOptions()
									lowercase__     =  False
									return options


			def 			UpperCAmelCase		(						self  :			Tuple)	->				Optional[Any]:




									"""simple docstring"""

									lowercase__     =  OnnxStableDiffusionPipeline.from_pretrained(
									    'CompVis/stable-diffusion-v1-4'			,				revision='onnx'			,				safety_checker=lowerCAmelCase			,				feature_extractor=lowerCAmelCase			,				provider=self.gpu_provider			,				sess_options=self.gpu_options			,				)
									sd_pipe.set_progress_bar_config(disable=lowerCAmelCase)

									lowercase__     =  'A painting of a squirrel eating a burger'
									np.random.seed(0)
									lowercase__     =  sd_pipe([prompt]			,				guidance_scale=6.0			,				num_inference_steps=10			,				output_type='np')
									lowercase__     =  output.images

									lowercase__     =  image[0, -3:, -3:, -1]

									assert image.shape == (1, 5_12, 5_12, 3)
									lowercase__     =  np.array([0.04_52, 0.03_90, 0.00_87, 0.03_50, 0.06_17, 0.03_64, 0.05_44, 0.05_23, 0.07_20])

									assert np.abs(image_slice.flatten() - expected_slice).max() < 1E-3


			def 			UpperCAmelCase		(						self  :			Optional[int])	->				Dict:




									"""simple docstring"""

									lowercase__     =  DDIMScheduler.from_pretrained(
									    'runwayml/stable-diffusion-v1-5'			,				subfolder='scheduler'			,				revision='onnx')
									lowercase__     =  OnnxStableDiffusionPipeline.from_pretrained(
									    'runwayml/stable-diffusion-v1-5'			,				revision='onnx'			,				scheduler=lowerCAmelCase			,				safety_checker=lowerCAmelCase			,				feature_extractor=lowerCAmelCase			,				provider=self.gpu_provider			,				sess_options=self.gpu_options			,				)
									sd_pipe.set_progress_bar_config(disable=lowerCAmelCase)

									lowercase__     =  'open neural network exchange'
									lowercase__     =  np.random.RandomState(0)
									lowercase__     =  sd_pipe([prompt]			,				guidance_scale=7.5			,				num_inference_steps=10			,				generator=lowerCAmelCase			,				output_type='np')
									lowercase__     =  output.images
									lowercase__     =  image[0, -3:, -3:, -1]

									assert image.shape == (1, 5_12, 5_12, 3)
									lowercase__     =  np.array([0.28_67, 0.19_74, 0.14_81, 0.72_94, 0.72_51, 0.66_67, 0.41_94, 0.56_42, 0.64_86])

									assert np.abs(image_slice.flatten() - expected_slice).max() < 1E-3


			def 			UpperCAmelCase		(						self  :			Any)	->				Optional[int]:




									"""simple docstring"""

									lowercase__     =  LMSDiscreteScheduler.from_pretrained(
									    'runwayml/stable-diffusion-v1-5'			,				subfolder='scheduler'			,				revision='onnx')
									lowercase__     =  OnnxStableDiffusionPipeline.from_pretrained(
									    'runwayml/stable-diffusion-v1-5'			,				revision='onnx'			,				scheduler=lowerCAmelCase			,				safety_checker=lowerCAmelCase			,				feature_extractor=lowerCAmelCase			,				provider=self.gpu_provider			,				sess_options=self.gpu_options			,				)
									sd_pipe.set_progress_bar_config(disable=lowerCAmelCase)

									lowercase__     =  'open neural network exchange'
									lowercase__     =  np.random.RandomState(0)
									lowercase__     =  sd_pipe([prompt]			,				guidance_scale=7.5			,				num_inference_steps=10			,				generator=lowerCAmelCase			,				output_type='np')
									lowercase__     =  output.images
									lowercase__     =  image[0, -3:, -3:, -1]

									assert image.shape == (1, 5_12, 5_12, 3)
									lowercase__     =  np.array([0.23_06, 0.19_59, 0.15_93, 0.65_49, 0.63_94, 0.54_08, 0.50_65, 0.60_10, 0.61_61])

									assert np.abs(image_slice.flatten() - expected_slice).max() < 1E-3


			def 			UpperCAmelCase		(						self  :			Tuple)	->				Union[str, Any]:




									"""simple docstring"""

									lowercase__     =  0

									def test_callback_fn(lowerCAmelCase  :			int			,				lowerCAmelCase  :			int			,				lowerCAmelCase  :			np.ndarray) -> None:
															lowercase__     =  True
															nonlocal number_of_steps
															number_of_steps += 1
															if step == 0:
																					assert latents.shape == (1, 4, 64, 64)
																					lowercase__     =  latents[0, -3:, -3:, -1]
																					lowercase__     =  np.array(
																					    [-0.67_72, -0.38_35, -1.24_56, 0.19_05, -1.09_74, 0.69_67, -1.93_53, 0.01_78, 1.01_67])

																					assert np.abs(latents_slice.flatten() - expected_slice).max() < 1E-3
															elif step == 5:
																					assert latents.shape == (1, 4, 64, 64)
																					lowercase__     =  latents[0, -3:, -3:, -1]
																					lowercase__     =  np.array(
																					    [-0.33_51, 0.22_41, -0.18_37, -0.23_25, -0.65_77, 0.33_93, -0.02_41, 0.58_99, 1.38_75])

																					assert np.abs(latents_slice.flatten() - expected_slice).max() < 1E-3

									lowercase__     =  False

									lowercase__     =  OnnxStableDiffusionPipeline.from_pretrained(
									    'runwayml/stable-diffusion-v1-5'			,				revision='onnx'			,				safety_checker=lowerCAmelCase			,				feature_extractor=lowerCAmelCase			,				provider=self.gpu_provider			,				sess_options=self.gpu_options			,				)
									pipe.set_progress_bar_config(disable=lowerCAmelCase)

									lowercase__     =  'Andromeda galaxy in a bottle'

									lowercase__     =  np.random.RandomState(0)
									pipe(
									    prompt=lowerCAmelCase			,				num_inference_steps=5			,				guidance_scale=7.5			,				generator=lowerCAmelCase			,				callback=lowerCAmelCase			,				callback_steps=1			,				)
									assert test_callback_fn.has_been_called
									assert number_of_steps == 6



			def 			UpperCAmelCase		(						self  :			str)	->				Any:




									"""simple docstring"""

									lowercase__     =  OnnxStableDiffusionPipeline.from_pretrained(
									    'runwayml/stable-diffusion-v1-5'			,				revision='onnx'			,				safety_checker=lowerCAmelCase			,				feature_extractor=lowerCAmelCase			,				provider=self.gpu_provider			,				sess_options=self.gpu_options			,				)
									assert isinstance(lowerCAmelCase			,				lowerCAmelCase)
									assert pipe.safety_checker is None

									lowercase__     =  pipe('example prompt'			,				num_inference_steps=2).images[0]
									assert image is not None

									# check that there's no error when saving a pipeline with one of the models being None
									with tempfile.TemporaryDirectory() as tmpdirname:
															pipe.save_pretrained(lowerCAmelCase)
															lowercase__     =  OnnxStableDiffusionPipeline.from_pretrained(lowerCAmelCase)

									# sanity check that the pipeline still works
									assert pipe.safety_checker is None
									lowercase__     =  pipe('example prompt'			,				num_inference_steps=2).images[0]
									assert image is not None





import json
import os
from typing import Dict, List, Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


a__      :				Optional[int]   	=			logging.get_logger(__name__)


a__      :				Dict   	=			{
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
    "tokenizer_config_file": "tokenizer_config.json",
}

a__      :				str   	=			{
    "vocab_file": {
        "facebook/blenderbot_small-90M": "https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/vocab.json"
    },
    "merges_file": {
        "facebook/blenderbot_small-90M": "https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/merges.txt"
    },
    "tokenizer_config_file": {
        "facebook/blenderbot_small-90M": (
            "https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/tokenizer_config.json"
        )
    },
}

a__      :				Any   	=			{"facebook/blenderbot_small-90M": 5_12}

def   _lowerCAmelCase   (						A__	):
      lowercase__     =  set()
      lowercase__     =  word[0]
      for char in word[1:]:
            pairs.add((prev_char, char)	)
            lowercase__     =  char

      lowercase__     =  set(A__	)
      return pairs



class       UpperCAmelCase__(							lowerCamelCase	):

   '''simple docstring'''




   A    :  List[str]       				=    VOCAB_FILES_NAMES
   A    :  str       				=    PRETRAINED_VOCAB_FILES_MAP
   A    :  int       				=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   A    :  Tuple       				=    ["input_ids", "attention_mask"]


   def __init__(						self  :			Optional[int]			,				lowerCAmelCase  :			Dict			,				lowerCAmelCase  :			Dict			,				lowerCAmelCase  :			int="__start__"			,				lowerCAmelCase  :			Dict="__end__"			,				lowerCAmelCase  :			Any="__unk__"			,				lowerCAmelCase  :			str="__null__"			,				**lowerCAmelCase  :			Optional[Any]			,				)	->				List[str]:




         """simple docstring"""

         super().__init__(unk_token=lowerCAmelCase			,				bos_token=lowerCAmelCase			,				eos_token=lowerCAmelCase			,				pad_token=lowerCAmelCase			,				**lowerCAmelCase)

         with open(lowerCAmelCase			,				encoding='utf-8') as vocab_handle:
               lowercase__     =  json.load(lowerCAmelCase)
         lowercase__     =  {v: k for k, v in self.encoder.items()}
         with open(lowerCAmelCase			,				encoding='utf-8') as merges_handle:
               lowercase__     =  merges_handle.read().split('\n')[1:-1]
         lowercase__     =  [tuple(merge.split()) for merge in merges]
         lowercase__     =  dict(zip(lowerCAmelCase			,				range(len(lowerCAmelCase))))
         lowercase__     =  {}


   @property
   def 			UpperCAmelCase		(						self  :			int)	->				int:




         """simple docstring"""

         return len(self.encoder)


   def 			UpperCAmelCase		(						self  :			str)	->				Dict:




         """simple docstring"""

         return dict(self.encoder			,				**self.added_tokens_encoder)


   def 			UpperCAmelCase		(						self  :			str			,				lowerCAmelCase  :			str)	->				str:




         """simple docstring"""

         if token in self.cache:
               return self.cache[token]
         lowercase__     =  re.sub('([.,!?()])'			,				R' \1'			,				lowerCAmelCase)
         lowercase__     =  re.sub('(\')'			,				R' \1 '			,				lowerCAmelCase)
         lowercase__     =  re.sub(R'\s{2,}'			,				' '			,				lowerCAmelCase)
         if "\n" in token:
               lowercase__     =  token.replace('\n'			,				' __newln__')

         lowercase__     =  token.split(' ')
         lowercase__     =  []
         for token in tokens:
               if not len(lowerCAmelCase):
                     continue

               lowercase__     =  token.lower()
               lowercase__     =  tuple(lowerCAmelCase)
               lowercase__     =  tuple(list(word[:-1]) + [word[-1] + '</w>'])
               lowercase__     =  get_pairs(lowerCAmelCase)

               if not pairs:
                     words.append(lowerCAmelCase)
                     continue

               while True:
                     lowercase__     =  min(lowerCAmelCase			,				key=lambda lowerCAmelCase: self.bpe_ranks.get(lowerCAmelCase			,				float('inf')))
                     if bigram not in self.bpe_ranks:
                           break
                     lowercase__,	lowercase__     =  bigram
                     lowercase__     =  []
                     lowercase__     =  0

                     while i < len(lowerCAmelCase):
                           try:
                                 lowercase__     =  word.index(lowerCAmelCase			,				lowerCAmelCase)
                                 new_word.extend(word[i:j])
                                 lowercase__     =  j
                           except ValueError:
                                 new_word.extend(word[i:])
                                 break

                           if word[i] == first and i < len(lowerCAmelCase) - 1 and word[i + 1] == second:
                                 new_word.append(first + second)
                                 i += 2
                           else:
                                 new_word.append(word[i])
                                 i += 1
                     lowercase__     =  tuple(lowerCAmelCase)
                     lowercase__     =  new_word
                     if len(lowerCAmelCase) == 1:
                           break
                     else:
                           lowercase__     =  get_pairs(lowerCAmelCase)
               lowercase__     =  '@@ '.join(lowerCAmelCase)
               lowercase__     =  word[:-4]

               lowercase__     =  word
               words.append(lowerCAmelCase)
         return " ".join(lowerCAmelCase)


   def 			UpperCAmelCase		(						self  :			List[str]			,				lowerCAmelCase  :			str)	->				List[str]:




         """simple docstring"""

         lowercase__     =  []

         lowercase__     =  re.findall(R'\S+\n?'			,				lowerCAmelCase)

         for token in words:
               split_tokens.extend(list(self.bpe(lowerCAmelCase).split(' ')))
         return split_tokens


   def 			UpperCAmelCase		(						self  :			int			,				lowerCAmelCase  :			str)	->				int:




         """simple docstring"""

         lowercase__     =  token.lower()
         return self.encoder.get(lowerCAmelCase			,				self.encoder.get(self.unk_token))


   def 			UpperCAmelCase		(						self  :			Optional[int]			,				lowerCAmelCase  :			int)	->				str:




         """simple docstring"""

         return self.decoder.get(lowerCAmelCase			,				self.unk_token)


   def 			UpperCAmelCase		(						self  :			Optional[Any]			,				lowerCAmelCase  :			List[str])	->				str:




         """simple docstring"""

         lowercase__     =  ' '.join(lowerCAmelCase).replace('@@ '			,				'').strip()
         return out_string



   def 			UpperCAmelCase		(						self  :			str			,				lowerCAmelCase  :			str			,				lowerCAmelCase  :			Optional[str] = None)	->				Tuple[str]:




         """simple docstring"""

         if not os.path.isdir(lowerCAmelCase):
               logger.error(f'''Vocabulary path ({save_directory}) should be a directory''')
               return
         lowercase__     =  os.path.join(
             lowerCAmelCase			,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])
         lowercase__     =  os.path.join(
             lowerCAmelCase			,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file'])

         with open(lowerCAmelCase			,				'w'			,				encoding='utf-8') as f:
               f.write(json.dumps(self.encoder			,				indent=2			,				sort_keys=lowerCAmelCase			,				ensure_ascii=lowerCAmelCase) + '\n')

         lowercase__     =  0
         with open(lowerCAmelCase			,				'w'			,				encoding='utf-8') as writer:
               writer.write('#version: 0.2\n')
               for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			,				key=lambda lowerCAmelCase: kv[1]):
                     if index != token_index:
                           logger.warning(
                               f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                               ' Please check that the tokenizer is not corrupted!')
                           lowercase__     =  token_index
                     writer.write(' '.join(lowerCAmelCase) + '\n')
                     index += 1

         return vocab_file, merge_file
