






import random
import unittest

import numpy as np

from diffusers import (
    DPMSolverMultistepScheduler,
    EulerAncestralDiscreteScheduler,
    EulerDiscreteScheduler,
    LMSDiscreteScheduler,
    OnnxStableDiffusionImgaImgPipeline,
    PNDMScheduler,
)
from diffusers.utils import floats_tensor
from diffusers.utils.testing_utils import (
    is_onnx_available,
    load_image,
    nightly,
    require_onnxruntime,
    require_torch_gpu,
)

from ..test_pipelines_onnx_common import OnnxPipelineTesterMixin


if is_onnx_available():
	import onnxruntime as ort

class 		lowerCAmelCase     ( __a      ,       unittest.TestCase			):

					'''simple docstring'''

					_A						:			Union[str, Any]									=      '''hf-internal-testing/tiny-random-OnnxStableDiffusionPipeline'''



					def       lowerCAmelCase				(  self				:	int		,     __a				:	List[Any]=0							)			->  str:


												"""simple docstring"""

												__lowercase      :							str				=     floats_tensor((1, 3, 128, 128)		,     rng=random.Random(__a							)							)
												__lowercase      :							int				=     np.random.RandomState(__a							)
												__lowercase      :							str				=     {
												    """prompt""": """A painting of a squirrel eating a burger""",
												    """image""": image,
												    """generator""": generator,
												    """num_inference_steps""": 3,
												    """strength""": 0.75,
												    """guidance_scale""": 7.5,
												    """output_type""": """numpy""",
												}
												return inputs



					def       lowerCAmelCase				(  self				:	List[str]							)			->  Tuple:


												"""simple docstring"""

												__lowercase      :							Dict				=     OnnxStableDiffusionImgaImgPipeline.from_pretrained(self.hub_checkpoint		,     provider="""CPUExecutionProvider"""							)
												pipe.set_progress_bar_config(disable=__a							)

												__lowercase      :							List[str]				=     self.get_dummy_inputs()
												__lowercase      :							Optional[Any]				=     pipe(**__a							).images
												__lowercase      :							List[str]				=     image[0, -3:, -3:, -1].flatten()

												assert image.shape == (1, 128, 128, 3)
												__lowercase      :							Optional[Any]				=     np.array([0.69643, 0.58484, 0.50314, 0.58760, 0.55368, 0.59643, 0.51529, 0.41217, 0.49087]							)
												assert np.abs(image_slice - expected_slice							).max() < 1E-1



					def       lowerCAmelCase				(  self				:	str							)			->  Any:


												"""simple docstring"""

												__lowercase      :							int				=     OnnxStableDiffusionImgaImgPipeline.from_pretrained(self.hub_checkpoint		,     provider="""CPUExecutionProvider"""							)
												__lowercase      :							Optional[Any]				=     PNDMScheduler.from_config(pipe.scheduler.config		,     skip_prk_steps=__a							)
												pipe.set_progress_bar_config(disable=__a							)

												__lowercase      :							Optional[int]				=     self.get_dummy_inputs()
												__lowercase      :							Union[str, Any]				=     pipe(**__a							).images
												__lowercase      :							Optional[int]				=     image[0, -3:, -3:, -1]

												assert image.shape == (1, 128, 128, 3)
												__lowercase      :							str				=     np.array([0.61737, 0.54642, 0.53183, 0.54465, 0.52742, 0.60525, 0.49969, 0.40655, 0.48154]							)

												assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-1



					def       lowerCAmelCase				(  self				:	List[Any]							)			->  Union[str, Any]:


												"""simple docstring"""

												__lowercase      :							Dict				=     OnnxStableDiffusionImgaImgPipeline.from_pretrained(self.hub_checkpoint		,     provider="""CPUExecutionProvider"""							)
												__lowercase      :							List[Any]				=     LMSDiscreteScheduler.from_config(pipe.scheduler.config							)
												pipe.set_progress_bar_config(disable=__a							)

												# warmup pass to apply optimizations
												__lowercase      :							Optional[int]				=     pipe(**self.get_dummy_inputs()							)

												__lowercase      :							Dict				=     self.get_dummy_inputs()
												__lowercase      :							Any				=     pipe(**__a							).images
												__lowercase      :							Tuple				=     image[0, -3:, -3:, -1]

												assert image.shape == (1, 128, 128, 3)
												__lowercase      :							Union[str, Any]				=     np.array([0.52761, 0.59977, 0.49033, 0.49619, 0.54282, 0.50311, 0.47600, 0.40918, 0.45203]							)

												assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-1



					def       lowerCAmelCase				(  self				:	List[str]							)			->  Optional[int]:


												"""simple docstring"""

												__lowercase      :							str				=     OnnxStableDiffusionImgaImgPipeline.from_pretrained(self.hub_checkpoint		,     provider="""CPUExecutionProvider"""							)
												__lowercase      :							Any				=     EulerDiscreteScheduler.from_config(pipe.scheduler.config							)
												pipe.set_progress_bar_config(disable=__a							)

												__lowercase      :							Dict				=     self.get_dummy_inputs()
												__lowercase      :							str				=     pipe(**__a							).images
												__lowercase      :							Optional[Any]				=     image[0, -3:, -3:, -1]

												assert image.shape == (1, 128, 128, 3)
												__lowercase      :							List[Any]				=     np.array([0.52911, 0.60004, 0.49229, 0.49805, 0.54502, 0.50680, 0.47777, 0.41028, 0.45304]							)

												assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-1



					def       lowerCAmelCase				(  self				:	int							)			->  Dict:


												"""simple docstring"""

												__lowercase      :							Tuple				=     OnnxStableDiffusionImgaImgPipeline.from_pretrained(self.hub_checkpoint		,     provider="""CPUExecutionProvider"""							)
												__lowercase      :							int				=     EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config							)
												pipe.set_progress_bar_config(disable=__a							)

												__lowercase      :							Optional[int]				=     self.get_dummy_inputs()
												__lowercase      :							Optional[int]				=     pipe(**__a							).images
												__lowercase      :							List[str]				=     image[0, -3:, -3:, -1]

												assert image.shape == (1, 128, 128, 3)
												__lowercase      :							Optional[Any]				=     np.array([0.52911, 0.60004, 0.49229, 0.49805, 0.54502, 0.50680, 0.47777, 0.41028, 0.45304]							)

												assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-1



					def       lowerCAmelCase				(  self				:	List[str]							)			->  Optional[Any]:


												"""simple docstring"""

												__lowercase      :							int				=     OnnxStableDiffusionImgaImgPipeline.from_pretrained(self.hub_checkpoint		,     provider="""CPUExecutionProvider"""							)
												__lowercase      :							List[Any]				=     DPMSolverMultistepScheduler.from_config(pipe.scheduler.config							)
												pipe.set_progress_bar_config(disable=__a							)

												__lowercase      :							str				=     self.get_dummy_inputs()
												__lowercase      :							Optional[Any]				=     pipe(**__a							).images
												__lowercase      :							List[Any]				=     image[0, -3:, -3:, -1]

												assert image.shape == (1, 128, 128, 3)
												__lowercase      :							List[Any]				=     np.array([0.65331, 0.58277, 0.48204, 0.56059, 0.53665, 0.56235, 0.50969, 0.40009, 0.46552]							)

												assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-1







@nightly
@require_onnxruntime
@require_torch_gpu
class 		lowerCAmelCase     ( unittest.TestCase			):

					'''simple docstring'''

					@property
					def       lowerCAmelCase				(  self				:	Tuple							)			->  Any:


												"""simple docstring"""

												return (
												    "CUDAExecutionProvider",
												    {
												        "gpu_mem_limit": "15000000000",  # 15GB
												        "arena_extend_strategy": "kSameAsRequested",
												    },
												)



					@property
					def       lowerCAmelCase				(  self				:	Optional[int]							)			->  int:


												"""simple docstring"""

												__lowercase      :							List[Any]				=     ort.SessionOptions()
												__lowercase      :							Tuple				=     False
												return options



					def       lowerCAmelCase				(  self				:	List[str]							)			->  Any:


												"""simple docstring"""

												__lowercase      :							Optional[int]				=     load_image(
												    """https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"""
												    """/img2img/sketch-mountains-input.jpg"""							)
												__lowercase      :							Tuple				=     init_image.resize((768, 512)							)
												# using the PNDM scheduler by default
												__lowercase      :							List[Any]				=     OnnxStableDiffusionImgaImgPipeline.from_pretrained(
												    """CompVis/stable-diffusion-v1-4"""		,     revision="""onnx"""		,     safety_checker=__a		,     feature_extractor=__a		,     provider=self.gpu_provider		,     sess_options=self.gpu_options		,     )
												pipe.set_progress_bar_config(disable=__a							)

												__lowercase      :							str				=     """A fantasy landscape, trending on artstation"""

												__lowercase      :							Tuple				=     np.random.RandomState(0							)
												__lowercase      :							Any				=     pipe(
												    prompt=__a		,     image=__a		,     strength=0.75		,     guidance_scale=7.5		,     num_inference_steps=10		,     generator=__a		,     output_type="""np"""		,     )
												__lowercase      :							int				=     output.images
												__lowercase      :							Tuple				=     images[0, 255:258, 383:386, -1]

												assert images.shape == (1, 512, 768, 3)
												__lowercase      :							int				=     np.array([0.4909, 0.5059, 0.5372, 0.4623, 0.4876, 0.5049, 0.4820, 0.4956, 0.5019]							)
												# TODO: lower the tolerance after finding the cause of onnxruntime reproducibility issues

												assert np.abs(image_slice.flatten() - expected_slice							).max() < 2E-2



					def       lowerCAmelCase				(  self				:	int							)			->  Any:


												"""simple docstring"""

												__lowercase      :							Dict				=     load_image(
												    """https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"""
												    """/img2img/sketch-mountains-input.jpg"""							)
												__lowercase      :							Optional[Any]				=     init_image.resize((768, 512)							)
												__lowercase      :							Dict				=     LMSDiscreteScheduler.from_pretrained(
												    """runwayml/stable-diffusion-v1-5"""		,     subfolder="""scheduler"""		,     revision="""onnx"""							)
												__lowercase      :							Optional[Any]				=     OnnxStableDiffusionImgaImgPipeline.from_pretrained(
												    """runwayml/stable-diffusion-v1-5"""		,     revision="""onnx"""		,     scheduler=__a		,     safety_checker=__a		,     feature_extractor=__a		,     provider=self.gpu_provider		,     sess_options=self.gpu_options		,     )
												pipe.set_progress_bar_config(disable=__a							)

												__lowercase      :							Any				=     """A fantasy landscape, trending on artstation"""

												__lowercase      :							Dict				=     np.random.RandomState(0							)
												__lowercase      :							Optional[int]				=     pipe(
												    prompt=__a		,     image=__a		,     strength=0.75		,     guidance_scale=7.5		,     num_inference_steps=20		,     generator=__a		,     output_type="""np"""		,     )
												__lowercase      :							str				=     output.images
												__lowercase      :							Tuple				=     images[0, 255:258, 383:386, -1]

												assert images.shape == (1, 512, 768, 3)
												__lowercase      :							str				=     np.array([0.8043, 0.926, 0.9581, 0.8119, 0.8954, 0.913, 0.7209, 0.7463, 0.7431]							)
												# TODO: lower the tolerance after finding the cause of onnxruntime reproducibility issues

												assert np.abs(image_slice.flatten() - expected_slice							).max() < 2E-2







import itertools
import json
import os
import unittest

from transformers import AddedToken, LongformerTokenizer, LongformerTokenizerFast
from transformers.models.longformer.tokenization_longformer import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 		lowerCAmelCase     ( __a      ,       unittest.TestCase			):

					'''simple docstring'''

					_A						:			str									=      LongformerTokenizer
					_A						:			int									=      True
					_A						:			Optional[int]									=      LongformerTokenizerFast
					_A						:			int									=      True



					def       lowerCAmelCase				(  self				:	Any							)			->  List[str]:


												"""simple docstring"""

												super().setUp()

												# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
												__lowercase      :							Union[str, Any]				=     [
												    """l""",
												    """o""",
												    """w""",
												    """e""",
												    """r""",
												    """s""",
												    """t""",
												    """i""",
												    """d""",
												    """n""",
												    """\u0120""",
												    """\u0120l""",
												    """\u0120n""",
												    """\u0120lo""",
												    """\u0120low""",
												    """er""",
												    """\u0120lowest""",
												    """\u0120newer""",
												    """\u0120wider""",
												    """<unk>""",
												]
												__lowercase      :							Union[str, Any]				=     dict(zip(__a		,     range(len(__a							)							)							)							)
												__lowercase      :							Any				=     ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
												__lowercase      :							Optional[int]				=     {"""unk_token""": """<unk>"""}

												__lowercase      :							Dict				=     os.path.join(self.tmpdirname		,     VOCAB_FILES_NAMES["""vocab_file"""]							)
												__lowercase      :							str				=     os.path.join(self.tmpdirname		,     VOCAB_FILES_NAMES["""merges_file"""]							)
												with open(self.vocab_file		,     """w"""		,     encoding="""utf-8"""							) as fp:
																			fp.write(json.dumps(__a							) + """\n"""							)
												with open(self.merges_file		,     """w"""		,     encoding="""utf-8"""							) as fp:
																			fp.write("""\n""".join(__a							)							)



					def       lowerCAmelCase				(  self				:	Optional[int]		,     **__a				:	Optional[Any]							)			->  str:


												"""simple docstring"""

												kwargs.update(self.special_tokens_map							)
												return self.tokenizer_class.from_pretrained(self.tmpdirname		,     **__a							)



					def       lowerCAmelCase				(  self				:	Tuple		,     **__a				:	Tuple							)			->  str:


												"""simple docstring"""

												kwargs.update(self.special_tokens_map							)
												return self.rust_tokenizer_class.from_pretrained(self.tmpdirname		,     **__a							)



					def       lowerCAmelCase				(  self				:	str		,     __a				:	Optional[int]							)			->  Union[str, Any]:


												"""simple docstring"""

												__lowercase      :							Union[str, Any]				=     """lower newer"""
												__lowercase      :							int				=     """lower newer"""
												return input_text, output_text



					def       lowerCAmelCase				(  self				:	List[Any]							)			->  Optional[int]:


												"""simple docstring"""

												__lowercase      :							Union[str, Any]				=     self.tokenizer_class(self.vocab_file		,     self.merges_file		,     **self.special_tokens_map							)
												__lowercase      :							Dict				=     """lower newer"""
												__lowercase      :							Optional[Any]				=     ["""l""", """o""", """w""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
												__lowercase      :							str				=     tokenizer.tokenize(__a							)  # , add_prefix_space=True)
												self.assertListEqual(__a		,     __a							)

												__lowercase      :							int				=     tokens + [tokenizer.unk_token]
												__lowercase      :							str				=     [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
												self.assertListEqual(tokenizer.convert_tokens_to_ids(__a							)		,     __a							)



					def       lowerCAmelCase				(  self				:	Optional[Any]							)			->  List[Any]:


												"""simple docstring"""

												__lowercase      :							Optional[Any]				=     self.get_tokenizer()

												self.assertListEqual(tokenizer.encode("""Hello world!"""		,     add_special_tokens=__a							)		,     [0, 31414, 232, 328, 2]							)
												self.assertListEqual(
												    tokenizer.encode("""Hello world! cécé herlolip 418"""		,     add_special_tokens=__a							)		,     [0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]		,     )



					@slow
					def       lowerCAmelCase				(  self				:	Tuple							)			->  str:


												"""simple docstring"""

												__lowercase      :							Any				=     self.tokenizer_class.from_pretrained("""allenai/longformer-base-4096"""							)

												__lowercase      :							Optional[Any]				=     tokenizer.encode("""sequence builders"""		,     add_special_tokens=__a							)
												__lowercase      :							List[str]				=     tokenizer.encode("""multi-sequence build"""		,     add_special_tokens=__a							)

												__lowercase      :							Optional[Any]				=     tokenizer.encode(
												    """sequence builders"""		,     add_special_tokens=__a		,     add_prefix_space=__a							)
												__lowercase      :							Union[str, Any]				=     tokenizer.encode(
												    """sequence builders"""		,     """multi-sequence build"""		,     add_special_tokens=__a		,     add_prefix_space=__a							)

												__lowercase      :							List[Any]				=     tokenizer.build_inputs_with_special_tokens(__a							)
												__lowercase      :							Any				=     tokenizer.build_inputs_with_special_tokens(__a		,     __a							)

												assert encoded_sentence == encoded_text_from_decode
												assert encoded_pair == encoded_pair_from_decode



					def       lowerCAmelCase				(  self				:	Dict							)			->  Tuple:


												"""simple docstring"""

												__lowercase      :							Optional[Any]				=     self.get_tokenizer()

												__lowercase      :							Tuple				=     """Encode this sequence."""
												__lowercase      :							Optional[Any]				=     tokenizer.byte_encoder[""" """.encode("""utf-8"""							)[0]]

												# Testing encoder arguments
												__lowercase      :							Dict				=     tokenizer.encode(__a		,     add_special_tokens=__a		,     add_prefix_space=__a							)
												__lowercase      :							Tuple				=     tokenizer.convert_ids_to_tokens(encoded[0]							)[0]
												self.assertNotEqual(__a		,     __a							)

												__lowercase      :							List[str]				=     tokenizer.encode(__a		,     add_special_tokens=__a		,     add_prefix_space=__a							)
												__lowercase      :							Any				=     tokenizer.convert_ids_to_tokens(encoded[0]							)[0]
												self.assertEqual(__a		,     __a							)

												tokenizer.add_special_tokens({"""bos_token""": """<s>"""}							)
												__lowercase      :							str				=     tokenizer.encode(__a		,     add_special_tokens=__a							)
												__lowercase      :							Dict				=     tokenizer.convert_ids_to_tokens(encoded[1]							)[0]
												self.assertNotEqual(__a		,     __a							)

												# Testing spaces after special tokens
												__lowercase      :							List[Any]				=     """<mask>"""
												tokenizer.add_special_tokens(
												    {"""mask_token""": AddedToken(__a		,     lstrip=__a		,     rstrip=__a							)}							)  # mask token has a left space
												__lowercase      :							Dict				=     tokenizer.convert_tokens_to_ids(__a							)

												__lowercase      :							List[str]				=     """Encode <mask> sequence"""
												__lowercase      :							List[str]				=     """Encode <mask>sequence"""

												__lowercase      :							Union[str, Any]				=     tokenizer.encode(__a							)
												__lowercase      :							Dict				=     encoded.index(__a							)
												__lowercase      :							List[Any]				=     tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]							)[0]
												self.assertEqual(__a		,     __a							)

												__lowercase      :							int				=     tokenizer.encode(__a							)
												__lowercase      :							Union[str, Any]				=     encoded.index(__a							)
												__lowercase      :							List[Any]				=     tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]							)[0]
												self.assertNotEqual(__a		,     __a							)



					def       lowerCAmelCase				(  self				:	int							)			->  str:


												"""simple docstring"""

												pass



					def       lowerCAmelCase				(  self				:	int							)			->  Dict:


												"""simple docstring"""

												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																			with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"							):
																										__lowercase      :							List[str]				=     self.rust_tokenizer_class.from_pretrained(__a		,     **__a							)
																										__lowercase      :							List[Any]				=     self.tokenizer_class.from_pretrained(__a		,     **__a							)
																										__lowercase      :							Optional[Any]				=     """A, <mask> AllenNLP sentence."""
																										__lowercase      :							Union[str, Any]				=     tokenizer_r.encode_plus(__a		,     add_special_tokens=__a		,     return_token_type_ids=__a							)
																										__lowercase      :							Optional[Any]				=     tokenizer_p.encode_plus(__a		,     add_special_tokens=__a		,     return_token_type_ids=__a							)

																										# token_type_ids should put 0 everywhere
																										self.assertEqual(sum(tokens_r["""token_type_ids"""]							)		,     sum(tokens_p["""token_type_ids"""]							)							)

																										# attention_mask should put 1 everywhere, so sum over length should be 1
																										self.assertEqual(
																										    sum(tokens_r["""attention_mask"""]							) / len(tokens_r["""attention_mask"""]							)		,     sum(tokens_p["""attention_mask"""]							) / len(tokens_p["""attention_mask"""]							)		,     )

																										__lowercase      :							Dict				=     tokenizer_r.convert_ids_to_tokens(tokens_r["""input_ids"""]							)
																										__lowercase      :							str				=     tokenizer_p.convert_ids_to_tokens(tokens_p["""input_ids"""]							)

																										# Rust correctly handles the space before the mask while python doesnt
																										self.assertSequenceEqual(tokens_p["""input_ids"""]		,     [0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]							)
																										self.assertSequenceEqual(tokens_r["""input_ids"""]		,     [0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]							)

																										self.assertSequenceEqual(
																										    __a		,     ["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]							)
																										self.assertSequenceEqual(
																										    __a		,     ["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]							)



					def       lowerCAmelCase				(  self				:	List[Any]							)			->  Dict:


												"""simple docstring"""

												for trim_offsets, add_prefix_space in itertools.product([True, False]		,     repeat=2							):
																			__lowercase      :							Dict				=     self.rust_tokenizer_class.from_pretrained(
																			    self.tmpdirname		,     use_fast=__a		,     add_prefix_space=__a		,     trim_offsets=__a							)

																			__lowercase      :							Optional[int]				=     json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()							)
																			__lowercase      :							Any				=     json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()							)

																			self.assertEqual(pre_tokenizer_state["""add_prefix_space"""]		,     __a							)

																			self.assertEqual(post_processor_state["""add_prefix_space"""]		,     __a							)
																			self.assertEqual(post_processor_state["""trim_offsets"""]		,     __a							)



					def       lowerCAmelCase				(  self				:	int							)			->  Tuple:


												"""simple docstring"""

												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																			with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"							):
																										__lowercase      :							List[str]				=     """hello"""  # `hello` is a token in the vocabulary of `pretrained_name`
																										__lowercase      :							int				=     F"{text_of_1_token} {text_of_1_token}"

																										__lowercase      :							List[str]				=     self.rust_tokenizer_class.from_pretrained(
																										    __a		,     use_fast=__a		,     add_prefix_space=__a		,     trim_offsets=__a							)
																										__lowercase      :							Any				=     tokenizer_r(__a		,     return_offsets_mapping=__a		,     add_special_tokens=__a							)
																										self.assertEqual(encoding.offset_mapping[0]		,     (0, len(__a							))							)
																										self.assertEqual(
																										    encoding.offset_mapping[1]		,     (len(__a							) + 1, len(__a							) + 1 + len(__a							))		,     )

																										__lowercase      :							str				=     self.rust_tokenizer_class.from_pretrained(
																										    __a		,     use_fast=__a		,     add_prefix_space=__a		,     trim_offsets=__a							)
																										__lowercase      :							Tuple				=     tokenizer_r(__a		,     return_offsets_mapping=__a		,     add_special_tokens=__a							)
																										self.assertEqual(encoding.offset_mapping[0]		,     (0, len(__a							))							)
																										self.assertEqual(
																										    encoding.offset_mapping[1]		,     (len(__a							) + 1, len(__a							) + 1 + len(__a							))		,     )

																										__lowercase      :							Optional[int]				=     self.rust_tokenizer_class.from_pretrained(
																										    __a		,     use_fast=__a		,     add_prefix_space=__a		,     trim_offsets=__a							)
																										__lowercase      :							str				=     tokenizer_r(__a		,     return_offsets_mapping=__a		,     add_special_tokens=__a							)
																										self.assertEqual(encoding.offset_mapping[0]		,     (0, len(__a							))							)
																										self.assertEqual(
																										    encoding.offset_mapping[1]		,     (len(__a							), len(__a							) + 1 + len(__a							))		,     )

																										__lowercase      :							str				=     self.rust_tokenizer_class.from_pretrained(
																										    __a		,     use_fast=__a		,     add_prefix_space=__a		,     trim_offsets=__a							)
																										__lowercase      :							int				=     tokenizer_r(__a		,     return_offsets_mapping=__a		,     add_special_tokens=__a							)
																										self.assertEqual(encoding.offset_mapping[0]		,     (0, len(__a							))							)
																										self.assertEqual(
																										    encoding.offset_mapping[1]		,     (len(__a							), len(__a							) + 1 + len(__a							))		,     )

																										__lowercase      :							Any				=     F" {text}"

																										# tokenizer_r = self.rust_tokenizer_class.from_pretrained(
																										#     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
																										# )
																										# encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
																										# self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
																										# self.assertEqual(
																										#     encoding.offset_mapping[1],
																										#     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
																										# )

																										__lowercase      :							Optional[Any]				=     self.rust_tokenizer_class.from_pretrained(
																										    __a		,     use_fast=__a		,     add_prefix_space=__a		,     trim_offsets=__a							)
																										__lowercase      :							str				=     tokenizer_r(__a		,     return_offsets_mapping=__a		,     add_special_tokens=__a							)
																										self.assertEqual(encoding.offset_mapping[0]		,     (1, 1 + len(__a							))							)
																										self.assertEqual(
																										    encoding.offset_mapping[1]		,     (1 + len(__a							) + 1, 1 + len(__a							) + 1 + len(__a							))		,     )

																										__lowercase      :							int				=     self.rust_tokenizer_class.from_pretrained(
																										    __a		,     use_fast=__a		,     add_prefix_space=__a		,     trim_offsets=__a							)
																										__lowercase      :							Dict				=     tokenizer_r(__a		,     return_offsets_mapping=__a		,     add_special_tokens=__a							)
																										self.assertEqual(encoding.offset_mapping[0]		,     (0, 1 + len(__a							))							)
																										self.assertEqual(
																										    encoding.offset_mapping[1]		,     (1 + len(__a							), 1 + len(__a							) + 1 + len(__a							))		,     )

																										__lowercase      :							int				=     self.rust_tokenizer_class.from_pretrained(
																										    __a		,     use_fast=__a		,     add_prefix_space=__a		,     trim_offsets=__a							)
																										__lowercase      :							Tuple				=     tokenizer_r(__a		,     return_offsets_mapping=__a		,     add_special_tokens=__a							)
																										self.assertEqual(encoding.offset_mapping[0]		,     (0, 1 + len(__a							))							)
																										self.assertEqual(
																										    encoding.offset_mapping[1]		,     (1 + len(__a							), 1 + len(__a							) + 1 + len(__a							))		,     )