






'''simple docstring'''







import argparse
import json

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image
from torchvision import transforms

from transformers import BitImageProcessor, FocalNetConfig, FocalNetForImageClassification
from transformers.image_utils import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD, PILImageResampling





def 		_UpperCAmelCase   (							__A							:   int      ):
				a_		:  Any											=      [2, 2, 6, 2] if '''tiny''' in model_name else [2, 2, 18, 2]
				a_		:  List[Any]											=      True if '''large''' in model_name or '''huge''' in model_name else False
				a_		:  str											=      True if '''large''' in model_name or '''huge''' in model_name else False
				a_		:  Optional[Any]											=      True if '''large''' in model_name or '''huge''' in model_name else False

				if "large" in model_name or "xlarge" in model_name or "huge" in model_name:
								if "fl3" in model_name:
												a_		:  Any											=      [3, 3, 3, 3]
												a_		:  Union[str, Any]											=      [5, 5, 5, 5]
								elif "fl4" in model_name:
												a_		:  Optional[Any]											=      [4, 4, 4, 4]
												a_		:  List[Any]											=      [3, 3, 3, 3]

				if "tiny" in model_name or "small" in model_name or "base" in model_name:
								a_		:  List[Any]											=      [3, 3, 3, 3]
								if "lrf" in model_name:
												a_		:  List[str]											=      [3, 3, 3, 3]
								else:
												a_		:  int											=      [2, 2, 2, 2]

				if "tiny" in model_name:
								a_		:  Any											=      96
				elif "small" in model_name:
								a_		:  int											=      96
				elif "base" in model_name:
								a_		:  Any											=      1_28
				elif "large" in model_name:
								a_		:  Union[str, Any]											=      1_92
				elif "xlarge" in model_name:
								a_		:  Optional[Any]											=      2_56
				elif "huge" in model_name:
								a_		:  Optional[int]											=      3_52

				# set label information
				a_		:  Any											=      '''huggingface/label-files'''
				if "large" in model_name or "huge" in model_name:
								a_		:  Tuple											=      '''imagenet-22k-id2label.json'''
				else:
								a_		:  Dict											=      '''imagenet-1k-id2label.json'''

				a_		:  Optional[int]											=      json.load(open(hf_hub_download(__A						,		__A						,		repo_type='''dataset'''      )						,		'''r'''      )      )
				a_		:  Optional[Any]											=      {int(__A      ): v for k, v in idalabel.items()}
				a_		:  Union[str, Any]											=      {v: k for k, v in idalabel.items()}

				a_		:  List[str]											=      FocalNetConfig(
				    embed_dim=__A						,		depths=__A						,		focal_levels=__A						,		focal_windows=__A						,		use_conv_embed=__A						,		idalabel=__A						,		labelaid=__A						,		use_post_layernorm=__A						,		use_layerscale=__A						,		)

				return config





def 		_UpperCAmelCase   (							__A							:   Tuple      ):
				if "patch_embed.proj" in name:
								a_		:  Dict											=      name.replace('''patch_embed.proj'''						,		'''embeddings.patch_embeddings.projection'''      )
				if "patch_embed.norm" in name:
								a_		:  Optional[int]											=      name.replace('''patch_embed.norm'''						,		'''embeddings.norm'''      )
				if "layers" in name:
								a_		:  Tuple											=      '''encoder.''' + name
				if "encoder.layers" in name:
								a_		:  str											=      name.replace('''encoder.layers'''						,		'''encoder.stages'''      )
				if "downsample.proj" in name:
								a_		:  Optional[Any]											=      name.replace('''downsample.proj'''						,		'''downsample.projection'''      )
				if "blocks" in name:
								a_		:  Optional[Any]											=      name.replace('''blocks'''						,		'''layers'''      )
				if "modulation.f.weight" in name or "modulation.f.bias" in name:
								a_		:  Dict											=      name.replace('''modulation.f'''						,		'''modulation.projection_in'''      )
				if "modulation.h.weight" in name or "modulation.h.bias" in name:
								a_		:  Optional[Any]											=      name.replace('''modulation.h'''						,		'''modulation.projection_context'''      )
				if "modulation.proj.weight" in name or "modulation.proj.bias" in name:
								a_		:  int											=      name.replace('''modulation.proj'''						,		'''modulation.projection_out'''      )

				if name == "norm.weight":
								a_		:  Dict											=      '''layernorm.weight'''
				if name == "norm.bias":
								a_		:  List[str]											=      '''layernorm.bias'''

				if "head" in name:
								a_		:  List[Any]											=      name.replace('''head'''						,		'''classifier'''      )
				else:
								a_		:  Tuple											=      '''focalnet.''' + name

				return name





def 		_UpperCAmelCase   (							__A							:   List[Any]						,		__A							:   Optional[Any]						,		__A							:   str=False      ):
				# fmt: off
				a_		:  Tuple											=      {
				    '''focalnet-tiny''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_tiny_srf.pth''',
				    '''focalnet-tiny-lrf''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_tiny_lrf.pth''',
				    '''focalnet-small''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_small_srf.pth''',
				    '''focalnet-small-lrf''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_small_lrf.pth''',
				    '''focalnet-base''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_base_srf.pth''',
				    '''focalnet-base-lrf''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_base_lrf.pth''',
				    '''focalnet-large-lrf-fl3''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_large_lrf_384.pth''',
				    '''focalnet-large-lrf-fl4''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_large_lrf_384_fl4.pth''',
				    '''focalnet-xlarge-lrf-fl3''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_xlarge_lrf_384.pth''',
				    '''focalnet-xlarge-lrf-fl4''': '''https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_xlarge_lrf_384_fl4.pth''',
				}
				# fmt: on

				a_		:  Any											=      model_name_to_url[model_name]
				print('''Checkpoint URL: '''						,		__A      )
				a_		:  Optional[Any]											=      torch.hub.load_state_dict_from_url(__A						,		map_location='''cpu'''      )['''model''']

				# rename keys
				for key in state_dict.copy().keys():
								a_		:  Union[str, Any]											=      state_dict.pop(__A      )
								a_		:  Optional[int]											=      val

				a_		:  Tuple											=      get_focalnet_config(__A      )
				a_		:  str											=      FocalNetForImageClassification(__A      )
				model.eval()

				# load state dict
				model.load_state_dict(__A      )

				# verify conversion
				a_		:  str											=      '''http://images.cocodataset.org/val2017/000000039769.jpg'''

				a_		:  Any											=      BitImageProcessor(
				    do_resize=__A						,		size={'''shortest_edge''': 2_56}						,		resample=PILImageResampling.BILINEAR						,		do_center_crop=__A						,		crop_size=2_24						,		do_normalize=__A						,		image_mean=__A						,		image_std=__A						,		)
				a_		:  Optional[int]											=      Image.open(requests.get(__A						,		stream=__A      ).raw      )
				a_		:  Optional[Any]											=      processor(images=__A						,		return_tensors='''pt'''      )

				a_		:  Union[str, Any]											=      transforms.Compose(
				    [
				        transforms.Resize(2_56      ),
				        transforms.CenterCrop(2_24      ),
				        transforms.ToTensor(),
				        transforms.Normalize(mean=[0.485, 0.456, 0.406]						,		std=[0.229, 0.224, 0.225]      ),
				    ]      )

				a_		:  int											=      image_transforms(__A      ).unsqueeze(0      )

				# verify pixel_values
				assert torch.allclose(inputs.pixel_values						,		__A						,		atol=1E-4      )

				a_		:  List[str]											=      model(**__A      )

				a_		:  Optional[Any]											=      outputs.logits.argmax(-1      ).item()
				print('''Predicted class:'''						,		model.config.idalabel[predicted_class_idx]      )

				print('''First values of logits:'''						,		outputs.logits[0, :3]      )

				if model_name == "focalnet-tiny":
								a_		:  Union[str, Any]											=      torch.tensor([0.2166, -0.4368, 0.2191]      )
				elif model_name == "focalnet-tiny-lrf":
								a_		:  List[Any]											=      torch.tensor([1.1669, 0.0125, -0.1695]      )
				elif model_name == "focalnet-small":
								a_		:  List[Any]											=      torch.tensor([0.4917, -0.0430, 0.1341]      )
				elif model_name == "focalnet-small-lrf":
								a_		:  Optional[Any]											=      torch.tensor([-0.2588, -0.5342, -0.2331]      )
				elif model_name == "focalnet-base":
								a_		:  Any											=      torch.tensor([-0.1655, -0.4090, -0.1730]      )
				elif model_name == "focalnet-base-lrf":
								a_		:  Any											=      torch.tensor([0.5306, -0.0483, -0.3928]      )
				assert torch.allclose(outputs.logits[0, :3]						,		__A						,		atol=1E-4      )
				print('''Looks ok!'''      )

				if pytorch_dump_folder_path is not None:
								print(f'Saving model and processor of {model_name} to {pytorch_dump_folder_path}'      )
								model.save_pretrained(__A      )
								processor.save_pretrained(__A      )

				if push_to_hub:
								print(f'Pushing model and processor of {model_name} to the hub...'      )
								model.push_to_hub(f'{model_name}'      )
								processor.push_to_hub(f'{model_name}'      )


if __name__ == "__main__":
		__lowerCAmelCase       		=      argparse.ArgumentParser()
		# Required parameters
		parser.add_argument(
		    '''--model_name''',
		    default='''focalnet-tiny''',
		    type=str,
		    help='''Name of the FocalNet model you\'d like to convert.''',
		)
		parser.add_argument(
		    '''--pytorch_dump_folder_path''', default=None, type=str, help='''Path to the output PyTorch model directory.'''
		)
		parser.add_argument(
		    '''--push_to_hub''',
		    action='''store_true''',
		    help='''Whether to push the model and processor to the hub.''',
		)

		__lowerCAmelCase       		=      parser.parse_args()
		convert_focalnet_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)












'''simple docstring'''







import itertools
import json
import os
import unittest

from transformers import AddedToken, LongformerTokenizer, LongformerTokenizerFast
from transformers.models.longformer.tokenization_longformer import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 	SCREAMING_SNAKE_CASE					(					SCREAMING_SNAKE_CASE_	,   unittest.TestCase       ):
			snake_case__  =				LongformerTokenizer
			snake_case__  =				True
			snake_case__  =				LongformerTokenizerFast
			snake_case__  =				True






			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[Any]					)			->			List[str]:
							super().setUp()

							# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
							a_		:  Tuple											=      [
							    '''l''',
							    '''o''',
							    '''w''',
							    '''e''',
							    '''r''',
							    '''s''',
							    '''t''',
							    '''i''',
							    '''d''',
							    '''n''',
							    '''\u0120''',
							    '''\u0120l''',
							    '''\u0120n''',
							    '''\u0120lo''',
							    '''\u0120low''',
							    '''er''',
							    '''\u0120lowest''',
							    '''\u0120newer''',
							    '''\u0120wider''',
							    '''<unk>''',
							]
							a_		:  Optional[Any]											=      dict(zip(__SCREAMING_SNAKE_CASE  ,						range(len(__SCREAMING_SNAKE_CASE					)					)					)					)
							a_		:  Union[str, Any]											=      ['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
							a_		:  Any											=      {'''unk_token''': '''<unk>'''}

							a_		:  Union[str, Any]											=      os.path.join(self.tmpdirname  ,						VOCAB_FILES_NAMES['''vocab_file''']					)
							a_		:  Optional[int]											=      os.path.join(self.tmpdirname  ,						VOCAB_FILES_NAMES['''merges_file''']					)
							with open(self.vocab_file  ,						'''w'''  ,						encoding='''utf-8'''					) as fp:
											fp.write(json.dumps(__SCREAMING_SNAKE_CASE					) + '''\n'''					)
							with open(self.merges_file  ,						'''w'''  ,						encoding='''utf-8'''					) as fp:
											fp.write('''\n'''.join(__SCREAMING_SNAKE_CASE					)					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Any  ,						**__SCREAMING_SNAKE_CASE      :			Any					)			->			int:
							kwargs.update(self.special_tokens_map					)
							return self.tokenizer_class.from_pretrained(self.tmpdirname  ,						**__SCREAMING_SNAKE_CASE					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Union[str, Any]  ,						**__SCREAMING_SNAKE_CASE      :			List[Any]					)			->			List[str]:
							kwargs.update(self.special_tokens_map					)
							return self.rust_tokenizer_class.from_pretrained(self.tmpdirname  ,						**__SCREAMING_SNAKE_CASE					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Dict  ,						__SCREAMING_SNAKE_CASE      :			List[Any]					)			->			Any:
							a_		:  Union[str, Any]											=      '''lower newer'''
							a_		:  List[Any]											=      '''lower newer'''
							return input_text, output_text






			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[Any]					)			->			List[str]:
							a_		:  Optional[Any]											=      self.tokenizer_class(self.vocab_file  ,						self.merges_file  ,						**self.special_tokens_map					)
							a_		:  List[str]											=      '''lower newer'''
							a_		:  str											=      ['''l''', '''o''', '''w''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
							a_		:  Optional[int]											=      tokenizer.tokenize(__SCREAMING_SNAKE_CASE					)  # , add_prefix_space=True)
							self.assertListEqual(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)

							a_		:  Dict											=      tokens + [tokenizer.unk_token]
							a_		:  Any											=      [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
							self.assertListEqual(tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE					)  ,						__SCREAMING_SNAKE_CASE					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			List[str]					)			->			Optional[Any]:
							a_		:  Union[str, Any]											=      self.get_tokenizer()

							self.assertListEqual(tokenizer.encode('''Hello world!'''  ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)  ,						[0, 3_1414, 232, 328, 2]					)
							self.assertListEqual(
							    tokenizer.encode('''Hello world! cécé herlolip 418'''  ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)  ,						[0, 3_1414, 232, 328, 740, 1140, 1_2695, 69, 4_6078, 1588, 2]  ,						)






			@slow
			def 			SCREAMING_SNAKE_CASE      ( self      :			int					)			->			Optional[Any]:
							a_		:  Dict											=      self.tokenizer_class.from_pretrained('''allenai/longformer-base-4096'''					)

							a_		:  Tuple											=      tokenizer.encode('''sequence builders'''  ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
							a_		:  Union[str, Any]											=      tokenizer.encode('''multi-sequence build'''  ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)

							a_		:  Any											=      tokenizer.encode(
							    '''sequence builders'''  ,						add_special_tokens=__SCREAMING_SNAKE_CASE  ,						add_prefix_space=__SCREAMING_SNAKE_CASE					)
							a_		:  Any											=      tokenizer.encode(
							    '''sequence builders'''  ,						'''multi-sequence build'''  ,						add_special_tokens=__SCREAMING_SNAKE_CASE  ,						add_prefix_space=__SCREAMING_SNAKE_CASE					)

							a_		:  List[str]											=      tokenizer.build_inputs_with_special_tokens(__SCREAMING_SNAKE_CASE					)
							a_		:  Union[str, Any]											=      tokenizer.build_inputs_with_special_tokens(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)

							assert encoded_sentence == encoded_text_from_decode
							assert encoded_pair == encoded_pair_from_decode






			def 			SCREAMING_SNAKE_CASE      ( self      :			str					)			->			int:
							a_		:  str											=      self.get_tokenizer()

							a_		:  int											=      '''Encode this sequence.'''
							a_		:  List[str]											=      tokenizer.byte_encoder[''' '''.encode('''utf-8'''					)[0]]

							# Testing encoder arguments
							a_		:  Dict											=      tokenizer.encode(__SCREAMING_SNAKE_CASE  ,						add_special_tokens=__SCREAMING_SNAKE_CASE  ,						add_prefix_space=__SCREAMING_SNAKE_CASE					)
							a_		:  Optional[Any]											=      tokenizer.convert_ids_to_tokens(encoded[0]					)[0]
							self.assertNotEqual(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)

							a_		:  Dict											=      tokenizer.encode(__SCREAMING_SNAKE_CASE  ,						add_special_tokens=__SCREAMING_SNAKE_CASE  ,						add_prefix_space=__SCREAMING_SNAKE_CASE					)
							a_		:  Any											=      tokenizer.convert_ids_to_tokens(encoded[0]					)[0]
							self.assertEqual(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)

							tokenizer.add_special_tokens({'''bos_token''': '''<s>'''}					)
							a_		:  Dict											=      tokenizer.encode(__SCREAMING_SNAKE_CASE  ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
							a_		:  Dict											=      tokenizer.convert_ids_to_tokens(encoded[1]					)[0]
							self.assertNotEqual(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)

							# Testing spaces after special tokens
							a_		:  Optional[Any]											=      '''<mask>'''
							tokenizer.add_special_tokens(
							    {'''mask_token''': AddedToken(__SCREAMING_SNAKE_CASE  ,						lstrip=__SCREAMING_SNAKE_CASE  ,						rstrip=__SCREAMING_SNAKE_CASE					)}					)  # mask token has a left space
							a_		:  Optional[int]											=      tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE					)

							a_		:  List[Any]											=      '''Encode <mask> sequence'''
							a_		:  List[str]											=      '''Encode <mask>sequence'''

							a_		:  int											=      tokenizer.encode(__SCREAMING_SNAKE_CASE					)
							a_		:  Optional[int]											=      encoded.index(__SCREAMING_SNAKE_CASE					)
							a_		:  Optional[int]											=      tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]					)[0]
							self.assertEqual(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)

							a_		:  Union[str, Any]											=      tokenizer.encode(__SCREAMING_SNAKE_CASE					)
							a_		:  Optional[int]											=      encoded.index(__SCREAMING_SNAKE_CASE					)
							a_		:  str											=      tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]					)[0]
							self.assertNotEqual(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[int]					)			->			Union[str, Any]:
							pass






			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[Any]					)			->			Any:
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'					):
															a_		:  Any											=      self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE  ,						**__SCREAMING_SNAKE_CASE					)
															a_		:  Any											=      self.tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE  ,						**__SCREAMING_SNAKE_CASE					)
															a_		:  str											=      '''A, <mask> AllenNLP sentence.'''
															a_		:  List[Any]											=      tokenizer_r.encode_plus(__SCREAMING_SNAKE_CASE  ,						add_special_tokens=__SCREAMING_SNAKE_CASE  ,						return_token_type_ids=__SCREAMING_SNAKE_CASE					)
															a_		:  Dict											=      tokenizer_p.encode_plus(__SCREAMING_SNAKE_CASE  ,						add_special_tokens=__SCREAMING_SNAKE_CASE  ,						return_token_type_ids=__SCREAMING_SNAKE_CASE					)

															# token_type_ids should put 0 everywhere
															self.assertEqual(sum(tokens_r['''token_type_ids''']					)  ,						sum(tokens_p['''token_type_ids''']					)					)

															# attention_mask should put 1 everywhere, so sum over length should be 1
															self.assertEqual(
															    sum(tokens_r['''attention_mask''']					) / len(tokens_r['''attention_mask''']					)  ,						sum(tokens_p['''attention_mask''']					) / len(tokens_p['''attention_mask''']					)  ,						)

															a_		:  str											=      tokenizer_r.convert_ids_to_tokens(tokens_r['''input_ids''']					)
															a_		:  Tuple											=      tokenizer_p.convert_ids_to_tokens(tokens_p['''input_ids''']					)

															# Rust correctly handles the space before the mask while python doesnt
															self.assertSequenceEqual(tokens_p['''input_ids''']  ,						[0, 250, 6, 5_0264, 3823, 487, 2_1992, 3645, 4, 2]					)
															self.assertSequenceEqual(tokens_r['''input_ids''']  ,						[0, 250, 6, 5_0264, 3823, 487, 2_1992, 3645, 4, 2]					)

															self.assertSequenceEqual(
															    __SCREAMING_SNAKE_CASE  ,						['''<s>''', '''A''', ''',''', '''<mask>''', '''ĠAllen''', '''N''', '''LP''', '''Ġsentence''', '''.''', '''</s>''']					)
															self.assertSequenceEqual(
															    __SCREAMING_SNAKE_CASE  ,						['''<s>''', '''A''', ''',''', '''<mask>''', '''ĠAllen''', '''N''', '''LP''', '''Ġsentence''', '''.''', '''</s>''']					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[Any]					)			->			List[Any]:
							for trim_offsets, add_prefix_space in itertools.product([True, False]  ,						repeat=2					):
											a_		:  Any											=      self.rust_tokenizer_class.from_pretrained(
											    self.tmpdirname  ,						use_fast=__SCREAMING_SNAKE_CASE  ,						add_prefix_space=__SCREAMING_SNAKE_CASE  ,						trim_offsets=__SCREAMING_SNAKE_CASE					)

											a_		:  Union[str, Any]											=      json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()					)
											a_		:  str											=      json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()					)

											self.assertEqual(pre_tokenizer_state['''add_prefix_space''']  ,						__SCREAMING_SNAKE_CASE					)

											self.assertEqual(post_processor_state['''add_prefix_space''']  ,						__SCREAMING_SNAKE_CASE					)
											self.assertEqual(post_processor_state['''trim_offsets''']  ,						__SCREAMING_SNAKE_CASE					)







			def 			SCREAMING_SNAKE_CASE      ( self      :			int					)			->			Optional[int]:
							# Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space` and
							# `trim_offsets`
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'					):
															a_		:  Dict											=      '''hello'''  # `hello` is a token in the vocabulary of `pretrained_name`
															a_		:  Union[str, Any]											=      f'{text_of_1_token} {text_of_1_token}'

															a_		:  Any											=      self.rust_tokenizer_class.from_pretrained(
															    __SCREAMING_SNAKE_CASE  ,						use_fast=__SCREAMING_SNAKE_CASE  ,						add_prefix_space=__SCREAMING_SNAKE_CASE  ,						trim_offsets=__SCREAMING_SNAKE_CASE					)
															a_		:  Optional[int]											=      tokenizer_r(__SCREAMING_SNAKE_CASE  ,						return_offsets_mapping=__SCREAMING_SNAKE_CASE  ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
															self.assertEqual(encoding.offset_mapping[0]  ,						(0, len(__SCREAMING_SNAKE_CASE					))					)
															self.assertEqual(
															    encoding.offset_mapping[1]  ,						(len(__SCREAMING_SNAKE_CASE					) + 1, len(__SCREAMING_SNAKE_CASE					) + 1 + len(__SCREAMING_SNAKE_CASE					))  ,						)

															a_		:  Any											=      self.rust_tokenizer_class.from_pretrained(
															    __SCREAMING_SNAKE_CASE  ,						use_fast=__SCREAMING_SNAKE_CASE  ,						add_prefix_space=__SCREAMING_SNAKE_CASE  ,						trim_offsets=__SCREAMING_SNAKE_CASE					)
															a_		:  str											=      tokenizer_r(__SCREAMING_SNAKE_CASE  ,						return_offsets_mapping=__SCREAMING_SNAKE_CASE  ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
															self.assertEqual(encoding.offset_mapping[0]  ,						(0, len(__SCREAMING_SNAKE_CASE					))					)
															self.assertEqual(
															    encoding.offset_mapping[1]  ,						(len(__SCREAMING_SNAKE_CASE					) + 1, len(__SCREAMING_SNAKE_CASE					) + 1 + len(__SCREAMING_SNAKE_CASE					))  ,						)

															a_		:  int											=      self.rust_tokenizer_class.from_pretrained(
															    __SCREAMING_SNAKE_CASE  ,						use_fast=__SCREAMING_SNAKE_CASE  ,						add_prefix_space=__SCREAMING_SNAKE_CASE  ,						trim_offsets=__SCREAMING_SNAKE_CASE					)
															a_		:  Union[str, Any]											=      tokenizer_r(__SCREAMING_SNAKE_CASE  ,						return_offsets_mapping=__SCREAMING_SNAKE_CASE  ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
															self.assertEqual(encoding.offset_mapping[0]  ,						(0, len(__SCREAMING_SNAKE_CASE					))					)
															self.assertEqual(
															    encoding.offset_mapping[1]  ,						(len(__SCREAMING_SNAKE_CASE					), len(__SCREAMING_SNAKE_CASE					) + 1 + len(__SCREAMING_SNAKE_CASE					))  ,						)

															a_		:  Tuple											=      self.rust_tokenizer_class.from_pretrained(
															    __SCREAMING_SNAKE_CASE  ,						use_fast=__SCREAMING_SNAKE_CASE  ,						add_prefix_space=__SCREAMING_SNAKE_CASE  ,						trim_offsets=__SCREAMING_SNAKE_CASE					)
															a_		:  Any											=      tokenizer_r(__SCREAMING_SNAKE_CASE  ,						return_offsets_mapping=__SCREAMING_SNAKE_CASE  ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
															self.assertEqual(encoding.offset_mapping[0]  ,						(0, len(__SCREAMING_SNAKE_CASE					))					)
															self.assertEqual(
															    encoding.offset_mapping[1]  ,						(len(__SCREAMING_SNAKE_CASE					), len(__SCREAMING_SNAKE_CASE					) + 1 + len(__SCREAMING_SNAKE_CASE					))  ,						)

															a_		:  Union[str, Any]											=      f' {text}'

															# tokenizer_r = self.rust_tokenizer_class.from_pretrained(
															#     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
															# )
															# encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
															# self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
															# self.assertEqual(
															#     encoding.offset_mapping[1],
															#     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
															# )

															a_		:  str											=      self.rust_tokenizer_class.from_pretrained(
															    __SCREAMING_SNAKE_CASE  ,						use_fast=__SCREAMING_SNAKE_CASE  ,						add_prefix_space=__SCREAMING_SNAKE_CASE  ,						trim_offsets=__SCREAMING_SNAKE_CASE					)
															a_		:  int											=      tokenizer_r(__SCREAMING_SNAKE_CASE  ,						return_offsets_mapping=__SCREAMING_SNAKE_CASE  ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
															self.assertEqual(encoding.offset_mapping[0]  ,						(1, 1 + len(__SCREAMING_SNAKE_CASE					))					)
															self.assertEqual(
															    encoding.offset_mapping[1]  ,						(1 + len(__SCREAMING_SNAKE_CASE					) + 1, 1 + len(__SCREAMING_SNAKE_CASE					) + 1 + len(__SCREAMING_SNAKE_CASE					))  ,						)

															a_		:  int											=      self.rust_tokenizer_class.from_pretrained(
															    __SCREAMING_SNAKE_CASE  ,						use_fast=__SCREAMING_SNAKE_CASE  ,						add_prefix_space=__SCREAMING_SNAKE_CASE  ,						trim_offsets=__SCREAMING_SNAKE_CASE					)
															a_		:  str											=      tokenizer_r(__SCREAMING_SNAKE_CASE  ,						return_offsets_mapping=__SCREAMING_SNAKE_CASE  ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
															self.assertEqual(encoding.offset_mapping[0]  ,						(0, 1 + len(__SCREAMING_SNAKE_CASE					))					)
															self.assertEqual(
															    encoding.offset_mapping[1]  ,						(1 + len(__SCREAMING_SNAKE_CASE					), 1 + len(__SCREAMING_SNAKE_CASE					) + 1 + len(__SCREAMING_SNAKE_CASE					))  ,						)

															a_		:  Optional[int]											=      self.rust_tokenizer_class.from_pretrained(
															    __SCREAMING_SNAKE_CASE  ,						use_fast=__SCREAMING_SNAKE_CASE  ,						add_prefix_space=__SCREAMING_SNAKE_CASE  ,						trim_offsets=__SCREAMING_SNAKE_CASE					)
															a_		:  int											=      tokenizer_r(__SCREAMING_SNAKE_CASE  ,						return_offsets_mapping=__SCREAMING_SNAKE_CASE  ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
															self.assertEqual(encoding.offset_mapping[0]  ,						(0, 1 + len(__SCREAMING_SNAKE_CASE					))					)
															self.assertEqual(
															    encoding.offset_mapping[1]  ,						(1 + len(__SCREAMING_SNAKE_CASE					), 1 + len(__SCREAMING_SNAKE_CASE					) + 1 + len(__SCREAMING_SNAKE_CASE					))  ,						)




