



"""simple docstring"""






import argparse
import json
from pathlib import Path

import requests
import timm
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import DeiTImageProcessor, ViTConfig, ViTForImageClassification, ViTImageProcessor, ViTModel
from transformers.utils import logging


logging.set_verbosity_info()
__snake_case							:     Optional[int]   =					logging.get_logger(__name__)


def     _lowercase       (						__snake_case      ,__snake_case=False			)		->     Optional[Any]:
			__lowerCAmelCase   :						str				= []
			for i in range(config.num_hidden_layers			):
						# encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
						rename_keys.append((F"""blocks.{i}.norm1.weight""", F"""vit.encoder.layer.{i}.layernorm_before.weight""")			)
						rename_keys.append((F"""blocks.{i}.norm1.bias""", F"""vit.encoder.layer.{i}.layernorm_before.bias""")			)
						rename_keys.append((F"""blocks.{i}.attn.proj.weight""", F"""vit.encoder.layer.{i}.attention.output.dense.weight""")			)
						rename_keys.append((F"""blocks.{i}.attn.proj.bias""", F"""vit.encoder.layer.{i}.attention.output.dense.bias""")			)
						rename_keys.append((F"""blocks.{i}.norm2.weight""", F"""vit.encoder.layer.{i}.layernorm_after.weight""")			)
						rename_keys.append((F"""blocks.{i}.norm2.bias""", F"""vit.encoder.layer.{i}.layernorm_after.bias""")			)
						rename_keys.append((F"""blocks.{i}.mlp.fc1.weight""", F"""vit.encoder.layer.{i}.intermediate.dense.weight""")			)
						rename_keys.append((F"""blocks.{i}.mlp.fc1.bias""", F"""vit.encoder.layer.{i}.intermediate.dense.bias""")			)
						rename_keys.append((F"""blocks.{i}.mlp.fc2.weight""", F"""vit.encoder.layer.{i}.output.dense.weight""")			)
						rename_keys.append((F"""blocks.{i}.mlp.fc2.bias""", F"""vit.encoder.layer.{i}.output.dense.bias""")			)

			# projection layer + position embeddings
			rename_keys.extend(
			    [
			        ("cls_token", "vit.embeddings.cls_token"),
			        ("patch_embed.proj.weight", "vit.embeddings.patch_embeddings.projection.weight"),
			        ("patch_embed.proj.bias", "vit.embeddings.patch_embeddings.projection.bias"),
			        ("pos_embed", "vit.embeddings.position_embeddings"),
			    ]			)

			if base_model:
						# layernorm + pooler
						rename_keys.extend(
						    [
						        ("norm.weight", "layernorm.weight"),
						        ("norm.bias", "layernorm.bias"),
						        ("pre_logits.fc.weight", "pooler.dense.weight"),
						        ("pre_logits.fc.bias", "pooler.dense.bias"),
						    ]			)

						# if just the base model, we should remove "vit" from all keys that start with "vit"
						__lowerCAmelCase   :						List[Any]				= [(pair[0], pair[1][4:]) if pair[1].startswith("vit"			) else pair for pair in rename_keys]
			else:
						# layernorm + classification head
						rename_keys.extend(
						    [
						        ("norm.weight", "vit.layernorm.weight"),
						        ("norm.bias", "vit.layernorm.bias"),
						        ("head.weight", "classifier.weight"),
						        ("head.bias", "classifier.bias"),
						    ]			)

			return rename_keys


def     _lowercase       (						__snake_case      ,__snake_case      ,__snake_case=False			)		->     int:
			for i in range(config.num_hidden_layers			):
						if base_model:
									__lowerCAmelCase   :						str				= ""
						else:
									__lowerCAmelCase   :						Any				= "vit."
						# read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
						__lowerCAmelCase   :						List[Any]				= state_dict.pop(F"""blocks.{i}.attn.qkv.weight"""			)
						__lowerCAmelCase   :						List[str]				= state_dict.pop(F"""blocks.{i}.attn.qkv.bias"""			)
						# next, add query, keys and values (in that order) to the state dict
						__lowerCAmelCase   :						Optional[Any]				= in_proj_weight[
						    : config.hidden_size, :
						]
						__lowerCAmelCase   :						Optional[int]				= in_proj_bias[: config.hidden_size]
						__lowerCAmelCase   :						List[Any]				= in_proj_weight[
						    config.hidden_size : config.hidden_size * 2, :
						]
						__lowerCAmelCase   :						Tuple				= in_proj_bias[
						    config.hidden_size : config.hidden_size * 2
						]
						__lowerCAmelCase   :						Optional[int]				= in_proj_weight[
						    -config.hidden_size :, :
						]
						__lowerCAmelCase   :						str				= in_proj_bias[-config.hidden_size :]


def     _lowercase       (						__snake_case			)		->     Optional[Any]:
			__lowerCAmelCase   :						Any				= ["head.weight", "head.bias"]
			for k in ignore_keys:
						state_dict.pop(__snake_case      ,__snake_case			)


def     _lowercase       (						__snake_case      ,__snake_case      ,__snake_case			)		->     List[Any]:
			__lowerCAmelCase   :						int				= dct.pop(__snake_case			)
			__lowerCAmelCase   :						str				= val


def     _lowercase       (						)		->     str:
			__lowerCAmelCase   :						Dict				= "http://images.cocodataset.org/val2017/000000039769.jpg"
			__lowerCAmelCase   :						Any				= Image.open(requests.get(__snake_case      ,stream=__snake_case			).raw			)
			return im


@torch.no_grad()
def     _lowercase       (						__snake_case      ,__snake_case			)		->     Union[str, Any]:
			__lowerCAmelCase   :						Tuple				= ViTConfig()
			__lowerCAmelCase   :						Tuple				= False
			# dataset (ImageNet-21k only or also fine-tuned on ImageNet 2012), patch_size and image_size
			if vit_name[-5:] == "in21k":
						__lowerCAmelCase   :						Optional[int]				= True
						__lowerCAmelCase   :						Dict				= int(vit_name[-12:-10]			)
						__lowerCAmelCase   :						Tuple				= int(vit_name[-9:-6]			)
			else:
						__lowerCAmelCase   :						str				= 1_000
						__lowerCAmelCase   :						Tuple				= "huggingface/label-files"
						__lowerCAmelCase   :						Optional[int]				= "imagenet-1k-id2label.json"
						__lowerCAmelCase   :						Tuple				= json.load(open(hf_hub_download(__snake_case      ,__snake_case      ,repo_type="dataset"			)      ,"r"			)			)
						__lowerCAmelCase   :						Optional[Any]				= {int(__snake_case			): v for k, v in idalabel.items()}
						__lowerCAmelCase   :						Union[str, Any]				= idalabel
						__lowerCAmelCase   :						int				= {v: k for k, v in idalabel.items()}
						__lowerCAmelCase   :						Optional[Any]				= int(vit_name[-6:-4]			)
						__lowerCAmelCase   :						Any				= int(vit_name[-3:]			)
			# size of the architecture
			if "deit" in vit_name:
						if vit_name[9:].startswith("tiny"			):
									__lowerCAmelCase   :						str				= 192
									__lowerCAmelCase   :						str				= 768
									__lowerCAmelCase   :						Dict				= 12
									__lowerCAmelCase   :						str				= 3
						elif vit_name[9:].startswith("small"			):
									__lowerCAmelCase   :						Tuple				= 384
									__lowerCAmelCase   :						List[Any]				= 1_536
									__lowerCAmelCase   :						Any				= 12
									__lowerCAmelCase   :						str				= 6
						else:
									pass
			else:
						if vit_name[4:].startswith("small"			):
									__lowerCAmelCase   :						Union[str, Any]				= 768
									__lowerCAmelCase   :						Union[str, Any]				= 2_304
									__lowerCAmelCase   :						str				= 8
									__lowerCAmelCase   :						List[Any]				= 8
						elif vit_name[4:].startswith("base"			):
									pass
						elif vit_name[4:].startswith("large"			):
									__lowerCAmelCase   :						str				= 1_024
									__lowerCAmelCase   :						List[str]				= 4_096
									__lowerCAmelCase   :						Any				= 24
									__lowerCAmelCase   :						List[Any]				= 16
						elif vit_name[4:].startswith("huge"			):
									__lowerCAmelCase   :						Union[str, Any]				= 1_280
									__lowerCAmelCase   :						List[Any]				= 5_120
									__lowerCAmelCase   :						str				= 32
									__lowerCAmelCase   :						Tuple				= 16

    # load original model from timm
			__lowerCAmelCase   :						List[str]				= timm.create_model(__snake_case      ,pretrained=__snake_case			)
			timm_model.eval()

			# load state_dict of original model, remove and rename some keys
			__lowerCAmelCase   :						Union[str, Any]				= timm_model.state_dict()
			if base_model:
						remove_classification_head_(__snake_case			)
			__lowerCAmelCase   :						str				= create_rename_keys(__snake_case      ,__snake_case			)
			for src, dest in rename_keys:
						rename_key(__snake_case      ,__snake_case      ,__snake_case			)
			read_in_q_k_v(__snake_case      ,__snake_case      ,__snake_case			)

			# load HuggingFace model
			if vit_name[-5:] == "in21k":
						__lowerCAmelCase   :						Tuple				= ViTModel(__snake_case			).eval()
			else:
						__lowerCAmelCase   :						List[str]				= ViTForImageClassification(__snake_case			).eval()
			model.load_state_dict(__snake_case			)

			# Check outputs on an image, prepared by ViTImageProcessor/DeiTImageProcessor
			if "deit" in vit_name:
						__lowerCAmelCase   :						Tuple				= DeiTImageProcessor(size=config.image_size			)
			else:
						__lowerCAmelCase   :						Optional[int]				= ViTImageProcessor(size=config.image_size			)
			__lowerCAmelCase   :						Dict				= image_processor(images=prepare_img()      ,return_tensors="pt"			)
			__lowerCAmelCase   :						Optional[int]				= encoding["pixel_values"]
			__lowerCAmelCase   :						int				= model(__snake_case			)

			if base_model:
						__lowerCAmelCase   :						Tuple				= timm_model.forward_features(__snake_case			)
						assert timm_pooled_output.shape == outputs.pooler_output.shape
						assert torch.allclose(__snake_case      ,outputs.pooler_output      ,atol=1e-3			)
			else:
						__lowerCAmelCase   :						Optional[int]				= timm_model(__snake_case			)
						assert timm_logits.shape == outputs.logits.shape
						assert torch.allclose(__snake_case      ,outputs.logits      ,atol=1e-3			)

			Path(__snake_case			).mkdir(exist_ok=__snake_case			)
			print(F"""Saving model {vit_name} to {pytorch_dump_folder_path}"""			)
			model.save_pretrained(__snake_case			)
			print(F"""Saving image processor to {pytorch_dump_folder_path}"""			)
			image_processor.save_pretrained(__snake_case			)


if __name__ == "__main__":
	__snake_case							:     Union[str, Any]   =					argparse.ArgumentParser()
	# Required parameters
	parser.add_argument(
	    '--vit_name',
	    default='vit_base_patch16_224',
	    type=str,
	    help='Name of the ViT timm model you\'d like to convert.',
	)
	parser.add_argument(
	    '--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model directory.'
	)

	__snake_case							:     Any   =					parser.parse_args()
	convert_vit_checkpoint(args.vit_name, args.pytorch_dump_folder_path)





"""simple docstring"""






import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class     A__     (						__SCREAMING_SNAKE_CASE						,	unittest.TestCase							):




	'''simple docstring'''





	SCREAMING_SNAKE_CASE					   =			RoCBertTokenizer
	SCREAMING_SNAKE_CASE					   =			None
	SCREAMING_SNAKE_CASE					   =			False
	SCREAMING_SNAKE_CASE					   =			True
	SCREAMING_SNAKE_CASE					   =			filter_non_english

	def 							_SCREAMING_SNAKE_CASE     (  self:	int)       -> Optional[Any]:

				"""simple docstring"""

				super().setUp()

				__lowerCAmelCase   :						Dict				= ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "你", "好", "是", "谁", "a", "b", "c", "d"]
				__lowerCAmelCase   :						str				= {}
				__lowerCAmelCase   :						List[Any]				= {}
				for i, value in enumerate(_SCREAMING_SNAKE_CASE):
							__lowerCAmelCase   :						int				= i
							__lowerCAmelCase   :						List[str]				= i
				__lowerCAmelCase   :						int				= os.path.join(self.tmpdirname ,    VOCAB_FILES_NAMES["vocab_file"])
				__lowerCAmelCase   :						Union[str, Any]				= os.path.join(self.tmpdirname ,    VOCAB_FILES_NAMES["word_shape_file"])
				__lowerCAmelCase   :						Any				= os.path.join(self.tmpdirname ,    VOCAB_FILES_NAMES["word_pronunciation_file"])
				with open(self.vocab_file ,    "w" ,    encoding="utf-8") as vocab_writer:
							vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))
				with open(self.word_shape_file ,    "w" ,    encoding="utf-8") as word_shape_writer:
							json.dump(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE ,    ensure_ascii=_SCREAMING_SNAKE_CASE)
				with open(self.word_pronunciation_file ,    "w" ,    encoding="utf-8") as word_pronunciation_writer:
							json.dump(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE ,    ensure_ascii=_SCREAMING_SNAKE_CASE)

	def 							_SCREAMING_SNAKE_CASE     (  self:	Dict)       -> str:

				"""simple docstring"""

				__lowerCAmelCase   :						Dict				= self.tokenizer_class(self.vocab_file ,    self.word_shape_file ,    self.word_pronunciation_file)

				__lowerCAmelCase   :						List[Any]				= tokenizer.tokenize("你好[SEP]你是谁")
				self.assertListEqual(_SCREAMING_SNAKE_CASE ,    ["你", "好", "[SEP]", "你", "是", "谁"])
				self.assertListEqual(tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE) ,    [5, 6, 2, 5, 7, 8])
				self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(_SCREAMING_SNAKE_CASE) ,    [5, 6, 2, 5, 7, 8])
				self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(_SCREAMING_SNAKE_CASE) ,    [5, 6, 2, 5, 7, 8])

	def 							_SCREAMING_SNAKE_CASE     (  self:	Dict)       -> int:

				"""simple docstring"""

				__lowerCAmelCase   :						List[str]				= RoCBertBasicTokenizer()

				self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz") ,    ["ah", "\u535A", "\u63A8", "zz"])

	def 							_SCREAMING_SNAKE_CASE     (  self:	Optional[Any])       -> Tuple:

				"""simple docstring"""

				__lowerCAmelCase   :						int				= RoCBertBasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  ") ,    ["hello", "!", "how", "are", "you", "?"])
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo") ,    ["hello"])

	def 							_SCREAMING_SNAKE_CASE     (  self:	str)       -> List[str]:

				"""simple docstring"""

				__lowerCAmelCase   :						Optional[int]				= RoCBertBasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE ,    strip_accents=_SCREAMING_SNAKE_CASE)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ") ,    ["hällo", "!", "how", "are", "you", "?"])
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo") ,    ["h\u00E9llo"])

	def 							_SCREAMING_SNAKE_CASE     (  self:	Tuple)       -> Dict:

				"""simple docstring"""

				__lowerCAmelCase   :						Tuple				= RoCBertBasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE ,    strip_accents=_SCREAMING_SNAKE_CASE)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ") ,    ["hallo", "!", "how", "are", "you", "?"])
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo") ,    ["hello"])

	def 							_SCREAMING_SNAKE_CASE     (  self:	Dict)       -> Dict:

				"""simple docstring"""

				__lowerCAmelCase   :						Dict				= RoCBertBasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ") ,    ["hallo", "!", "how", "are", "you", "?"])
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo") ,    ["hello"])

	def 							_SCREAMING_SNAKE_CASE     (  self:	List[str])       -> List[str]:

				"""simple docstring"""

				__lowerCAmelCase   :						Any				= RoCBertBasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  ") ,    ["HeLLo", "!", "how", "Are", "yoU", "?"])

	def 							_SCREAMING_SNAKE_CASE     (  self:	Optional[Any])       -> Tuple:

				"""simple docstring"""

				__lowerCAmelCase   :						str				= RoCBertBasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE ,    strip_accents=_SCREAMING_SNAKE_CASE)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ") ,    ["HäLLo", "!", "how", "Are", "yoU", "?"])

	def 							_SCREAMING_SNAKE_CASE     (  self:	Any)       -> Dict:

				"""simple docstring"""

				__lowerCAmelCase   :						str				= RoCBertBasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE ,    strip_accents=_SCREAMING_SNAKE_CASE)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ") ,    ["HaLLo", "!", "how", "Are", "yoU", "?"])

	def 							_SCREAMING_SNAKE_CASE     (  self:	List[str])       -> str:

				"""simple docstring"""

				__lowerCAmelCase   :						Optional[Any]				= RoCBertBasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE ,    never_split=["[UNK]"])

				self.assertListEqual(
				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]") ,    ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"])

	def 							_SCREAMING_SNAKE_CASE     (  self:	List[str])       -> Optional[int]:

				"""simple docstring"""

				__lowerCAmelCase   :						str				= ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

				__lowerCAmelCase   :						Union[str, Any]				= {}
				for i, token in enumerate(_SCREAMING_SNAKE_CASE):
							__lowerCAmelCase   :						Optional[int]				= i
				__lowerCAmelCase   :						Any				= RoCBertWordpieceTokenizer(vocab=_SCREAMING_SNAKE_CASE ,    unk_token="[UNK]")

				self.assertListEqual(tokenizer.tokenize("") ,    [])

				self.assertListEqual(tokenizer.tokenize("unwanted running") ,    ["un", "##want", "##ed", "runn", "##ing"])

				self.assertListEqual(tokenizer.tokenize("unwantedX running") ,    ["[UNK]", "runn", "##ing"])

	def 							_SCREAMING_SNAKE_CASE     (  self:	str)       -> int:

				"""simple docstring"""

				self.assertTrue(_is_whitespace(" "))
				self.assertTrue(_is_whitespace("\t"))
				self.assertTrue(_is_whitespace("\r"))
				self.assertTrue(_is_whitespace("\n"))
				self.assertTrue(_is_whitespace("\u00A0"))

				self.assertFalse(_is_whitespace("A"))
				self.assertFalse(_is_whitespace("-"))

	def 							_SCREAMING_SNAKE_CASE     (  self:	List[Any])       -> Dict:

				"""simple docstring"""

				self.assertTrue(_is_control("\u0005"))

				self.assertFalse(_is_control("A"))
				self.assertFalse(_is_control(" "))
				self.assertFalse(_is_control("\t"))
				self.assertFalse(_is_control("\r"))

	def 							_SCREAMING_SNAKE_CASE     (  self:	List[Any])       -> str:

				"""simple docstring"""

				self.assertTrue(_is_punctuation("-"))
				self.assertTrue(_is_punctuation("$"))
				self.assertTrue(_is_punctuation("`"))
				self.assertTrue(_is_punctuation("."))

				self.assertFalse(_is_punctuation("A"))
				self.assertFalse(_is_punctuation(" "))

	def 							_SCREAMING_SNAKE_CASE     (  self:	Dict)       -> Tuple:

				"""simple docstring"""

				__lowerCAmelCase   :						Dict				= self.get_tokenizer()

				# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
				self.assertListEqual([tokenizer.tokenize(_SCREAMING_SNAKE_CASE) for t in ["Test", "\xad", "test"]] ,    [["[UNK]"], [], ["[UNK]"]])

				if self.test_rust_tokenizer:
							__lowerCAmelCase   :						List[str]				= self.get_rust_tokenizer()
							self.assertListEqual(
							    [rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE) for t in ["Test", "\xad", "test"]] ,    [["[UNK]"], [], ["[UNK]"]])

	def 							_SCREAMING_SNAKE_CASE     (  self:	int)       -> List[str]:

				"""simple docstring"""

				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
										__lowerCAmelCase   :						List[Any]				= self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE ,    **_SCREAMING_SNAKE_CASE)

										__lowerCAmelCase   :						Optional[Any]				= F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
										__lowerCAmelCase   :						List[Any]				= tokenizer_r.encode_plus(
										    _SCREAMING_SNAKE_CASE ,    return_attention_mask=_SCREAMING_SNAKE_CASE ,    return_token_type_ids=_SCREAMING_SNAKE_CASE ,    return_offsets_mapping=_SCREAMING_SNAKE_CASE ,    add_special_tokens=_SCREAMING_SNAKE_CASE ,    )

										__lowerCAmelCase   :						Optional[Any]				= tokenizer_r.do_lower_case if hasattr(_SCREAMING_SNAKE_CASE ,    "do_lower_case") else False
										__lowerCAmelCase   :						int				= (
										    [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), "A"),
										        ((1, 2), ","),
										        ((3, 5), "na"),
										        ((5, 6), "##ï"),
										        ((6, 8), "##ve"),
										        ((9, 15), tokenizer_r.mask_token),
										        ((16, 21), "Allen"),
										        ((21, 23), "##NL"),
										        ((23, 24), "##P"),
										        ((25, 33), "sentence"),
										        ((33, 34), "."),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										    if not do_lower_case
										    else [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), "a"),
										        ((1, 2), ","),
										        ((3, 8), "naive"),
										        ((9, 15), tokenizer_r.mask_token),
										        ((16, 21), "allen"),
										        ((21, 23), "##nl"),
										        ((23, 24), "##p"),
										        ((25, 33), "sentence"),
										        ((33, 34), "."),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										)

										self.assertEqual(
										    [e[1] for e in expected_results] ,    tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]))
										self.assertEqual([e[0] for e in expected_results] ,    tokens["offset_mapping"])

	def 							_SCREAMING_SNAKE_CASE     (  self:	List[str])       -> Optional[Any]:

				"""simple docstring"""

				__lowerCAmelCase   :						List[str]				= ["的", "人", "有"]
				__lowerCAmelCase   :						Tuple				= "".join(_SCREAMING_SNAKE_CASE)
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
										__lowerCAmelCase   :						Any				= True
										__lowerCAmelCase   :						Dict				= self.tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE ,    **_SCREAMING_SNAKE_CASE)
										__lowerCAmelCase   :						int				= self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE ,    **_SCREAMING_SNAKE_CASE)

										__lowerCAmelCase   :						Dict				= tokenizer_p.encode(_SCREAMING_SNAKE_CASE ,    add_special_tokens=_SCREAMING_SNAKE_CASE)
										__lowerCAmelCase   :						List[Any]				= tokenizer_r.encode(_SCREAMING_SNAKE_CASE ,    add_special_tokens=_SCREAMING_SNAKE_CASE)

										__lowerCAmelCase   :						Optional[Any]				= tokenizer_r.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE)
										__lowerCAmelCase   :						Optional[int]				= tokenizer_p.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE)

										# it is expected that each Chinese character is not preceded by "##"
										self.assertListEqual(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE)
										self.assertListEqual(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE)

										__lowerCAmelCase   :						int				= False
										__lowerCAmelCase   :						List[Any]				= self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE ,    **_SCREAMING_SNAKE_CASE)
										__lowerCAmelCase   :						List[Any]				= self.tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE ,    **_SCREAMING_SNAKE_CASE)

										__lowerCAmelCase   :						Any				= tokenizer_r.encode(_SCREAMING_SNAKE_CASE ,    add_special_tokens=_SCREAMING_SNAKE_CASE)
										__lowerCAmelCase   :						List[Any]				= tokenizer_p.encode(_SCREAMING_SNAKE_CASE ,    add_special_tokens=_SCREAMING_SNAKE_CASE)

										__lowerCAmelCase   :						int				= tokenizer_r.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE)
										__lowerCAmelCase   :						Dict				= tokenizer_p.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE)

										# it is expected that only the first Chinese character is not preceded by "##".
										__lowerCAmelCase   :						Union[str, Any]				= [
										    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(_SCREAMING_SNAKE_CASE)
										]
										self.assertListEqual(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE)
										self.assertListEqual(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE)

	@slow
	def 							_SCREAMING_SNAKE_CASE     (  self:	Dict)       -> str:

				"""simple docstring"""

				__lowerCAmelCase   :						Tuple				= self.tokenizer_class(self.vocab_file ,    self.word_shape_file ,    self.word_pronunciation_file)

				__lowerCAmelCase   :						List[Any]				= tokenizer.encode("你好" ,    add_special_tokens=_SCREAMING_SNAKE_CASE)
				__lowerCAmelCase   :						Union[str, Any]				= tokenizer.encode("你是谁" ,    add_special_tokens=_SCREAMING_SNAKE_CASE)

				__lowerCAmelCase   :						List[str]				= tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE)
				__lowerCAmelCase   :						str				= tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE)

				assert encoded_sentence == [1] + text + [2]
				assert encoded_pair == [1] + text + [2] + text_a + [2]







	def 							_SCREAMING_SNAKE_CASE     (  self:	str)       -> Optional[int]:

				"""simple docstring"""

				__lowerCAmelCase   :						Union[str, Any]				= self.get_tokenizers(do_lower_case=_SCREAMING_SNAKE_CASE)
				for tokenizer in tokenizers:
							with self.subTest(F"""{tokenizer.__class__.__name__}"""):
										__lowerCAmelCase   :						Dict				= "你好，你是谁"
										__lowerCAmelCase   :						Tuple				= tokenizer.tokenize(_SCREAMING_SNAKE_CASE)
										__lowerCAmelCase   :						Optional[Any]				= tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE)
										__lowerCAmelCase   :						List[str]				= tokenizer.convert_tokens_to_shape_ids(_SCREAMING_SNAKE_CASE)
										__lowerCAmelCase   :						str				= tokenizer.convert_tokens_to_pronunciation_ids(_SCREAMING_SNAKE_CASE)
										__lowerCAmelCase   :						int				= tokenizer.prepare_for_model(
										    _SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE ,    add_special_tokens=_SCREAMING_SNAKE_CASE)

										__lowerCAmelCase   :						Union[str, Any]				= tokenizer.encode_plus(_SCREAMING_SNAKE_CASE ,    add_special_tokens=_SCREAMING_SNAKE_CASE)

										self.assertEqual(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE)