





import argparse
import json
import os

import numpy as np
import PIL
import requests
import tensorflow.keras.applications.efficientnet as efficientnet
import torch
from huggingface_hub import hf_hub_download
from PIL import Image
from tensorflow.keras.preprocessing import image

from transformers import (
    EfficientNetConfig,
    EfficientNetForImageClassification,
    EfficientNetImageProcessor,
)
from transformers.utils import logging


logging.set_verbosity_info()
snake_case_     :       List[str]						=  logging.get_logger(__name__)

snake_case_     :       List[Any]						=  {
    "b0": efficientnet.EfficientNetBa,
    "b1": efficientnet.EfficientNetBa,
    "b2": efficientnet.EfficientNetBa,
    "b3": efficientnet.EfficientNetBa,
    "b4": efficientnet.EfficientNetBa,
    "b5": efficientnet.EfficientNetBa,
    "b6": efficientnet.EfficientNetBa,
    "b7": efficientnet.EfficientNetBa,
}

snake_case_     :       Optional[Any]						=  {
    "b0": {
        "hidden_dim": 12_80,
        "width_coef": 1.0,
        "depth_coef": 1.0,
        "image_size": 2_24,
        "dropout_rate": 0.2,
        "dw_padding": [],
    },
    "b1": {
        "hidden_dim": 12_80,
        "width_coef": 1.0,
        "depth_coef": 1.1,
        "image_size": 2_40,
        "dropout_rate": 0.2,
        "dw_padding": [16],
    },
    "b2": {
        "hidden_dim": 14_08,
        "width_coef": 1.1,
        "depth_coef": 1.2,
        "image_size": 2_60,
        "dropout_rate": 0.3,
        "dw_padding": [5, 8, 16],
    },
    "b3": {
        "hidden_dim": 15_36,
        "width_coef": 1.2,
        "depth_coef": 1.4,
        "image_size": 3_00,
        "dropout_rate": 0.3,
        "dw_padding": [5, 18],
    },
    "b4": {
        "hidden_dim": 17_92,
        "width_coef": 1.4,
        "depth_coef": 1.8,
        "image_size": 3_80,
        "dropout_rate": 0.4,
        "dw_padding": [6],
    },
    "b5": {
        "hidden_dim": 20_48,
        "width_coef": 1.6,
        "depth_coef": 2.2,
        "image_size": 4_56,
        "dropout_rate": 0.4,
        "dw_padding": [13, 27],
    },
    "b6": {
        "hidden_dim": 23_04,
        "width_coef": 1.8,
        "depth_coef": 2.6,
        "image_size": 5_28,
        "dropout_rate": 0.5,
        "dw_padding": [31],
    },
    "b7": {
        "hidden_dim": 25_60,
        "width_coef": 2.0,
        "depth_coef": 3.1,
        "image_size": 6_00,
        "dropout_rate": 0.5,
        "dw_padding": [18],
    },
}





def        lowerCamelCase_      (  SCREAMING_SNAKE_CASE__      :			str   ) ->	Dict:
						UpperCAmelCase_			:      int     						=  EfficientNetConfig()
						UpperCAmelCase_			:      str     						=  CONFIG_MAP[model_name]['''hidden_dim''']
						UpperCAmelCase_			:      Any     						=  CONFIG_MAP[model_name]['''width_coef''']
						UpperCAmelCase_			:      Tuple     						=  CONFIG_MAP[model_name]['''depth_coef''']
						UpperCAmelCase_			:      List[Any]     						=  CONFIG_MAP[model_name]['''image_size''']
						UpperCAmelCase_			:      int     						=  CONFIG_MAP[model_name]['''dropout_rate''']
						UpperCAmelCase_			:      Optional[int]     						=  CONFIG_MAP[model_name]['''dw_padding''']

						UpperCAmelCase_			:      int     						=  '''huggingface/label-files'''
						UpperCAmelCase_			:      Dict     						=  '''imagenet-1k-id2label.json'''
						UpperCAmelCase_			:      Union[str, Any]     						=  1000
						UpperCAmelCase_			:      Tuple     						=  json.load(open(hf_hub_download(SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__,			repo_type='''dataset'''   ),			'''r'''   )   )
						UpperCAmelCase_			:      str     						=  {int(SCREAMING_SNAKE_CASE__   ): v for k, v in idalabel.items()}

						UpperCAmelCase_			:      Any     						=  idalabel
						UpperCAmelCase_			:      List[str]     						=  {v: k for k, v in idalabel.items()}
						return config





def        lowerCamelCase_      (  ) ->	Optional[Any]:
						UpperCAmelCase_			:      Optional[int]     						=  '''http://images.cocodataset.org/val2017/000000039769.jpg'''
						UpperCAmelCase_			:      List[Any]     						=  Image.open(requests.get(SCREAMING_SNAKE_CASE__,			stream=SCREAMING_SNAKE_CASE__   ).raw   )
						return im





def        lowerCamelCase_      (  SCREAMING_SNAKE_CASE__      :			Optional[int]   ) ->	List[str]:
						UpperCAmelCase_			:      Optional[Any]     						=  CONFIG_MAP[model_name]['''image_size''']
						UpperCAmelCase_			:      str     						=  EfficientNetImageProcessor(
						    size={'''height''': size, '''width''': size},			image_mean=[0.4_85, 0.4_56, 0.4_06],			image_std=[0.47_85_39_44, 0.4_73_28_64, 0.47_43_41_63],			do_center_crop=SCREAMING_SNAKE_CASE__,			)
						return preprocessor





def        lowerCamelCase_      (  SCREAMING_SNAKE_CASE__      :			int   ) ->	Tuple:
						UpperCAmelCase_			:      Dict     						=  [v.split('''_'''   )[0].split('''block'''   )[1] for v in original_param_names if v.startswith('''block'''   )]
						UpperCAmelCase_			:      Optional[int]     						=  sorted(set(SCREAMING_SNAKE_CASE__   )   )
						UpperCAmelCase_			:      Union[str, Any]     						=  len(SCREAMING_SNAKE_CASE__   )
						UpperCAmelCase_			:      str     						=  {b: str(SCREAMING_SNAKE_CASE__   ) for b, i in zip(SCREAMING_SNAKE_CASE__,			range(SCREAMING_SNAKE_CASE__   )   )}

						UpperCAmelCase_			:      Optional[int]     						=  []
						rename_keys.append(('''stem_conv/kernel:0''', '''embeddings.convolution.weight''')   )
						rename_keys.append(('''stem_bn/gamma:0''', '''embeddings.batchnorm.weight''')   )
						rename_keys.append(('''stem_bn/beta:0''', '''embeddings.batchnorm.bias''')   )
						rename_keys.append(('''stem_bn/moving_mean:0''', '''embeddings.batchnorm.running_mean''')   )
						rename_keys.append(('''stem_bn/moving_variance:0''', '''embeddings.batchnorm.running_var''')   )

						for b in block_names:
												UpperCAmelCase_			:      Union[str, Any]     						=  block_name_mapping[b]
												rename_keys.append((F"""block{b}_expand_conv/kernel:0""", F"""encoder.blocks.{hf_b}.expansion.expand_conv.weight""")   )
												rename_keys.append((F"""block{b}_expand_bn/gamma:0""", F"""encoder.blocks.{hf_b}.expansion.expand_bn.weight""")   )
												rename_keys.append((F"""block{b}_expand_bn/beta:0""", F"""encoder.blocks.{hf_b}.expansion.expand_bn.bias""")   )
												rename_keys.append(
												    (F"""block{b}_expand_bn/moving_mean:0""", F"""encoder.blocks.{hf_b}.expansion.expand_bn.running_mean""")   )
												rename_keys.append(
												    (F"""block{b}_expand_bn/moving_variance:0""", F"""encoder.blocks.{hf_b}.expansion.expand_bn.running_var""")   )
												rename_keys.append(
												    (F"""block{b}_dwconv/depthwise_kernel:0""", F"""encoder.blocks.{hf_b}.depthwise_conv.depthwise_conv.weight""")   )
												rename_keys.append((F"""block{b}_bn/gamma:0""", F"""encoder.blocks.{hf_b}.depthwise_conv.depthwise_norm.weight""")   )
												rename_keys.append((F"""block{b}_bn/beta:0""", F"""encoder.blocks.{hf_b}.depthwise_conv.depthwise_norm.bias""")   )
												rename_keys.append(
												    (F"""block{b}_bn/moving_mean:0""", F"""encoder.blocks.{hf_b}.depthwise_conv.depthwise_norm.running_mean""")   )
												rename_keys.append(
												    (F"""block{b}_bn/moving_variance:0""", F"""encoder.blocks.{hf_b}.depthwise_conv.depthwise_norm.running_var""")   )

												rename_keys.append((F"""block{b}_se_reduce/kernel:0""", F"""encoder.blocks.{hf_b}.squeeze_excite.reduce.weight""")   )
												rename_keys.append((F"""block{b}_se_reduce/bias:0""", F"""encoder.blocks.{hf_b}.squeeze_excite.reduce.bias""")   )
												rename_keys.append((F"""block{b}_se_expand/kernel:0""", F"""encoder.blocks.{hf_b}.squeeze_excite.expand.weight""")   )
												rename_keys.append((F"""block{b}_se_expand/bias:0""", F"""encoder.blocks.{hf_b}.squeeze_excite.expand.bias""")   )
												rename_keys.append(
												    (F"""block{b}_project_conv/kernel:0""", F"""encoder.blocks.{hf_b}.projection.project_conv.weight""")   )
												rename_keys.append((F"""block{b}_project_bn/gamma:0""", F"""encoder.blocks.{hf_b}.projection.project_bn.weight""")   )
												rename_keys.append((F"""block{b}_project_bn/beta:0""", F"""encoder.blocks.{hf_b}.projection.project_bn.bias""")   )
												rename_keys.append(
												    (F"""block{b}_project_bn/moving_mean:0""", F"""encoder.blocks.{hf_b}.projection.project_bn.running_mean""")   )
												rename_keys.append(
												    (F"""block{b}_project_bn/moving_variance:0""", F"""encoder.blocks.{hf_b}.projection.project_bn.running_var""")   )

						rename_keys.append(('''top_conv/kernel:0''', '''encoder.top_conv.weight''')   )
						rename_keys.append(('''top_bn/gamma:0''', '''encoder.top_bn.weight''')   )
						rename_keys.append(('''top_bn/beta:0''', '''encoder.top_bn.bias''')   )
						rename_keys.append(('''top_bn/moving_mean:0''', '''encoder.top_bn.running_mean''')   )
						rename_keys.append(('''top_bn/moving_variance:0''', '''encoder.top_bn.running_var''')   )

						UpperCAmelCase_			:      Optional[int]     						=  {}
						for item in rename_keys:
												if item[0] in original_param_names:
																		UpperCAmelCase_			:      Optional[int]     						=  '''efficientnet.''' + item[1]

						UpperCAmelCase_			:      Any     						=  '''classifier.weight'''
						UpperCAmelCase_			:      Dict     						=  '''classifier.bias'''
						return key_mapping





def        lowerCamelCase_      (  SCREAMING_SNAKE_CASE__      :			Optional[Any],			SCREAMING_SNAKE_CASE__      :			int,			SCREAMING_SNAKE_CASE__      :			Optional[int]   ) ->	Optional[int]:
						for key, value in tf_params.items():
												if "normalization" in key:
																		continue

												UpperCAmelCase_			:      List[str]     						=  key_mapping[key]
												if "_conv" in key and "kernel" in key:
																		UpperCAmelCase_			:      Union[str, Any]     						=  torch.from_numpy(SCREAMING_SNAKE_CASE__   ).permute(3,			2,			0,			1   )
												elif "depthwise_kernel" in key:
																		UpperCAmelCase_			:      Tuple     						=  torch.from_numpy(SCREAMING_SNAKE_CASE__   ).permute(2,			3,			0,			1   )
												elif "kernel" in key:
																		UpperCAmelCase_			:      str     						=  torch.from_numpy(np.transpose(SCREAMING_SNAKE_CASE__   )   )
												else:
																		UpperCAmelCase_			:      Tuple     						=  torch.from_numpy(SCREAMING_SNAKE_CASE__   )

												# Replace HF parameters with original TF model parameters
												assert hf_params[hf_key].shape == new_hf_value.shape
												hf_params[hf_key].copy_(SCREAMING_SNAKE_CASE__   )





@torch.no_grad()
def        lowerCamelCase_      (  SCREAMING_SNAKE_CASE__      :			List[Any],			SCREAMING_SNAKE_CASE__      :			Dict,			SCREAMING_SNAKE_CASE__      :			Any,			SCREAMING_SNAKE_CASE__      :			Optional[Any]   ) ->	Any:
						UpperCAmelCase_			:      Optional[Any]     						=  model_classes[model_name](
						    include_top=SCREAMING_SNAKE_CASE__,			weights='''imagenet''',			input_tensor=SCREAMING_SNAKE_CASE__,			input_shape=SCREAMING_SNAKE_CASE__,			pooling=SCREAMING_SNAKE_CASE__,			classes=1000,			classifier_activation='''softmax''',			)

						UpperCAmelCase_			:      int     						=  original_model.trainable_variables
						UpperCAmelCase_			:      List[Any]     						=  original_model.non_trainable_variables
						UpperCAmelCase_			:      Tuple     						=  {param.name: param.numpy() for param in tf_params}
						for param in tf_non_train_params:
												UpperCAmelCase_			:      Dict     						=  param.numpy()
						UpperCAmelCase_			:      Dict     						=  list(tf_params.keys()   )

						# Load HuggingFace model
						UpperCAmelCase_			:      str     						=  get_efficientnet_config(SCREAMING_SNAKE_CASE__   )
						UpperCAmelCase_			:      Optional[int]     						=  EfficientNetForImageClassification(SCREAMING_SNAKE_CASE__   ).eval()
						UpperCAmelCase_			:      Optional[int]     						=  hf_model.state_dict()

						# Create src-to-dst parameter name mapping dictionary
						print('''Converting parameters...'''   )
						UpperCAmelCase_			:      List[Any]     						=  rename_keys(SCREAMING_SNAKE_CASE__   )
						replace_params(SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__   )

						# Initialize preprocessor and preprocess input image
						UpperCAmelCase_			:      Optional[Any]     						=  convert_image_processor(SCREAMING_SNAKE_CASE__   )
						UpperCAmelCase_			:      Union[str, Any]     						=  preprocessor(images=prepare_img(),			return_tensors='''pt'''   )

						# HF model inference
						hf_model.eval()
						with torch.no_grad():
												UpperCAmelCase_			:      str     						=  hf_model(**SCREAMING_SNAKE_CASE__   )
						UpperCAmelCase_			:      Dict     						=  outputs.logits.detach().numpy()

						# Original model inference
						UpperCAmelCase_			:      List[str]     						=  False
						UpperCAmelCase_			:      Union[str, Any]     						=  CONFIG_MAP[model_name]['''image_size''']
						UpperCAmelCase_			:      str     						=  prepare_img().resize((image_size, image_size),			resample=PIL.Image.NEAREST   )
						UpperCAmelCase_			:      Union[str, Any]     						=  image.img_to_array(SCREAMING_SNAKE_CASE__   )
						UpperCAmelCase_			:      Optional[Any]     						=  np.expand_dims(SCREAMING_SNAKE_CASE__,			axis=0   )
						UpperCAmelCase_			:      Optional[int]     						=  original_model.predict(SCREAMING_SNAKE_CASE__   )

						# Check whether original and HF model outputs match  -> np.allclose
						assert np.allclose(SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__,			atol=1E-3   ), "The predicted logits are not the same."
						print('''Model outputs match!'''   )

						if save_model:
												# Create folder to save model
												if not os.path.isdir(SCREAMING_SNAKE_CASE__   ):
																		os.mkdir(SCREAMING_SNAKE_CASE__   )
												# Save converted model and image processor
												hf_model.save_pretrained(SCREAMING_SNAKE_CASE__   )
												preprocessor.save_pretrained(SCREAMING_SNAKE_CASE__   )

						if push_to_hub:
												# Push model and image processor to hub
												print(F"""Pushing converted {model_name} to the hub..."""   )
												UpperCAmelCase_			:      List[str]     						=  F"""efficientnet-{model_name}"""
												preprocessor.push_to_hub(SCREAMING_SNAKE_CASE__   )
												hf_model.push_to_hub(SCREAMING_SNAKE_CASE__   )


if __name__ == "__main__":
					snake_case_     :       List[str]						=  argparse.ArgumentParser()
					# Required parameters
					parser.add_argument(
					    "--model_name",
					    default="b0",
					    type=str,
					    help="Version name of the EfficientNet model you want to convert, select from [b0, b1, b2, b3, b4, b5, b6, b7].",
					)
					parser.add_argument(
					    "--pytorch_dump_folder_path",
					    default="hf_model",
					    type=str,
					    help="Path to the output PyTorch model directory.",
					)
					parser.add_argument("--save_model", action="store_true", help="Save model to local")
					parser.add_argument("--push_to_hub", action="store_true", help="Push model and image processor to the hub")

					snake_case_     :       List[Any]						=  parser.parse_args()
					convert_efficientnet_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.save_model, args.push_to_hub)












'''simple docstring'''







import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


snake_case_     :       str						=  logging.get_logger(__name__)

snake_case_     :       int						=  "▁"

snake_case_     :       str						=  {"vocab_file": "vocab.txt", "sentencepiece_model_ckpt": "sentencepiece.bpe.model"}

snake_case_     :       int						=  {
    "sentencepiece_model_file": "sentencepiece.bpe.model",
    "vocab_file": "vocab.txt",
}

snake_case_     :       Optional[Any]						=  {
    "vocab_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
    },
    "sentencepiece_model_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
    },
}

snake_case_     :       Dict						=  {
    "ernie-m-base": 5_14,
    "ernie-m-large": 5_14,
}

snake_case_     :       Any						=  {
    "ernie-m-base": {"do_lower_case": False},
    "ernie-m-large": {"do_lower_case": False},
}


class 					__a      (lowerCamelCase     ):
							__a					:   List[str]									=       ["input_ids"]

							__a					:   Union[str, Any]									=       VOCAB_FILES_NAMES
							__a					:   Tuple									=       PRETRAINED_INIT_CONFIGURATION
							__a					:   Dict									=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__a					:   Optional[int]									=       PRETRAINED_VOCAB_FILES_MAP
							__a					:   Union[str, Any]									=       RESOURCE_FILES_NAMES

							def __init__(							self							:     Union[str, Any]       ,     __magic_name__							:     Dict       ,     __magic_name__							:     int=None       ,     __magic_name__							:     str=False       ,     __magic_name__							:     int="utf8"       ,     __magic_name__							:     Optional[int]="[UNK]"       ,     __magic_name__							:     Dict="[SEP]"       ,     __magic_name__							:     List[Any]="[PAD]"       ,     __magic_name__							:     str="[CLS]"       ,     __magic_name__							:     Optional[int]="[MASK]"       ,     __magic_name__							:     Optional[Dict[str, Any]] = None       ,     **__magic_name__							:     Union[str, Any]       ,     )					->       None:







													"""simple docstring"""
													# Mask token behave like a normal word, i.e. include the space before it and
													# is included in the raw text, there should be a match in a non-normalized sentence.

													UpperCAmelCase_			:      List[Any]     						=  {} if sp_model_kwargs is None else sp_model_kwargs
													super().__init__(
													    do_lower_case=__magic_name__       ,     unk_token=__magic_name__       ,     sep_token=__magic_name__       ,     pad_token=__magic_name__       ,     cls_token=__magic_name__       ,     mask_token=__magic_name__       ,     vocab_file=__magic_name__       ,     encoding=__magic_name__       ,     sp_model_kwargs=self.sp_model_kwargs       ,     **__magic_name__       ,     )
													UpperCAmelCase_			:      Optional[Any]     						=  do_lower_case
													UpperCAmelCase_			:      List[str]     						=  sentencepiece_model_ckpt
													UpperCAmelCase_			:      Tuple     						=  spm.SentencePieceProcessor(**self.sp_model_kwargs   )
													self.sp_model.Load(__magic_name__   )

													# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
													if vocab_file is not None:
																			UpperCAmelCase_			:      List[Any]     						=  self.load_vocab(filepath=__magic_name__   )
													else:
																			UpperCAmelCase_			:      str     						=  {self.sp_model.id_to_piece(__magic_name__   ): id for id in range(self.sp_model.get_piece_size()   )}
													UpperCAmelCase_			:      int     						=  {v: k for k, v in self.vocab.items()}

							def     UpperCAmelCase__	(							self							:     List[str]       ,     __magic_name__							:     Any   )					->       Any:







													"""simple docstring"""
													if text is None:
																			return None

													UpperCAmelCase_			:      str     						=  self.tokenize(__magic_name__   )
													UpperCAmelCase_						,       UpperCAmelCase_			:      str     						=  '''''', []

													for i, ch in enumerate(__magic_name__   ):
																			if ch in self.SP_CHAR_MAPPING:
																									UpperCAmelCase_			:      Optional[int]     						=  self.SP_CHAR_MAPPING.get(__magic_name__   )
																			else:
																									UpperCAmelCase_			:      Union[str, Any]     						=  unicodedata.normalize('''NFKC'''       ,     __magic_name__   )
																			if self.is_whitespace(__magic_name__   ):
																									continue
																			normalized_text += ch
																			char_mapping.extend([i] * len(__magic_name__   )   )

													UpperCAmelCase_						,       UpperCAmelCase_						,       UpperCAmelCase_			:      Union[str, Any]     						=  normalized_text, [], 0

													if self.do_lower_case:
																			UpperCAmelCase_			:      Optional[int]     						=  text.lower()

													for token in split_tokens:
																			if token[:1] == "▁":
																									UpperCAmelCase_			:      Tuple     						=  token[1:]
																			UpperCAmelCase_			:      int     						=  text[offset:].index(__magic_name__   ) + offset
																			UpperCAmelCase_			:      Optional[int]     						=  start + len(__magic_name__   )

																			token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)   )
																			UpperCAmelCase_			:      int     						=  end
													return token_mapping

							@property
							def     UpperCAmelCase__	(							self							:     Any   )					->       Any:







													"""simple docstring"""
													return len(self.vocab   )

							def     UpperCAmelCase__	(							self							:     List[Any]   )					->       int:







													"""simple docstring"""
													return dict(self.vocab       ,     **self.added_tokens_encoder   )

							def __getstate__(							self							:     str   )					->       Any:







													"""simple docstring"""
													UpperCAmelCase_			:      Optional[Any]     						=  self.__dict__.copy()
													UpperCAmelCase_			:      Optional[Any]     						=  None
													return state

							def __setstate__(							self							:     str       ,     __magic_name__							:     Any   )					->       Dict:







													"""simple docstring"""
													UpperCAmelCase_			:      Dict     						=  d

													# for backward compatibility
													if not hasattr(self       ,     '''sp_model_kwargs'''   ):
																			UpperCAmelCase_			:      int     						=  {}

													UpperCAmelCase_			:      List[Any]     						=  spm.SentencePieceProcessor(**self.sp_model_kwargs   )
													self.sp_model.Load(self.sentencepiece_model_ckpt   )

							def     UpperCAmelCase__	(							self							:     Optional[int]       ,     __magic_name__							:     Any   )					->       List[str]:







													"""simple docstring"""
													return "".join((self.SP_CHAR_MAPPING.get(__magic_name__       ,     __magic_name__   ) for c in text)   )

							def     UpperCAmelCase__	(							self							:     List[str]       ,     __magic_name__							:     Tuple       ,     __magic_name__							:     Any=False       ,     __magic_name__							:     List[str]=64       ,     __magic_name__							:     List[str]=0.1   )					->       List[str]:







													"""simple docstring"""

													if self.sp_model_kwargs.get('''enable_sampling'''   ) is True:
																			UpperCAmelCase_			:      Dict     						=  True
													if self.sp_model_kwargs.get('''alpha'''   ) is not None:
																			UpperCAmelCase_			:      Union[str, Any]     						=  self.sp_model_kwargs.get('''alpha'''   )
													if self.sp_model_kwargs.get('''nbest_size'''   ) is not None:
																			UpperCAmelCase_			:      Any     						=  self.sp_model_kwargs.get('''nbest_size'''   )

													if not enable_sampling:
																			UpperCAmelCase_			:      Dict     						=  self.sp_model.EncodeAsPieces(__magic_name__   )
													else:
																			UpperCAmelCase_			:      Dict     						=  self.sp_model.SampleEncodeAsPieces(__magic_name__       ,     __magic_name__       ,     __magic_name__   )
													UpperCAmelCase_			:      List[Any]     						=  []
													for pi, piece in enumerate(__magic_name__   ):
																			if piece == SPIECE_UNDERLINE:
																									if not pieces[pi + 1].startswith(__magic_name__   ) and pi != 0:
																															new_pieces.append(__magic_name__   )
																															continue
																									else:
																															continue
																			UpperCAmelCase_			:      List[str]     						=  0
																			for i, chunk in enumerate(__magic_name__   ):
																									if chunk == SPIECE_UNDERLINE:
																															continue
																									if self.is_ch_char(__magic_name__   ) or self.is_punct(__magic_name__   ):
																															if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																					new_pieces.append(piece[lst_i:i]   )
																															new_pieces.append(__magic_name__   )
																															UpperCAmelCase_			:      List[Any]     						=  i + 1
																									elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
																															if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																					new_pieces.append(piece[lst_i:i]   )
																															UpperCAmelCase_			:      List[str]     						=  i
																									elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
																															if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																					new_pieces.append(piece[lst_i:i]   )
																															UpperCAmelCase_			:      str     						=  i
																			if len(__magic_name__   ) > lst_i:
																									new_pieces.append(piece[lst_i:]   )
													return new_pieces

							def     UpperCAmelCase__	(							self							:     List[Any]       ,     __magic_name__							:     Optional[int]   )					->       List[str]:







													"""simple docstring"""
													UpperCAmelCase_			:      Optional[Any]     						=  ''''''.join(__magic_name__   ).replace(__magic_name__       ,     ''' '''   ).strip()
													return out_string

							def     UpperCAmelCase__	(							self							:     List[Any]       ,     __magic_name__							:     Union[str, Any]   )					->       List[str]:







													"""simple docstring"""
													UpperCAmelCase_			:      str     						=  self.convert_ids_to_tokens(__magic_name__   )
													UpperCAmelCase_			:      Optional[Any]     						=  ''''''.join(__magic_name__   ).replace(__magic_name__       ,     ''' '''   ).strip()
													return out_string

							def     UpperCAmelCase__	(							self							:     str       ,     __magic_name__							:     Optional[Any]   )					->       List[Any]:







													"""simple docstring"""
													return self.vocab.get(__magic_name__       ,     self.vocab.get(self.unk_token   )   )

							def     UpperCAmelCase__	(							self							:     Tuple       ,     __magic_name__							:     Union[str, Any]   )					->       Union[str, Any]:







													"""simple docstring"""
													return self.reverse_vocab.get(__magic_name__       ,     self.unk_token   )

							def     UpperCAmelCase__	(							self							:     Tuple       ,     __magic_name__							:     Any       ,     __magic_name__							:     Union[str, Any]=None   )					->       Any:







													"""simple docstring"""
													if token_ids_a is None:
																			return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
													UpperCAmelCase_			:      Union[str, Any]     						=  [self.cls_token_id]
													UpperCAmelCase_			:      List[Any]     						=  [self.sep_token_id]
													return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep

							def     UpperCAmelCase__	(							self							:     Any       ,     __magic_name__							:     Optional[Any]       ,     __magic_name__							:     List[str]=None   )					->       int:







													"""simple docstring"""
													if offset_mapping_a is None:
																			return [(0, 0)] + offset_mapping_a + [(0, 0)]

													return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]

							def     UpperCAmelCase__	(							self							:     Dict       ,     __magic_name__							:     Optional[Any]       ,     __magic_name__							:     List[str]=None       ,     __magic_name__							:     Optional[Any]=False   )					->       Optional[int]:







													"""simple docstring"""

													if already_has_special_tokens:
																			if token_ids_a is not None:
																									raise ValueError(
																									    '''You should not supply a second sequence if the provided sequence of '''
																									    '''ids is already formatted with special tokens for the model.'''   )
																			return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

													if token_ids_a is not None:
																			return [1] + ([0] * len(__magic_name__   )) + [1, 1] + ([0] * len(__magic_name__   )) + [1]
													return [1] + ([0] * len(__magic_name__   )) + [1]

							def     UpperCAmelCase__	(							self							:     List[str]       ,     __magic_name__							:     List[int]       ,     __magic_name__							:     Optional[List[int]] = None   )					->       List[int]:







													"""simple docstring"""
													# called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
													if token_ids_a is None:
																			# [CLS] X [SEP]
																			return (len(__magic_name__   ) + 2) * [0]

													# [CLS] A [SEP] [SEP] B [SEP]
													return [0] * (len(__magic_name__   ) + 1) + [1] * (len(__magic_name__   ) + 3)

							def     UpperCAmelCase__	(							self							:     Dict       ,     __magic_name__							:     str   )					->       Tuple:







													"""simple docstring"""
													if "\u4e00" <= char <= "\u9fff":
																			return True
													return False

							def     UpperCAmelCase__	(							self							:     int       ,     __magic_name__							:     Optional[int]   )					->       str:







													"""simple docstring"""
													if ("a" <= char <= "z") or ("A" <= char <= "Z"):
																			return True
													return False

							def     UpperCAmelCase__	(							self							:     int       ,     __magic_name__							:     Optional[Any]   )					->       Dict:







													"""simple docstring"""
													if char in ",;:.?!~，；：。？！《》【】":
																			return True
													return False

							def     UpperCAmelCase__	(							self							:     Tuple       ,     __magic_name__							:     Any   )					->       Union[str, Any]:







													"""simple docstring"""
													if char == " " or char == "\t" or char == "\n" or char == "\r":
																			return True
													if len(__magic_name__   ) == 1:
																			UpperCAmelCase_			:      Optional[Any]     						=  unicodedata.category(__magic_name__   )
																			if cat == "Zs":
																									return True
													return False

							def     UpperCAmelCase__	(							self							:     Union[str, Any]       ,     __magic_name__							:     Tuple   )					->       Any:







													"""simple docstring"""
													UpperCAmelCase_			:      Optional[Any]     						=  {}
													with io.open(__magic_name__       ,     '''r'''       ,     encoding='''utf-8'''   ) as f:
																			for index, line in enumerate(__magic_name__   ):
																									UpperCAmelCase_			:      List[Any]     						=  line.rstrip('''\n'''   )
																									UpperCAmelCase_			:      Dict     						=  int(__magic_name__   )

													return token_to_idx



							def     UpperCAmelCase__	(							self							:     Dict       ,     __magic_name__							:     str       ,     __magic_name__							:     Optional[str] = None   )					->       Tuple[str]:







													"""simple docstring"""
													UpperCAmelCase_			:      Union[str, Any]     						=  0
													if os.path.isdir(__magic_name__   ):
																			UpperCAmelCase_			:      Any     						=  os.path.join(
																			    __magic_name__       ,     (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']   )
													else:
																			UpperCAmelCase_			:      List[str]     						=  (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
													with open(__magic_name__       ,     '''w'''       ,     encoding='''utf-8'''   ) as writer:
																			for token, token_index in sorted(self.vocab.items()       ,     key=lambda __magic_name__   : kv[1]   ):
																									if index != token_index:
																															logger.warning(
																															    F"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
																															    ''' Please check that the vocabulary is not corrupted!'''   )
																															UpperCAmelCase_			:      Dict     						=  token_index
																									writer.write(token + '''\n'''   )
																									index += 1

													UpperCAmelCase_			:      Union[str, Any]     						=  os.path.join(__magic_name__       ,     '''sentencepiece.bpe.model'''   )
													with open(__magic_name__       ,     '''wb'''   ) as fi:
																			UpperCAmelCase_			:      Optional[int]     						=  self.sp_model.serialized_model_proto()
																			fi.write(__magic_name__   )

													return (vocab_file,)





