


import argparse
import json
from pathlib import Path

import requests
import timm
import torch
from huggingface_hub import hf_hub_download
from PIL import Image
from timm.data import resolve_data_config
from timm.data.transforms_factory import create_transform

from transformers import (
    BitConfig,
    ViTHybridConfig,
    ViTHybridForImageClassification,
    ViTHybridImageProcessor,
    ViTHybridModel,
)
from transformers.image_utils import PILImageResampling
from transformers.utils import logging


logging.set_verbosity_info()
a					  =     logging.get_logger(__name__)


def 			UpperCAmelCase_				(      UpperCAmelCase__						,							UpperCAmelCase__=False  ):
					lowercase_  							=			[]

					# fmt: off
					# stem:
					rename_keys.append(("""cls_token""", """vit.embeddings.cls_token""")  )
					rename_keys.append(("""pos_embed""", """vit.embeddings.position_embeddings""")  )

					rename_keys.append(("""patch_embed.proj.weight""", """vit.embeddings.patch_embeddings.projection.weight""")  )
					rename_keys.append(("""patch_embed.proj.bias""", """vit.embeddings.patch_embeddings.projection.bias""")  )

					# backbone
					rename_keys.append(("""patch_embed.backbone.stem.conv.weight""", """vit.embeddings.patch_embeddings.backbone.bit.embedder.convolution.weight""")  )
					rename_keys.append(("""patch_embed.backbone.stem.norm.weight""", """vit.embeddings.patch_embeddings.backbone.bit.embedder.norm.weight""")  )
					rename_keys.append(("""patch_embed.backbone.stem.norm.bias""", """vit.embeddings.patch_embeddings.backbone.bit.embedder.norm.bias""")  )

					for stage_idx in range(len(config.backbone_config.depths  )  ):
										for layer_idx in range(config.backbone_config.depths[stage_idx]  ):
															rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.conv1.weight''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.conv1.weight''')  )
															rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.norm1.weight''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.norm1.weight''')  )
															rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.norm1.bias''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.norm1.bias''')  )
															rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.conv2.weight''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.conv2.weight''')  )
															rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.norm2.weight''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.norm2.weight''')  )
															rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.norm2.bias''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.norm2.bias''')  )
															rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.conv3.weight''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.conv3.weight''')  )
															rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.norm3.weight''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.norm3.weight''')  )
															rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.norm3.bias''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.norm3.bias''')  )

										rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.0.downsample.conv.weight''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.0.downsample.conv.weight''')  )
										rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.0.downsample.norm.weight''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.0.downsample.norm.weight''')  )
										rename_keys.append((F'''patch_embed.backbone.stages.{stage_idx}.blocks.0.downsample.norm.bias''', F'''vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.0.downsample.norm.bias''')  )

					# transformer encoder
					for i in range(config.num_hidden_layers  ):
										# encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
										rename_keys.append((F'''blocks.{i}.norm1.weight''', F'''vit.encoder.layer.{i}.layernorm_before.weight''')  )
										rename_keys.append((F'''blocks.{i}.norm1.bias''', F'''vit.encoder.layer.{i}.layernorm_before.bias''')  )
										rename_keys.append((F'''blocks.{i}.attn.proj.weight''', F'''vit.encoder.layer.{i}.attention.output.dense.weight''')  )
										rename_keys.append((F'''blocks.{i}.attn.proj.bias''', F'''vit.encoder.layer.{i}.attention.output.dense.bias''')  )
										rename_keys.append((F'''blocks.{i}.norm2.weight''', F'''vit.encoder.layer.{i}.layernorm_after.weight''')  )
										rename_keys.append((F'''blocks.{i}.norm2.bias''', F'''vit.encoder.layer.{i}.layernorm_after.bias''')  )
										rename_keys.append((F'''blocks.{i}.mlp.fc1.weight''', F'''vit.encoder.layer.{i}.intermediate.dense.weight''')  )
										rename_keys.append((F'''blocks.{i}.mlp.fc1.bias''', F'''vit.encoder.layer.{i}.intermediate.dense.bias''')  )
										rename_keys.append((F'''blocks.{i}.mlp.fc2.weight''', F'''vit.encoder.layer.{i}.output.dense.weight''')  )
										rename_keys.append((F'''blocks.{i}.mlp.fc2.bias''', F'''vit.encoder.layer.{i}.output.dense.bias''')  )

					if base_model:
										# layernorm + pooler
										rename_keys.extend(
										    [
										        ("""norm.weight""", """layernorm.weight"""),
										        ("""norm.bias""", """layernorm.bias"""),
										        ("""pre_logits.fc.weight""", """pooler.dense.weight"""),
										        ("""pre_logits.fc.bias""", """pooler.dense.bias"""),
										    ]  )

										# if just the base model, we should remove "vit" from all keys that start with "vit"
										lowercase_  							=			[(pair[0], pair[1][4:]) if pair[1].startswith("""vit"""  ) else pair for pair in rename_keys]
					else:
										# layernorm + classification head
										rename_keys.extend(
										    [
										        ("""norm.weight""", """vit.layernorm.weight"""),
										        ("""norm.bias""", """vit.layernorm.bias"""),
										        ("""head.weight""", """classifier.weight"""),
										        ("""head.bias""", """classifier.bias"""),
										    ]  )
					# fmt: on

					return rename_keys


def 			UpperCAmelCase_				(      UpperCAmelCase__						,							UpperCAmelCase__						,							UpperCAmelCase__=False  ):
					for i in range(config.num_hidden_layers  ):
										if base_model:
															lowercase_  							=			""
										else:
															lowercase_  							=			"vit."
										# read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
										lowercase_  							=			state_dict.pop(F'''blocks.{i}.attn.qkv.weight'''  )
										lowercase_  							=			state_dict.pop(F'''blocks.{i}.attn.qkv.bias'''  )
										# next, add query, keys and values (in that order) to the state dict
										lowercase_  							=			in_proj_weight[
										    : config.hidden_size, :
										]
										lowercase_  							=			in_proj_bias[: config.hidden_size]
										lowercase_  							=			in_proj_weight[
										    config.hidden_size : config.hidden_size * 2, :
										]
										lowercase_  							=			in_proj_bias[
										    config.hidden_size : config.hidden_size * 2
										]
										lowercase_  							=			in_proj_weight[
										    -config.hidden_size :, :
										]
										lowercase_  							=			in_proj_bias[-config.hidden_size :]


def 			UpperCAmelCase_				(      UpperCAmelCase__  ):
					lowercase_  							=			["head.weight", "head.bias"]
					for k in ignore_keys:
										state_dict.pop(_lowerCAmelCase						,							_lowerCAmelCase  )


def 			UpperCAmelCase_				(      UpperCAmelCase__						,							UpperCAmelCase__						,							UpperCAmelCase__  ):
					lowercase_  							=			dct.pop(_lowerCAmelCase  )
					lowercase_  							=			val


def 			UpperCAmelCase_				(      ):
					lowercase_  							=			"http://images.cocodataset.org/val2017/000000039769.jpg"
					lowercase_  							=			Image.open(requests.get(_lowerCAmelCase						,							stream=_lowerCAmelCase  ).raw  )
					return im


@torch.no_grad()
def 			UpperCAmelCase_				(      UpperCAmelCase__						,							UpperCAmelCase__						,							UpperCAmelCase__=False  ):
					lowercase_  							=			BitConfig(
					    global_padding="""same"""						,							layer_type="""bottleneck"""						,							depths=(3, 4, 9)						,							out_features=["""stage3"""]						,							embedding_dynamic_padding=_lowerCAmelCase						,							)
					lowercase_  							=			ViTHybridConfig(backbone_config=_lowerCAmelCase						,							image_size=3_8_4						,							num_labels=1_0_0_0  )
					lowercase_  							=			False

					# load original model from timm
					lowercase_  							=			timm.create_model(_lowerCAmelCase						,							pretrained=_lowerCAmelCase  )
					timm_model.eval()

					# load state_dict of original model, remove and rename some keys
					lowercase_  							=			timm_model.state_dict()
					if base_model:
										remove_classification_head_(_lowerCAmelCase  )
					lowercase_  							=			create_rename_keys(_lowerCAmelCase						,							_lowerCAmelCase  )
					for src, dest in rename_keys:
										rename_key(_lowerCAmelCase						,							_lowerCAmelCase						,							_lowerCAmelCase  )
					read_in_q_k_v(_lowerCAmelCase						,							_lowerCAmelCase						,							_lowerCAmelCase  )

					lowercase_  							=			"huggingface/label-files"
					lowercase_  							=			"imagenet-1k-id2label.json"
					lowercase_  							=			json.load(open(hf_hub_download(_lowerCAmelCase						,							_lowerCAmelCase						,							repo_type="""dataset"""  )						,							"""r"""  )  )
					lowercase_  							=			{int(_lowerCAmelCase  ): v for k, v in idalabel.items()}
					lowercase_  							=			idalabel
					lowercase_  							=			{v: k for k, v in idalabel.items()}

					# load HuggingFace model
					if vit_name[-5:] == "in21k":
										lowercase_  							=			ViTHybridModel(_lowerCAmelCase  ).eval()
					else:
										lowercase_  							=			ViTHybridForImageClassification(_lowerCAmelCase  ).eval()
					model.load_state_dict(_lowerCAmelCase  )

					# create image processor
					lowercase_  							=			create_transform(**resolve_data_config({}						,							model=_lowerCAmelCase  )  )
					lowercase_  							=			transform.transforms

					lowercase_  							=			{
					    "bilinear": PILImageResampling.BILINEAR,
					    "bicubic": PILImageResampling.BICUBIC,
					    "nearest": PILImageResampling.NEAREST,
					}

					lowercase_  							=			ViTHybridImageProcessor(
					    do_resize=_lowerCAmelCase						,							size={"""shortest_edge""": timm_transforms[0].size}						,							resample=pillow_resamplings[timm_transforms[0].interpolation.value]						,							do_center_crop=_lowerCAmelCase						,							crop_size={"""height""": timm_transforms[1].size[0], """width""": timm_transforms[1].size[1]}						,							do_normalize=_lowerCAmelCase						,							image_mean=timm_transforms[-1].mean.tolist()						,							image_std=timm_transforms[-1].std.tolist()						,							)

					lowercase_  							=			prepare_img()
					lowercase_  							=			transform(_lowerCAmelCase  ).unsqueeze(0  )
					lowercase_  							=			processor(_lowerCAmelCase						,							return_tensors="""pt"""  ).pixel_values

					# verify pixel values
					assert torch.allclose(_lowerCAmelCase						,							_lowerCAmelCase  )

					# verify logits
					with torch.no_grad():
										lowercase_  							=			model(_lowerCAmelCase  )
										lowercase_  							=			outputs.logits

					print("""Predicted class:"""						,							logits.argmax(-1  ).item()  )
					if base_model:
										lowercase_  							=			timm_model.forward_features(_lowerCAmelCase  )
										assert timm_pooled_output.shape == outputs.pooler_output.shape
										assert torch.allclose(_lowerCAmelCase						,							outputs.pooler_output						,							atol=1e-3  )
					else:
										lowercase_  							=			timm_model(_lowerCAmelCase  )
										assert timm_logits.shape == outputs.logits.shape
										assert torch.allclose(_lowerCAmelCase						,							outputs.logits						,							atol=1e-3  )
					print("""Looks ok!"""  )

					if pytorch_dump_folder_path is not None:
										Path(_lowerCAmelCase  ).mkdir(exist_ok=_lowerCAmelCase  )
										print(F'''Saving model {vit_name} to {pytorch_dump_folder_path}'''  )
										model.save_pretrained(_lowerCAmelCase  )
										print(F'''Saving processor to {pytorch_dump_folder_path}'''  )
										processor.save_pretrained(_lowerCAmelCase  )

					if push_to_hub:
										print(F'''Pushing model and processor to the hub {vit_name}'''  )
										model.push_to_hub(F'''ybelkada/{vit_name}'''  )
										processor.push_to_hub(F'''ybelkada/{vit_name}'''  )


if __name__ == "__main__":
							a					  =     argparse.ArgumentParser()
							# Required parameters
							parser.add_argument(
							    '--vit_name',
							    default='vit_base_r50_s16_384',
							    type=str,
							    help='Name of the hybrid ViT timm model you\'d like to convert.',
							)
							parser.add_argument(
							    '--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model directory.'
							)
							parser.add_argument(
							    '--push_to_hub', action='store_true', help='Whether to upload the model to the HuggingFace hub.'
							)

							a					  =     parser.parse_args()
							convert_vit_checkpoint(args.vit_name, args.pytorch_dump_folder_path, args.push_to_hub)






from dataclasses import asdict, dataclass
from typing import Optional

from ...configuration_utils import PretrainedConfig
from ...utils import logging


a					  =     logging.get_logger(__name__)

# TODO Update this
a					  =     {
    'facebook/esm-1b': 'https://huggingface.co/facebook/esm-1b/resolve/main/config.json',
    # See all ESM models at https://huggingface.co/models?filter=esm
}







class        UpperCamelCase__   (	__magic_name__   ):
					__SCREAMING_SNAKE_CASE       :  Tuple						=   'esm'

					def __init__(     self       :   Optional[Any]							,					UpperCamelCase__       :   Dict=None							,					UpperCamelCase__       :   Tuple=None							,					UpperCamelCase__       :   List[Any]=None							,					UpperCamelCase__       :   List[Any]=768							,					UpperCamelCase__       :   Tuple=12							,					UpperCamelCase__       :   Optional[int]=12							,					UpperCamelCase__       :   Dict=3_072							,					UpperCamelCase__       :   Union[str, Any]=0.1							,					UpperCamelCase__       :   Dict=0.1							,					UpperCamelCase__       :   Optional[int]=1_026							,					UpperCamelCase__       :   Any=0.02							,					UpperCamelCase__       :   Dict=1e-12							,					UpperCamelCase__       :   List[str]="absolute"							,					UpperCamelCase__       :   Union[str, Any]=True							,					UpperCamelCase__       :   Tuple=None							,					UpperCamelCase__       :   Union[str, Any]=False							,					UpperCamelCase__       :   Optional[int]=False							,					UpperCamelCase__       :   Dict=None							,					UpperCamelCase__       :   str=None							,					**UpperCamelCase__       :   str							,					):






										'''simple docstring'''






										super().__init__(pad_token_id=UpperCamelCase__							,					mask_token_id=UpperCamelCase__							,					**UpperCamelCase__       )

										lowercase_  							=			vocab_size
										lowercase_  							=			hidden_size
										lowercase_  							=			num_hidden_layers
										lowercase_  							=			num_attention_heads
										lowercase_  							=			intermediate_size
										lowercase_  							=			hidden_dropout_prob
										lowercase_  							=			attention_probs_dropout_prob
										lowercase_  							=			max_position_embeddings
										lowercase_  							=			initializer_range
										lowercase_  							=			layer_norm_eps
										lowercase_  							=			position_embedding_type
										lowercase_  							=			use_cache
										lowercase_  							=			emb_layer_norm_before
										lowercase_  							=			token_dropout
										lowercase_  							=			is_folding_model
										if is_folding_model:
															if esmfold_config is None:
																				logger.info("""No esmfold_config supplied for folding model, using default values."""       )
																				lowercase_  							=			EsmFoldConfig()
															elif isinstance(UpperCamelCase__							,					UpperCamelCase__       ):
																				lowercase_  							=			EsmFoldConfig(**UpperCamelCase__       )
															lowercase_  							=			esmfold_config
															if vocab_list is None:
																				logger.warning("""No vocab_list supplied for folding model, assuming the ESM-2 vocabulary!"""       )
																				lowercase_  							=			get_default_vocab_list()
															else:
																				lowercase_  							=			vocab_list
										else:
															lowercase_  							=			None
															lowercase_  							=			None
										if self.esmfold_config is not None and getattr(self.esmfold_config							,					"""use_esm_attn_map"""							,					UpperCamelCase__       ):
															raise ValueError("""The HuggingFace port of ESMFold does not support use_esm_attn_map at this time!"""       )







					def   UpperCAmelCase__       (     self       :   List[Any]       ):






										'''simple docstring'''






										lowercase_  							=			super().to_dict()
										if isinstance(self.esmfold_config							,					UpperCamelCase__       ):
															lowercase_  							=			self.esmfold_config.to_dict()
										return output







@dataclass
class        UpperCamelCase__   :
					__SCREAMING_SNAKE_CASE       :  str						=   None
					__SCREAMING_SNAKE_CASE       :  bool						=   True
					__SCREAMING_SNAKE_CASE       :  bool						=   False
					__SCREAMING_SNAKE_CASE       :  bool						=   False
					__SCREAMING_SNAKE_CASE       :  bool						=   False
					__SCREAMING_SNAKE_CASE       :  float						=   0

					__SCREAMING_SNAKE_CASE       :  bool						=   True
					__SCREAMING_SNAKE_CASE       :  bool						=   False

					__SCREAMING_SNAKE_CASE       :  int						=   128
					__SCREAMING_SNAKE_CASE       :  "TrunkConfig"						=   None

					def   UpperCAmelCase__       (     self       :   Tuple       ):






										'''simple docstring'''






										if self.trunk is None:
															lowercase_  							=			TrunkConfig()
										elif isinstance(self.trunk							,					UpperCamelCase__       ):
															lowercase_  							=			TrunkConfig(**self.trunk       )







					def   UpperCAmelCase__       (     self       :   Union[str, Any]       ):






										'''simple docstring'''






										lowercase_  							=			asdict(self       )
										lowercase_  							=			self.trunk.to_dict()
										return output







@dataclass
class        UpperCamelCase__   :
					__SCREAMING_SNAKE_CASE       :  int						=   48
					__SCREAMING_SNAKE_CASE       :  int						=   1024
					__SCREAMING_SNAKE_CASE       :  int						=   128
					__SCREAMING_SNAKE_CASE       :  int						=   32
					__SCREAMING_SNAKE_CASE       :  int						=   32
					__SCREAMING_SNAKE_CASE       :  int						=   32
					__SCREAMING_SNAKE_CASE       :  float						=   0
					__SCREAMING_SNAKE_CASE       :  float						=   0
					__SCREAMING_SNAKE_CASE       :  bool						=   False
					__SCREAMING_SNAKE_CASE       :  int						=   4
					__SCREAMING_SNAKE_CASE       :  Optional[int]						=   128
					__SCREAMING_SNAKE_CASE       :  "StructureModuleConfig"						=   None

					def   UpperCAmelCase__       (     self       :   Union[str, Any]       ):






										'''simple docstring'''






										if self.structure_module is None:
															lowercase_  							=			StructureModuleConfig()
										elif isinstance(self.structure_module							,					UpperCamelCase__       ):
															lowercase_  							=			StructureModuleConfig(**self.structure_module       )

										if self.max_recycles <= 0:
															raise ValueError(F'''`max_recycles` should be positive, got {self.max_recycles}.'''       )
										if self.sequence_state_dim % self.sequence_state_dim != 0:
															raise ValueError(
															    """`sequence_state_dim` should be a round multiple of `sequence_state_dim`, got"""
															    F''' {self.sequence_state_dim} and {self.sequence_state_dim}.'''       )
										if self.pairwise_state_dim % self.pairwise_state_dim != 0:
															raise ValueError(
															    """`pairwise_state_dim` should be a round multiple of `pairwise_state_dim`, got"""
															    F''' {self.pairwise_state_dim} and {self.pairwise_state_dim}.'''       )

										lowercase_  							=			self.sequence_state_dim // self.sequence_head_width
										lowercase_  							=			self.pairwise_state_dim // self.pairwise_head_width

										if self.sequence_state_dim != sequence_num_heads * self.sequence_head_width:
															raise ValueError(
															    """`sequence_state_dim` should be equal to `sequence_num_heads * sequence_head_width, got"""
															    F''' {self.sequence_state_dim} != {sequence_num_heads} * {self.sequence_head_width}.'''       )
										if self.pairwise_state_dim != pairwise_num_heads * self.pairwise_head_width:
															raise ValueError(
															    """`pairwise_state_dim` should be equal to `pairwise_num_heads * pairwise_head_width, got"""
															    F''' {self.pairwise_state_dim} != {pairwise_num_heads} * {self.pairwise_head_width}.'''       )
										if self.pairwise_state_dim % 2 != 0:
															raise ValueError(F'''`pairwise_state_dim` should be even, got {self.pairwise_state_dim}.'''       )

										if self.dropout >= 0.4:
															raise ValueError(F'''`dropout` should not be greater than 0.4, got {self.dropout}.'''       )







					def   UpperCAmelCase__       (     self       :   Any       ):






										'''simple docstring'''






										lowercase_  							=			asdict(self       )
										lowercase_  							=			self.structure_module.to_dict()
										return output







@dataclass
class        UpperCamelCase__   :
					__SCREAMING_SNAKE_CASE       :  int						=   384
					__SCREAMING_SNAKE_CASE       :  int						=   128
					__SCREAMING_SNAKE_CASE       :  int						=   16
					__SCREAMING_SNAKE_CASE       :  int						=   128
					__SCREAMING_SNAKE_CASE       :  int						=   12
					__SCREAMING_SNAKE_CASE       :  int						=   4
					__SCREAMING_SNAKE_CASE       :  int						=   8
					__SCREAMING_SNAKE_CASE       :  float						=   0.1
					__SCREAMING_SNAKE_CASE       :  int						=   8
					__SCREAMING_SNAKE_CASE       :  int						=   1
					__SCREAMING_SNAKE_CASE       :  int						=   2
					__SCREAMING_SNAKE_CASE       :  int						=   7
					__SCREAMING_SNAKE_CASE       :  int						=   10
					__SCREAMING_SNAKE_CASE       :  float						=   1e-8
					__SCREAMING_SNAKE_CASE       :  float						=   1e5

					def   UpperCAmelCase__       (     self       :   List[str]       ):






										'''simple docstring'''






										return asdict(self       )



def 			UpperCAmelCase_				(      ):
					return (
					    "<cls>",
					    "<pad>",
					    "<eos>",
					    "<unk>",
					    "L",
					    "A",
					    "G",
					    "V",
					    "S",
					    "E",
					    "R",
					    "T",
					    "I",
					    "D",
					    "P",
					    "K",
					    "Q",
					    "N",
					    "F",
					    "Y",
					    "M",
					    "H",
					    "W",
					    "C",
					    "X",
					    "B",
					    "U",
					    "Z",
					    "O",
					    ".",
					    "-",
					    "<null_1>",
					    "<mask>",
					)


