



import argparse
import collections
import os
import re
import tempfile

import pandas as pd
from datasets import Dataset
from huggingface_hub import hf_hub_download, upload_folder

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/update_metadata.py
_snake_case	=						"src/transformers"


# This is to make sure the transformers module imported is the one in the repo.
_snake_case	=						direct_transformers_import(TRANSFORMERS_PATH)


# Regexes that match TF/Flax/PT model names.
_snake_case	=						re.compile(R"TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")
_snake_case	=						re.compile(R"Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
_snake_case	=						re.compile(R"(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")


# Fill this with tuples (pipeline_tag, model_mapping, auto_model)
_snake_case	=						[
    ("pretraining", "MODEL_FOR_PRETRAINING_MAPPING_NAMES", "AutoModelForPreTraining"),
    ("feature-extraction", "MODEL_MAPPING_NAMES", "AutoModel"),
    ("audio-classification", "MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES", "AutoModelForAudioClassification"),
    ("text-generation", "MODEL_FOR_CAUSAL_LM_MAPPING_NAMES", "AutoModelForCausalLM"),
    ("automatic-speech-recognition", "MODEL_FOR_CTC_MAPPING_NAMES", "AutoModelForCTC"),
    ("image-classification", "MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES", "AutoModelForImageClassification"),
    ("image-segmentation", "MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES", "AutoModelForImageSegmentation"),
    ("fill-mask", "MODEL_FOR_MASKED_LM_MAPPING_NAMES", "AutoModelForMaskedLM"),
    ("object-detection", "MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES", "AutoModelForObjectDetection"),
    (
        "zero-shot-object-detection",
        "MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES",
        "AutoModelForZeroShotObjectDetection",
    ),
    ("question-answering", "MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES", "AutoModelForQuestionAnswering"),
    ("text2text-generation", "MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES", "AutoModelForSeq2SeqLM"),
    ("text-classification", "MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES", "AutoModelForSequenceClassification"),
    ("automatic-speech-recognition", "MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES", "AutoModelForSpeechSeq2Seq"),
    (
        "table-question-answering",
        "MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES",
        "AutoModelForTableQuestionAnswering",
    ),
    ("token-classification", "MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES", "AutoModelForTokenClassification"),
    ("multiple-choice", "MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES", "AutoModelForMultipleChoice"),
    (
        "next-sentence-prediction",
        "MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES",
        "AutoModelForNextSentencePrediction",
    ),
    (
        "audio-frame-classification",
        "MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES",
        "AutoModelForAudioFrameClassification",
    ),
    ("audio-xvector", "MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES", "AutoModelForAudioXVector"),
    (
        "document-question-answering",
        "MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES",
        "AutoModelForDocumentQuestionAnswering",
    ),
    (
        "visual-question-answering",
        "MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING_NAMES",
        "AutoModelForVisualQuestionAnswering",
    ),
    ("image-to-text", "MODEL_FOR_FOR_VISION_2_SEQ_MAPPING_NAMES", "AutoModelForVision2Seq"),
    (
        "zero-shot-image-classification",
        "MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES",
        "AutoModelForZeroShotImageClassification",
    ),
    ("depth-estimation", "MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES", "AutoModelForDepthEstimation"),
    ("video-classification", "MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES", "AutoModelForVideoClassification"),
    ("mask-generation", "MODEL_FOR_MASK_GENERATION_MAPPING_NAMES", "AutoModelForMaskGeneration"),
]






def        A       (      _lowerCamelCase						):





							'''simple docstring'''




							_lowerCAmelCase  :							List[str]				     =					re.finditer(".+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)"       ,				_lowerCamelCase						)
							return [m.group(0						) for m in matches]






def        A       (      ):





							'''simple docstring'''




							_lowerCAmelCase  :							str				     =					transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
							_lowerCAmelCase  :							Tuple				     =					{
							    config.replace("Config"       ,				""						): model_type for model_type, config in config_maping_names.items()
							}

							# Dictionaries flagging if each model prefix has a backend in PT/TF/Flax.
							_lowerCAmelCase  :							List[str]				     =					collections.defaultdict(_lowerCamelCase						)
							_lowerCAmelCase  :							List[str]				     =					collections.defaultdict(_lowerCamelCase						)
							_lowerCAmelCase  :							Optional[Any]				     =					collections.defaultdict(_lowerCamelCase						)

							# Let's lookup through all transformers object (once) and find if models are supported by a given backend.
							for attr_name in dir(_lowerCamelCase						):
														_lowerCAmelCase  :							List[Any]				     =					None
														if _re_tf_models.match(_lowerCamelCase						) is not None:
																					_lowerCAmelCase  :							Dict				     =					tf_models
																					_lowerCAmelCase  :							Optional[int]				     =					_re_tf_models.match(_lowerCamelCase						).groups()[0]
														elif _re_flax_models.match(_lowerCamelCase						) is not None:
																					_lowerCAmelCase  :							Union[str, Any]				     =					flax_models
																					_lowerCAmelCase  :							Union[str, Any]				     =					_re_flax_models.match(_lowerCamelCase						).groups()[0]
														elif _re_pt_models.match(_lowerCamelCase						) is not None:
																					_lowerCAmelCase  :							str				     =					pt_models
																					_lowerCAmelCase  :							List[str]				     =					_re_pt_models.match(_lowerCamelCase						).groups()[0]

														if lookup_dict is not None:
																					while len(_lowerCamelCase						) > 0:
																												if attr_name in model_prefix_to_model_type:
																																			_lowerCAmelCase  :							Optional[Any]				     =					True
																																			break
																												# Try again after removing the last word in the name
																												_lowerCAmelCase  :							Tuple				     =					"".join(camel_case_split(_lowerCamelCase						)[:-1]						)

							_lowerCAmelCase  :							Union[str, Any]				     =					set(list(pt_models.keys()						) + list(tf_models.keys()						) + list(flax_models.keys()						)						)
							_lowerCAmelCase  :							List[Any]				     =					list(_lowerCamelCase						)
							all_models.sort()

							_lowerCAmelCase  :							List[str]				     =					{"model_type": all_models}
							_lowerCAmelCase  :							Tuple				     =					[pt_models[t] for t in all_models]
							_lowerCAmelCase  :							Optional[int]				     =					[tf_models[t] for t in all_models]
							_lowerCAmelCase  :							Dict				     =					[flax_models[t] for t in all_models]

							# Now let's use the auto-mapping names to make sure
							_lowerCAmelCase  :							Optional[int]				     =					{}
							for t in all_models:
														if t in transformers_module.models.auto.processing_auto.PROCESSOR_MAPPING_NAMES:
																					_lowerCAmelCase  :							Optional[int]				     =					"AutoProcessor"
														elif t in transformers_module.models.auto.tokenization_auto.TOKENIZER_MAPPING_NAMES:
																					_lowerCAmelCase  :							Dict				     =					"AutoTokenizer"
														elif t in transformers_module.models.auto.feature_extraction_auto.FEATURE_EXTRACTOR_MAPPING_NAMES:
																					_lowerCAmelCase  :							List[Any]				     =					"AutoFeatureExtractor"
														else:
																					# Default to AutoTokenizer if a model has nothing, for backward compatibility.
																					_lowerCAmelCase  :							Tuple				     =					"AutoTokenizer"

							_lowerCAmelCase  :							Union[str, Any]				     =					[processors[t] for t in all_models]

							return pd.DataFrame(_lowerCamelCase						)






def        A       (      _lowerCamelCase						):





							'''simple docstring'''




							_lowerCAmelCase  :							Any				     =					[
							    transformers_module.models.auto.modeling_auto,
							    transformers_module.models.auto.modeling_tf_auto,
							    transformers_module.models.auto.modeling_flax_auto,
							]
							for pipeline_tag, model_mapping, auto_class in PIPELINE_TAGS_AND_AUTO_MODELS:
														_lowerCAmelCase  :							str				     =					[model_mapping, F"TF_{model_mapping}", F"FLAX_{model_mapping}"]
														_lowerCAmelCase  :							Dict				     =					[auto_class, F"TF_{auto_class}", F"Flax_{auto_class}"]
														# Loop through all three frameworks
														for module, cls, mapping in zip(_lowerCamelCase       ,				_lowerCamelCase       ,				_lowerCamelCase						):
																					# The type of pipeline may not exist in this framework
																					if not hasattr(_lowerCamelCase       ,				_lowerCamelCase						):
																												continue
																					# First extract all model_names
																					_lowerCAmelCase  :							int				     =					[]
																					for name in getattr(_lowerCamelCase       ,				_lowerCamelCase						).values():
																												if isinstance(_lowerCamelCase       ,				_lowerCamelCase						):
																																			model_names.append(_lowerCamelCase						)
																												else:
																																			model_names.extend(list(_lowerCamelCase						)						)

            # Add pipeline tag and auto model class for those models
																					table.update({model_name: (pipeline_tag, cls) for model_name in model_names}						)

							return table






def        A       (      _lowerCamelCase       ,				_lowerCamelCase						):





							'''simple docstring'''




							_lowerCAmelCase  :							Dict				     =					get_frameworks_table()
							_lowerCAmelCase  :							Union[str, Any]				     =					Dataset.from_pandas(_lowerCamelCase						)

							_lowerCAmelCase  :							int				     =					hf_hub_download(
							    "huggingface/transformers-metadata"       ,				"pipeline_tags.json"       ,				repo_type="dataset"       ,				token=_lowerCamelCase						)
							_lowerCAmelCase  :							int				     =					Dataset.from_json(_lowerCamelCase						)
							_lowerCAmelCase  :							Tuple				     =					{
							    tags_dataset[i]["model_class"]: (tags_dataset[i]["pipeline_tag"], tags_dataset[i]["auto_class"])
							    for i in range(len(_lowerCamelCase						)						)
							}
							_lowerCAmelCase  :							Tuple				     =					update_pipeline_and_auto_class_table(_lowerCamelCase						)

							# Sort the model classes to avoid some nondeterministic updates to create false update commits.
							_lowerCAmelCase  :							Optional[Any]				     =					sorted(table.keys()						)
							_lowerCAmelCase  :							str				     =					pd.DataFrame(
							    {
							        "model_class": model_classes,
							        "pipeline_tag": [table[m][0] for m in model_classes],
							        "auto_class": [table[m][1] for m in model_classes],
							    }						)
							_lowerCAmelCase  :							int				     =					Dataset.from_pandas(_lowerCamelCase						)

							with tempfile.TemporaryDirectory() as tmp_dir:
														frameworks_dataset.to_json(os.path.join(_lowerCamelCase       ,				"frameworks.json"						)						)
														tags_dataset.to_json(os.path.join(_lowerCamelCase       ,				"pipeline_tags.json"						)						)

														if commit_sha is not None:
																					_lowerCAmelCase  :							Optional[Any]				     =					(
																					    F"Update with commit {commit_sha}\n\nSee: "
																					    F"https://github.com/huggingface/transformers/commit/{commit_sha}"
																					)
														else:
																					_lowerCAmelCase  :							Tuple				     =					"Update"

														upload_folder(
														    repo_id="huggingface/transformers-metadata"       ,				folder_path=_lowerCamelCase       ,				repo_type="dataset"       ,				token=_lowerCamelCase       ,				commit_message=_lowerCamelCase       ,				)






def        A       (      ):





							'''simple docstring'''




							_lowerCAmelCase  :							Any				     =					{tag: cls for tag, _, cls in PIPELINE_TAGS_AND_AUTO_MODELS}
							_lowerCAmelCase  :							Dict				     =					transformers_module.pipelines.SUPPORTED_TASKS
							_lowerCAmelCase  :							List[Any]				     =					[]
							for key in pipeline_tasks:
														if key not in in_table:
																					_lowerCAmelCase  :							Any				     =					pipeline_tasks[key]["pt"]
																					if isinstance(_lowerCamelCase       ,				(list, tuple)						):
																												_lowerCAmelCase  :							Optional[Any]				     =					model[0]
																					_lowerCAmelCase  :							List[str]				     =					model.__name__
																					if model not in in_table.values():
																												missing.append(_lowerCamelCase						)

							if len(_lowerCamelCase						) > 0:
														_lowerCAmelCase  :							str				     =					", ".join(_lowerCamelCase						)
														raise ValueError(
														    "The following pipeline tags are not present in the `PIPELINE_TAGS_AND_AUTO_MODELS` constant inside "
														    F"`utils/update_metadata.py`: {msg}. Please add them!"						)


if __name__ == "__main__":
						_snake_case	=						argparse.ArgumentParser()
						parser.add_argument("--token", type=str, help="The token to use to push to the transformers-metadata dataset.")
						parser.add_argument("--commit_sha", type=str, help="The sha of the commit going with this update.")
						parser.add_argument("--check-only", action="store_true", help="Activate to just check all pipelines are present.")
						_snake_case	=						parser.parse_args()

						if args.check_only:
												check_pipeline_tags()
						else:
												update_metadata(args.token, args.commit_sha)





from ...configuration_utils import PretrainedConfig
from ...utils import logging


_snake_case	=						logging.get_logger(__name__)

_snake_case	=						{
    "weiweishi/roc-bert-base-zh": "https://huggingface.co/weiweishi/roc-bert-base-zh/resolve/main/config.json",
}



class 						UpperCAmelCase_					( a):
							lowerCamelCase__         =	'roc_bert'







							def __init__(     self,    __a=3_0522,    __a=768,    __a=12,    __a=12,    __a=3072,    __a="gelu",    __a=0.1,    __a=0.1,    __a=512,    __a=2,    __a=0.02,    __a=1E-12,    __a=True,    __a=0,    __a="absolute",    __a=None,    __a=True,    __a=True,    __a=768,    __a=910,    __a=512,    __a=2_4858,    __a=True,    **__a,    ):

														'''simple docstring'''

														_lowerCAmelCase  :							Optional[Any]				     =					vocab_size
														_lowerCAmelCase  :							Optional[Any]				     =					max_position_embeddings
														_lowerCAmelCase  :							Union[str, Any]				     =					hidden_size
														_lowerCAmelCase  :							str				     =					num_hidden_layers
														_lowerCAmelCase  :							List[Any]				     =					num_attention_heads
														_lowerCAmelCase  :							int				     =					intermediate_size
														_lowerCAmelCase  :							Optional[int]				     =					hidden_act
														_lowerCAmelCase  :							Optional[Any]				     =					hidden_dropout_prob
														_lowerCAmelCase  :							Optional[Any]				     =					attention_probs_dropout_prob
														_lowerCAmelCase  :							Union[str, Any]				     =					initializer_range
														_lowerCAmelCase  :							Optional[Any]				     =					type_vocab_size
														_lowerCAmelCase  :							int				     =					layer_norm_eps
														_lowerCAmelCase  :							Union[str, Any]				     =					use_cache
														_lowerCAmelCase  :							Optional[int]				     =					enable_pronunciation
														_lowerCAmelCase  :							Dict				     =					enable_shape
														_lowerCAmelCase  :							Optional[Any]				     =					pronunciation_embed_dim
														_lowerCAmelCase  :							Any				     =					pronunciation_vocab_size
														_lowerCAmelCase  :							List[str]				     =					shape_embed_dim
														_lowerCAmelCase  :							int				     =					shape_vocab_size
														_lowerCAmelCase  :							Optional[int]				     =					concat_input
														_lowerCAmelCase  :							Dict				     =					position_embedding_type
														_lowerCAmelCase  :							Tuple				     =					classifier_dropout
														super().__init__(pad_token_id=__a,    **__a)
