



'''simple docstring'''


import argparse
import collections
import os
import re
import tempfile

import pandas as pd
from datasets import Dataset
from huggingface_hub import hf_hub_download, upload_folder

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/update_metadata.py
lowercase_      :				Tuple   					=    '''src/transformers'''


# This is to make sure the transformers module imported is the one in the repo.
lowercase_      :				str   					=    direct_transformers_import(TRANSFORMERS_PATH)


# Regexes that match TF/Flax/PT model names.
lowercase_      :				str   					=    re.compile(r'''TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)''')
lowercase_      :				List[Any]   					=    re.compile(r'''Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)''')
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
lowercase_      :				Tuple   					=    re.compile(r'''(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)''')


# Fill this with tuples (pipeline_tag, model_mapping, auto_model)
lowercase_      :				Union[str, Any]   					=    [
    ('''pretraining''', '''MODEL_FOR_PRETRAINING_MAPPING_NAMES''', '''AutoModelForPreTraining'''),
    ('''feature-extraction''', '''MODEL_MAPPING_NAMES''', '''AutoModel'''),
    ('''audio-classification''', '''MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForAudioClassification'''),
    ('''text-generation''', '''MODEL_FOR_CAUSAL_LM_MAPPING_NAMES''', '''AutoModelForCausalLM'''),
    ('''automatic-speech-recognition''', '''MODEL_FOR_CTC_MAPPING_NAMES''', '''AutoModelForCTC'''),
    ('''image-classification''', '''MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForImageClassification'''),
    ('''image-segmentation''', '''MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES''', '''AutoModelForImageSegmentation'''),
    ('''fill-mask''', '''MODEL_FOR_MASKED_LM_MAPPING_NAMES''', '''AutoModelForMaskedLM'''),
    ('''object-detection''', '''MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES''', '''AutoModelForObjectDetection'''),
    (
        '''zero-shot-object-detection''',
        '''MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES''',
        '''AutoModelForZeroShotObjectDetection''',
    ),
    ('''question-answering''', '''MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES''', '''AutoModelForQuestionAnswering'''),
    ('''text2text-generation''', '''MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES''', '''AutoModelForSeq2SeqLM'''),
    ('''text-classification''', '''MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForSequenceClassification'''),
    ('''automatic-speech-recognition''', '''MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES''', '''AutoModelForSpeechSeq2Seq'''),
    (
        '''table-question-answering''',
        '''MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES''',
        '''AutoModelForTableQuestionAnswering''',
    ),
    ('''token-classification''', '''MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForTokenClassification'''),
    ('''multiple-choice''', '''MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES''', '''AutoModelForMultipleChoice'''),
    (
        '''next-sentence-prediction''',
        '''MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES''',
        '''AutoModelForNextSentencePrediction''',
    ),
    (
        '''audio-frame-classification''',
        '''MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES''',
        '''AutoModelForAudioFrameClassification''',
    ),
    ('''audio-xvector''', '''MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES''', '''AutoModelForAudioXVector'''),
    (
        '''document-question-answering''',
        '''MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES''',
        '''AutoModelForDocumentQuestionAnswering''',
    ),
    (
        '''visual-question-answering''',
        '''MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING_NAMES''',
        '''AutoModelForVisualQuestionAnswering''',
    ),
    ('''image-to-text''', '''MODEL_FOR_FOR_VISION_2_SEQ_MAPPING_NAMES''', '''AutoModelForVision2Seq'''),
    (
        '''zero-shot-image-classification''',
        '''MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES''',
        '''AutoModelForZeroShotImageClassification''',
    ),
    ('''depth-estimation''', '''MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES''', '''AutoModelForDepthEstimation'''),
    ('''video-classification''', '''MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForVideoClassification'''),
    ('''mask-generation''', '''MODEL_FOR_MASK_GENERATION_MAPPING_NAMES''', '''AutoModelForMaskGeneration'''),
]







def      SCREAMING_SNAKE_CASE				(				lowercase_    :      List[str]						):
						lowercase        =							re.finditer(""".+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)""" ,  lowercase_						)
						return [m.group(0						) for m in matches]







def      SCREAMING_SNAKE_CASE				(				):
						lowercase        =							transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
						lowercase        =							{
						    config.replace("""Config""" ,  """"""						): model_type for model_type, config in config_maping_names.items()
						}

						# Dictionaries flagging if each model prefix has a backend in PT/TF/Flax.
						lowercase        =							collections.defaultdict(lowercase_						)
						lowercase        =							collections.defaultdict(lowercase_						)
						lowercase        =							collections.defaultdict(lowercase_						)

						# Let's lookup through all transformers object (once) and find if models are supported by a given backend.
						for attr_name in dir(lowercase_						):
												lowercase        =							None
												if _re_tf_models.match(lowercase_						) is not None:
																		lowercase        =							tf_models
																		lowercase        =							_re_tf_models.match(lowercase_						).groups()[0]
												elif _re_flax_models.match(lowercase_						) is not None:
																		lowercase        =							flax_models
																		lowercase        =							_re_flax_models.match(lowercase_						).groups()[0]
												elif _re_pt_models.match(lowercase_						) is not None:
																		lowercase        =							pt_models
																		lowercase        =							_re_pt_models.match(lowercase_						).groups()[0]

												if lookup_dict is not None:
																		while len(lowercase_						) > 0:
																								if attr_name in model_prefix_to_model_type:
																														lowercase        =							True
																														break
																								# Try again after removing the last word in the name
																								lowercase        =							"""""".join(camel_case_split(lowercase_						)[:-1]						)

						lowercase        =							set(list(pt_models.keys()						) + list(tf_models.keys()						) + list(flax_models.keys()						)						)
						lowercase        =							list(lowercase_						)
						all_models.sort()

						lowercase        =							{"""model_type""": all_models}
						lowercase        =							[pt_models[t] for t in all_models]
						lowercase        =							[tf_models[t] for t in all_models]
						lowercase        =							[flax_models[t] for t in all_models]

						# Now let's use the auto-mapping names to make sure
						lowercase        =							{}
						for t in all_models:
												if t in transformers_module.models.auto.processing_auto.PROCESSOR_MAPPING_NAMES:
																		lowercase        =							"""AutoProcessor"""
												elif t in transformers_module.models.auto.tokenization_auto.TOKENIZER_MAPPING_NAMES:
																		lowercase        =							"""AutoTokenizer"""
												elif t in transformers_module.models.auto.feature_extraction_auto.FEATURE_EXTRACTOR_MAPPING_NAMES:
																		lowercase        =							"""AutoFeatureExtractor"""
												else:
																		# Default to AutoTokenizer if a model has nothing, for backward compatibility.
																		lowercase        =							"""AutoTokenizer"""

						lowercase        =							[processors[t] for t in all_models]

						return pd.DataFrame(lowercase_						)







def      SCREAMING_SNAKE_CASE				(				lowercase_    :      Optional[int]						):
						lowercase        =							[
						    transformers_module.models.auto.modeling_auto,
						    transformers_module.models.auto.modeling_tf_auto,
						    transformers_module.models.auto.modeling_flax_auto,
						]
						for pipeline_tag, model_mapping, auto_class in PIPELINE_TAGS_AND_AUTO_MODELS:
												lowercase        =							[model_mapping, F"""TF_{model_mapping}""", F"""FLAX_{model_mapping}"""]
												lowercase        =							[auto_class, F"""TF_{auto_class}""", F"""Flax_{auto_class}"""]
												# Loop through all three frameworks
												for module, cls, mapping in zip(lowercase_ ,  lowercase_ ,  lowercase_						):
																		# The type of pipeline may not exist in this framework
																		if not hasattr(lowercase_ ,  lowercase_						):
																								continue
																		# First extract all model_names
																		lowercase        =							[]
																		for name in getattr(lowercase_ ,  lowercase_						).values():
																								if isinstance(lowercase_ ,  lowercase_						):
																														model_names.append(lowercase_						)
																								else:
																														model_names.extend(list(lowercase_						)						)

            # Add pipeline tag and auto model class for those models
																		table.update({model_name: (pipeline_tag, cls) for model_name in model_names}						)

						return table







def      SCREAMING_SNAKE_CASE				(				lowercase_    :      Dict ,  lowercase_    :      Any						):
						lowercase        =							get_frameworks_table()
						lowercase        =							Dataset.from_pandas(lowercase_						)

						lowercase        =							hf_hub_download(
						    """huggingface/transformers-metadata""" ,  """pipeline_tags.json""" ,  repo_type="""dataset""" ,  token=lowercase_						)
						lowercase        =							Dataset.from_json(lowercase_						)
						lowercase        =							{
						    tags_dataset[i]["""model_class"""]: (tags_dataset[i]["""pipeline_tag"""], tags_dataset[i]["""auto_class"""])
						    for i in range(len(lowercase_						)						)
						}
						lowercase        =							update_pipeline_and_auto_class_table(lowercase_						)

						# Sort the model classes to avoid some nondeterministic updates to create false update commits.
						lowercase        =							sorted(table.keys()						)
						lowercase        =							pd.DataFrame(
						    {
						        """model_class""": model_classes,
						        """pipeline_tag""": [table[m][0] for m in model_classes],
						        """auto_class""": [table[m][1] for m in model_classes],
						    }						)
						lowercase        =							Dataset.from_pandas(lowercase_						)

						with tempfile.TemporaryDirectory() as tmp_dir:
												frameworks_dataset.to_json(os.path.join(lowercase_ ,  """frameworks.json"""						)						)
												tags_dataset.to_json(os.path.join(lowercase_ ,  """pipeline_tags.json"""						)						)

												if commit_sha is not None:
																		lowercase        =							(
																		    F"""Update with commit {commit_sha}\n\nSee: """
																		    F"""https://github.com/huggingface/transformers/commit/{commit_sha}"""
																		)
												else:
																		lowercase        =							"""Update"""

												upload_folder(
												    repo_id="""huggingface/transformers-metadata""" ,  folder_path=lowercase_ ,  repo_type="""dataset""" ,  token=lowercase_ ,  commit_message=lowercase_ ,  )







def      SCREAMING_SNAKE_CASE				(				):
						lowercase        =							{tag: cls for tag, _, cls in PIPELINE_TAGS_AND_AUTO_MODELS}
						lowercase        =							transformers_module.pipelines.SUPPORTED_TASKS
						lowercase        =							[]
						for key in pipeline_tasks:
												if key not in in_table:
																		lowercase        =							pipeline_tasks[key]["""pt"""]
																		if isinstance(lowercase_ ,  (list, tuple)						):
																								lowercase        =							model[0]
																		lowercase        =							model.__name__
																		if model not in in_table.values():
																								missing.append(lowercase_						)

						if len(lowercase_						) > 0:
												lowercase        =							""", """.join(lowercase_						)
												raise ValueError(
												    """The following pipeline tags are not present in the `PIPELINE_TAGS_AND_AUTO_MODELS` constant inside """
												    F"""`utils/update_metadata.py`: {msg}. Please add them!"""						)


if __name__ == "__main__":
						lowercase_      :				Any   					=    argparse.ArgumentParser()
						parser.add_argument('''--token''', type=str, help='''The token to use to push to the transformers-metadata dataset.''')
						parser.add_argument('''--commit_sha''', type=str, help='''The sha of the commit going with this update.''')
						parser.add_argument('''--check-only''', action='''store_true''', help='''Activate to just check all pipelines are present.''')
						lowercase_      :				Optional[Any]   					=    parser.parse_args()

						if args.check_only:
												check_pipeline_tags()
						else:
												update_metadata(args.token, args.commit_sha)







'''simple docstring'''


# coding=utf-8
# Copyright 2023 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# this script dumps information about the environment

import os
import platform
import sys


lowercase_      :				List[str]   					=    '''3'''

print('''Python version:''', sys.version)

print('''OS platform:''', platform.platform())
print('''OS architecture:''', platform.machine())

try:
						import torch

						print('''Torch version:''', torch.__version__)
						print('''Cuda available:''', torch.cuda.is_available())
						print('''Cuda version:''', torch.version.cuda)
						print('''CuDNN version:''', torch.backends.cudnn.version())
						print('''Number of GPUs available:''', torch.cuda.device_count())
except ImportError:
						print('''Torch version:''', None)

try:
						import transformers

						print('''transformers version:''', transformers.__version__)
except ImportError:
						print('''transformers version:''', None)


