






import argparse
import collections
import os
import re
import tempfile

import pandas as pd
from datasets import Dataset
from huggingface_hub import hf_hub_download, upload_folder

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/update_metadata.py
lowerCamelCase : str						   =       '''src/transformers'''


# This is to make sure the transformers module imported is the one in the repo.
lowerCamelCase : str						   =       direct_transformers_import(TRANSFORMERS_PATH)


# Regexes that match TF/Flax/PT model names.
lowerCamelCase : Dict						   =       re.compile(r'''TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)''')
lowerCamelCase : List[str]						   =       re.compile(r'''Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)''')
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
lowerCamelCase : str						   =       re.compile(r'''(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)''')


# Fill this with tuples (pipeline_tag, model_mapping, auto_model)
lowerCamelCase : Dict						   =       [
    ('''pretraining''', '''MODEL_FOR_PRETRAINING_MAPPING_NAMES''', '''AutoModelForPreTraining'''),
    ('''feature-extraction''', '''MODEL_MAPPING_NAMES''', '''AutoModel'''),
    ('''audio-classification''', '''MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForAudioClassification'''),
    ('''text-generation''', '''MODEL_FOR_CAUSAL_LM_MAPPING_NAMES''', '''AutoModelForCausalLM'''),
    ('''automatic-speech-recognition''', '''MODEL_FOR_CTC_MAPPING_NAMES''', '''AutoModelForCTC'''),
    ('''image-classification''', '''MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForImageClassification'''),
    ('''image-segmentation''', '''MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES''', '''AutoModelForImageSegmentation'''),
    ('''fill-mask''', '''MODEL_FOR_MASKED_LM_MAPPING_NAMES''', '''AutoModelForMaskedLM'''),
    ('''object-detection''', '''MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES''', '''AutoModelForObjectDetection'''),
    (
        '''zero-shot-object-detection''',
        '''MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES''',
        '''AutoModelForZeroShotObjectDetection''',
    ),
    ('''question-answering''', '''MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES''', '''AutoModelForQuestionAnswering'''),
    ('''text2text-generation''', '''MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES''', '''AutoModelForSeq2SeqLM'''),
    ('''text-classification''', '''MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForSequenceClassification'''),
    ('''automatic-speech-recognition''', '''MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES''', '''AutoModelForSpeechSeq2Seq'''),
    (
        '''table-question-answering''',
        '''MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES''',
        '''AutoModelForTableQuestionAnswering''',
    ),
    ('''token-classification''', '''MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForTokenClassification'''),
    ('''multiple-choice''', '''MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES''', '''AutoModelForMultipleChoice'''),
    (
        '''next-sentence-prediction''',
        '''MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES''',
        '''AutoModelForNextSentencePrediction''',
    ),
    (
        '''audio-frame-classification''',
        '''MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES''',
        '''AutoModelForAudioFrameClassification''',
    ),
    ('''audio-xvector''', '''MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES''', '''AutoModelForAudioXVector'''),
    (
        '''document-question-answering''',
        '''MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES''',
        '''AutoModelForDocumentQuestionAnswering''',
    ),
    (
        '''visual-question-answering''',
        '''MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING_NAMES''',
        '''AutoModelForVisualQuestionAnswering''',
    ),
    ('''image-to-text''', '''MODEL_FOR_FOR_VISION_2_SEQ_MAPPING_NAMES''', '''AutoModelForVision2Seq'''),
    (
        '''zero-shot-image-classification''',
        '''MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES''',
        '''AutoModelForZeroShotImageClassification''',
    ),
    ('''depth-estimation''', '''MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES''', '''AutoModelForDepthEstimation'''),
    ('''video-classification''', '''MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForVideoClassification'''),
    ('''mask-generation''', '''MODEL_FOR_MASK_GENERATION_MAPPING_NAMES''', '''AutoModelForMaskGeneration'''),
]
def       snake_case_		(     lowerCAmelCase_       :						List[Any]      ):
       __lowercase      :							List[Any]				=     re.finditer(""".+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)"""       ,       lowerCAmelCase_      )
       return [m.group(0      ) for m in matches]
def       snake_case_		(     ):
       __lowercase      :							Dict				=     transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
       __lowercase      :							Dict				=     {
           config.replace("""Config"""       ,       """"""      ): model_type for model_type, config in config_maping_names.items()
       }

       # Dictionaries flagging if each model prefix has a backend in PT/TF/Flax.
       __lowercase      :							str				=     collections.defaultdict(lowerCAmelCase_      )
       __lowercase      :							Optional[Any]				=     collections.defaultdict(lowerCAmelCase_      )
       __lowercase      :							Any				=     collections.defaultdict(lowerCAmelCase_      )

       # Let's lookup through all transformers object (once) and find if models are supported by a given backend.
       for attr_name in dir(lowerCAmelCase_      ):
              __lowercase      :							Any				=     None
              if _re_tf_models.match(lowerCAmelCase_      ) is not None:
                     __lowercase      :							Optional[int]				=     tf_models
                     __lowercase      :							Any				=     _re_tf_models.match(lowerCAmelCase_      ).groups()[0]
              elif _re_flax_models.match(lowerCAmelCase_      ) is not None:
                     __lowercase      :							List[Any]				=     flax_models
                     __lowercase      :							str				=     _re_flax_models.match(lowerCAmelCase_      ).groups()[0]
              elif _re_pt_models.match(lowerCAmelCase_      ) is not None:
                     __lowercase      :							Dict				=     pt_models
                     __lowercase      :							Union[str, Any]				=     _re_pt_models.match(lowerCAmelCase_      ).groups()[0]

              if lookup_dict is not None:
                     while len(lowerCAmelCase_      ) > 0:
                            if attr_name in model_prefix_to_model_type:
                                   __lowercase      :							Union[str, Any]				=     True
                                   break
                            # Try again after removing the last word in the name
                            __lowercase      :							str				=     """""".join(camel_case_split(lowerCAmelCase_      )[:-1]      )

       __lowercase      :							List[Any]				=     set(list(pt_models.keys()      ) + list(tf_models.keys()      ) + list(flax_models.keys()      )      )
       __lowercase      :							Optional[Any]				=     list(lowerCAmelCase_      )
       all_models.sort()

       __lowercase      :							int				=     {"""model_type""": all_models}
       __lowercase      :							Dict				=     [pt_models[t] for t in all_models]
       __lowercase      :							List[Any]				=     [tf_models[t] for t in all_models]
       __lowercase      :							Tuple				=     [flax_models[t] for t in all_models]

       # Now let's use the auto-mapping names to make sure
       __lowercase      :							Any				=     {}
       for t in all_models:
              if t in transformers_module.models.auto.processing_auto.PROCESSOR_MAPPING_NAMES:
                     __lowercase      :							Union[str, Any]				=     """AutoProcessor"""
              elif t in transformers_module.models.auto.tokenization_auto.TOKENIZER_MAPPING_NAMES:
                     __lowercase      :							int				=     """AutoTokenizer"""
              elif t in transformers_module.models.auto.feature_extraction_auto.FEATURE_EXTRACTOR_MAPPING_NAMES:
                     __lowercase      :							List[Any]				=     """AutoFeatureExtractor"""
              else:
                     # Default to AutoTokenizer if a model has nothing, for backward compatibility.
                     __lowercase      :							str				=     """AutoTokenizer"""

       __lowercase      :							str				=     [processors[t] for t in all_models]

       return pd.DataFrame(lowerCAmelCase_      )
def       snake_case_		(     lowerCAmelCase_       :						List[str]      ):
       __lowercase      :							Any				=     [
           transformers_module.models.auto.modeling_auto,
           transformers_module.models.auto.modeling_tf_auto,
           transformers_module.models.auto.modeling_flax_auto,
       ]
       for pipeline_tag, model_mapping, auto_class in PIPELINE_TAGS_AND_AUTO_MODELS:
              __lowercase      :							Union[str, Any]				=     [model_mapping, F"TF_{model_mapping}", F"FLAX_{model_mapping}"]
              __lowercase      :							Optional[Any]				=     [auto_class, F"TF_{auto_class}", F"Flax_{auto_class}"]
              # Loop through all three frameworks
              for module, cls, mapping in zip(lowerCAmelCase_       ,       lowerCAmelCase_       ,       lowerCAmelCase_      ):
                     # The type of pipeline may not exist in this framework
                     if not hasattr(lowerCAmelCase_       ,       lowerCAmelCase_      ):
                            continue
                     # First extract all model_names
                     __lowercase      :							Optional[int]				=     []
                     for name in getattr(lowerCAmelCase_       ,       lowerCAmelCase_      ).values():
                            if isinstance(lowerCAmelCase_       ,       lowerCAmelCase_      ):
                                   model_names.append(lowerCAmelCase_      )
                            else:
                                   model_names.extend(list(lowerCAmelCase_      )      )

            # Add pipeline tag and auto model class for those models
                     table.update({model_name: (pipeline_tag, cls) for model_name in model_names}      )

       return table
def       snake_case_		(     lowerCAmelCase_       :						Optional[Any]       ,       lowerCAmelCase_       :						str      ):
       __lowercase      :							List[str]				=     get_frameworks_table()
       __lowercase      :							Optional[int]				=     Dataset.from_pandas(lowerCAmelCase_      )

       __lowercase      :							Any				=     hf_hub_download(
           """huggingface/transformers-metadata"""       ,       """pipeline_tags.json"""       ,       repo_type="""dataset"""       ,       token=lowerCAmelCase_      )
       __lowercase      :							str				=     Dataset.from_json(lowerCAmelCase_      )
       __lowercase      :							Dict				=     {
           tags_dataset[i]["""model_class"""]: (tags_dataset[i]["""pipeline_tag"""], tags_dataset[i]["""auto_class"""])
           for i in range(len(lowerCAmelCase_      )      )
       }
       __lowercase      :							Optional[Any]				=     update_pipeline_and_auto_class_table(lowerCAmelCase_      )

       # Sort the model classes to avoid some nondeterministic updates to create false update commits.
       __lowercase      :							Optional[int]				=     sorted(table.keys()      )
       __lowercase      :							Union[str, Any]				=     pd.DataFrame(
           {
               """model_class""": model_classes,
               """pipeline_tag""": [table[m][0] for m in model_classes],
               """auto_class""": [table[m][1] for m in model_classes],
           }      )
       __lowercase      :							List[str]				=     Dataset.from_pandas(lowerCAmelCase_      )

       with tempfile.TemporaryDirectory() as tmp_dir:
              frameworks_dataset.to_json(os.path.join(lowerCAmelCase_       ,       """frameworks.json"""      )      )
              tags_dataset.to_json(os.path.join(lowerCAmelCase_       ,       """pipeline_tags.json"""      )      )

              if commit_sha is not None:
                     __lowercase      :							Any				=     (
                         F"Update with commit {commit_sha}\n\nSee: "
                         F"https://github.com/huggingface/transformers/commit/{commit_sha}"
                     )
              else:
                     __lowercase      :							Optional[int]				=     """Update"""

              upload_folder(
                  repo_id="""huggingface/transformers-metadata"""       ,       folder_path=lowerCAmelCase_       ,       repo_type="""dataset"""       ,       token=lowerCAmelCase_       ,       commit_message=lowerCAmelCase_       ,       )
def       snake_case_		(     ):
       __lowercase      :							Dict				=     {tag: cls for tag, _, cls in PIPELINE_TAGS_AND_AUTO_MODELS}
       __lowercase      :							Union[str, Any]				=     transformers_module.pipelines.SUPPORTED_TASKS
       __lowercase      :							List[str]				=     []
       for key in pipeline_tasks:
              if key not in in_table:
                     __lowercase      :							Union[str, Any]				=     pipeline_tasks[key]["""pt"""]
                     if isinstance(lowerCAmelCase_       ,       (list, tuple)      ):
                            __lowercase      :							Tuple				=     model[0]
                     __lowercase      :							str				=     model.__name__
                     if model not in in_table.values():
                            missing.append(lowerCAmelCase_      )

       if len(lowerCAmelCase_      ) > 0:
              __lowercase      :							str				=     """, """.join(lowerCAmelCase_      )
              raise ValueError(
                  """The following pipeline tags are not present in the `PIPELINE_TAGS_AND_AUTO_MODELS` constant inside """
                  F"`utils/update_metadata.py`: {msg}. Please add them!"      )


if __name__ == "__main__":
 lowerCamelCase : Union[str, Any]						   =       argparse.ArgumentParser()
 parser.add_argument('''--token''', type=str, help='''The token to use to push to the transformers-metadata dataset.''')
 parser.add_argument('''--commit_sha''', type=str, help='''The sha of the commit going with this update.''')
 parser.add_argument('''--check-only''', action='''store_true''', help='''Activate to just check all pipelines are present.''')
 lowerCamelCase : Optional[Any]						   =       parser.parse_args()

 if args.check_only:
  check_pipeline_tags()
 else:
  update_metadata(args.token, args.commit_sha)







from .imports import is_tqdm_available


if is_tqdm_available():
	from tqdm.auto import tqdm as _tqdm

from ..state import PartialState
def       snake_case_		(     lowerCAmelCase_       :						bool = True       ,       *lowerCAmelCase_       :						int       ,       **lowerCAmelCase_       :						List[str]      ):
							if not is_tqdm_available():
														raise ImportError("""Accelerate's `tqdm` module requires `tqdm` to be installed. Please run `pip install tqdm`."""      )
							__lowercase      :							List[str]				=     False
							if main_process_only:
														__lowercase      :							Optional[int]				=     PartialState().local_process_index == 0
							return _tqdm(*lowerCAmelCase_       ,       **lowerCAmelCase_       ,       disable=lowerCAmelCase_      )