




'''simple docstring'''





import importlib
import json
import os
from collections import OrderedDict
from typing import Dict, Optional, Union

# Build the list of all feature extractors
from ...configuration_utils import PretrainedConfig
from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
from ...feature_extraction_utils import FeatureExtractionMixin
from ...utils import CONFIG_NAME, FEATURE_EXTRACTOR_NAME, get_file_from_repo, logging
from .auto_factory import _LazyAutoMapping
from .configuration_auto import (
    CONFIG_MAPPING_NAMES,
    AutoConfig,
    model_type_to_module_name,
    replace_list_option_in_docstrings,
)


lowerCamelCase       :	Union[str, Any]									=     logging.get_logger(__name__)

lowerCamelCase       :	str									=     OrderedDict(
    [
        ("audio-spectrogram-transformer", "ASTFeatureExtractor"),
        ("beit", "BeitFeatureExtractor"),
        ("chinese_clip", "ChineseCLIPFeatureExtractor"),
        ("clap", "ClapFeatureExtractor"),
        ("clip", "CLIPFeatureExtractor"),
        ("clipseg", "ViTFeatureExtractor"),
        ("conditional_detr", "ConditionalDetrFeatureExtractor"),
        ("convnext", "ConvNextFeatureExtractor"),
        ("cvt", "ConvNextFeatureExtractor"),
        ("data2vec-audio", "Wav2Vec2FeatureExtractor"),
        ("data2vec-vision", "BeitFeatureExtractor"),
        ("deformable_detr", "DeformableDetrFeatureExtractor"),
        ("deit", "DeiTFeatureExtractor"),
        ("detr", "DetrFeatureExtractor"),
        ("dinat", "ViTFeatureExtractor"),
        ("donut-swin", "DonutFeatureExtractor"),
        ("dpt", "DPTFeatureExtractor"),
        ("encodec", "EncodecFeatureExtractor"),
        ("flava", "FlavaFeatureExtractor"),
        ("glpn", "GLPNFeatureExtractor"),
        ("groupvit", "CLIPFeatureExtractor"),
        ("hubert", "Wav2Vec2FeatureExtractor"),
        ("imagegpt", "ImageGPTFeatureExtractor"),
        ("layoutlmv2", "LayoutLMv2FeatureExtractor"),
        ("layoutlmv3", "LayoutLMv3FeatureExtractor"),
        ("levit", "LevitFeatureExtractor"),
        ("maskformer", "MaskFormerFeatureExtractor"),
        ("mctct", "MCTCTFeatureExtractor"),
        ("mobilenet_v1", "MobileNetV1FeatureExtractor"),
        ("mobilenet_v2", "MobileNetV2FeatureExtractor"),
        ("mobilevit", "MobileViTFeatureExtractor"),
        ("nat", "ViTFeatureExtractor"),
        ("owlvit", "OwlViTFeatureExtractor"),
        ("perceiver", "PerceiverFeatureExtractor"),
        ("poolformer", "PoolFormerFeatureExtractor"),
        ("regnet", "ConvNextFeatureExtractor"),
        ("resnet", "ConvNextFeatureExtractor"),
        ("segformer", "SegformerFeatureExtractor"),
        ("sew", "Wav2Vec2FeatureExtractor"),
        ("sew-d", "Wav2Vec2FeatureExtractor"),
        ("speech_to_text", "Speech2TextFeatureExtractor"),
        ("speecht5", "SpeechT5FeatureExtractor"),
        ("swiftformer", "ViTFeatureExtractor"),
        ("swin", "ViTFeatureExtractor"),
        ("swinv2", "ViTFeatureExtractor"),
        ("table-transformer", "DetrFeatureExtractor"),
        ("timesformer", "VideoMAEFeatureExtractor"),
        ("tvlt", "TvltFeatureExtractor"),
        ("unispeech", "Wav2Vec2FeatureExtractor"),
        ("unispeech-sat", "Wav2Vec2FeatureExtractor"),
        ("van", "ConvNextFeatureExtractor"),
        ("videomae", "VideoMAEFeatureExtractor"),
        ("vilt", "ViltFeatureExtractor"),
        ("vit", "ViTFeatureExtractor"),
        ("vit_mae", "ViTFeatureExtractor"),
        ("vit_msn", "ViTFeatureExtractor"),
        ("wav2vec2", "Wav2Vec2FeatureExtractor"),
        ("wav2vec2-conformer", "Wav2Vec2FeatureExtractor"),
        ("wavlm", "Wav2Vec2FeatureExtractor"),
        ("whisper", "WhisperFeatureExtractor"),
        ("xclip", "CLIPFeatureExtractor"),
        ("yolos", "YolosFeatureExtractor"),
    ]
)

lowerCamelCase       :	Any									=     _LazyAutoMapping(CONFIG_MAPPING_NAMES, FEATURE_EXTRACTOR_MAPPING_NAMES)

def     _SCREAMING_SNAKE_CASE    (							lowercase			: str  ):


      '''simple docstring'''



      for module_name, extractors in FEATURE_EXTRACTOR_MAPPING_NAMES.items():
            if class_name in extractors:
                  lowerCamelCase_	  =					model_type_to_module_name(lowercase  )

                  lowerCamelCase_	  =					importlib.import_module(f""".{module_name}"""			,							'transformers.models'  )
                  try:
                        return getattr(lowercase			,							lowercase  )
                  except AttributeError:
                        continue

      for _, extractor in FEATURE_EXTRACTOR_MAPPING._extra_content.items():
            if getattr(lowercase			,							'__name__'			,							lowercase  ) == class_name:
                  return extractor

    # We did not fine the class, but maybe it's because a dep is missing. In that case, the class will be in the main
    # init and we return the proper dummy to get an appropriate error message.
      lowerCamelCase_	  =					importlib.import_module('transformers'  )
      if hasattr(lowercase			,							lowercase  ):
            return getattr(lowercase			,							lowercase  )

      return None

def     _SCREAMING_SNAKE_CASE    (							lowercase			: Union[str, os.PathLike]			,							lowercase			: Optional[Union[str, os.PathLike]] = None			,							lowercase			: bool = False			,							lowercase			: bool = False			,							lowercase			: Optional[Dict[str, str]] = None			,							lowercase			: Optional[Union[bool, str]] = None			,							lowercase			: Optional[str] = None			,							lowercase			: bool = False			,							**lowercase			: Any			,							):


      '''simple docstring'''



      lowerCamelCase_	  =					get_file_from_repo(
          lowercase			,							lowercase			,							cache_dir=lowercase			,							force_download=lowercase			,							resume_download=lowercase			,							proxies=lowercase			,							use_auth_token=lowercase			,							revision=lowercase			,							local_files_only=lowercase			,							)
      if resolved_config_file is None:
            logger.info(
                'Could not locate the feature extractor configuration file, will try to use the model config instead.'  )
            return {}

      with open(lowercase			,							encoding='utf-8'  ) as reader:
            return json.load(lowercase  )







class 						A:



 '''simple docstring'''


 def __init__(       self      :						Union[str, Any]						)     ->			Optional[Any]:

       """simple docstring"""






       raise EnvironmentError(
           'AutoFeatureExtractor is designed to be instantiated '
           'using the `AutoFeatureExtractor.from_pretrained(pretrained_model_name_or_path)` method.'						)

 @classmethod
 @replace_list_option_in_docstrings(A_						)
 def        a__				(       cls      :						Union[str, Any]		,							A_      :						List[str]		,							**A_      :						List[str]						)     ->			int:

       """simple docstring"""






       lowerCamelCase_	  =					kwargs.pop('config'		,							A_						)
       lowerCamelCase_	  =					kwargs.pop('trust_remote_code'		,							A_						)
       lowerCamelCase_	  =					True

       lowerCamelCase_  ,       lowerCamelCase_	  =					FeatureExtractionMixin.get_feature_extractor_dict(A_		,							**A_						)
       lowerCamelCase_	  =					config_dict.get('feature_extractor_type'		,							A_						)
       lowerCamelCase_	  =					None
       if "AutoFeatureExtractor" in config_dict.get('auto_map'		,							{}						):
             lowerCamelCase_	  =					config_dict['auto_map']['AutoFeatureExtractor']

       # If we don't find the feature extractor class in the feature extractor config, let's try the model config.
       if feature_extractor_class is None and feature_extractor_auto_map is None:
             if not isinstance(A_		,							A_						):
                   lowerCamelCase_	  =					AutoConfig.from_pretrained(A_		,							**A_						)
             # It could be in `config.feature_extractor_type``
             lowerCamelCase_	  =					getattr(A_		,							'feature_extractor_type'		,							A_						)
             if hasattr(A_		,							'auto_map'						) and "AutoFeatureExtractor" in config.auto_map:
                   lowerCamelCase_	  =					config.auto_map['AutoFeatureExtractor']

       if feature_extractor_class is not None:
             lowerCamelCase_	  =					feature_extractor_class_from_name(A_						)

       lowerCamelCase_	  =					feature_extractor_auto_map is not None
       lowerCamelCase_	  =					feature_extractor_class is not None or type(A_						) in FEATURE_EXTRACTOR_MAPPING
       lowerCamelCase_	  =					resolve_trust_remote_code(
           A_		,							A_		,							A_		,							A_						)

       if has_remote_code and trust_remote_code:
             lowerCamelCase_	  =					get_class_from_dynamic_module(
                 A_		,							A_		,							**A_						)
             lowerCamelCase_	  =					kwargs.pop('code_revision'		,							A_						)
             if os.path.isdir(A_						):
                   feature_extractor_class.register_for_auto_class()
             return feature_extractor_class.from_dict(A_		,							**A_						)
       elif feature_extractor_class is not None:
             return feature_extractor_class.from_dict(A_		,							**A_						)
       # Last try: we use the FEATURE_EXTRACTOR_MAPPING.
       elif type(A_						) in FEATURE_EXTRACTOR_MAPPING:
             lowerCamelCase_	  =					FEATURE_EXTRACTOR_MAPPING[type(A_						)]
             return feature_extractor_class.from_dict(A_		,							**A_						)

       raise ValueError(
           f"""Unrecognized feature extractor in {pretrained_model_name_or_path}. Should have a """
           f"""`feature_extractor_type` key in its {FEATURE_EXTRACTOR_NAME} of {CONFIG_NAME}, or one of the following """
           f"""`model_type` keys in its {CONFIG_NAME}: {", ".join(c for c in FEATURE_EXTRACTOR_MAPPING_NAMES.keys()						)}"""						)





 @staticmethod
 def        a__				(       A_      :						List[Any]		,							A_      :						Optional[int]						)     ->			Any:

       """simple docstring"""






       FEATURE_EXTRACTOR_MAPPING.register(A_		,							A_						)









import argparse
import json
import logging
import os
import sys
from unittest.mock import patch

from transformers.testing_utils import TestCasePlus, get_gpu_count, slow


lowerCamelCase       :	List[Any]									=     [
    os.path.join(os.path.dirname(__file__), dirname)
    for dirname in [
        "text-classification",
        "language-modeling",
        "summarization",
        "token-classification",
        "question-answering",
    ]
]
sys.path.extend(SRC_DIRS)


if SRC_DIRS is not None:
 import run_clm_flax
 import run_flax_glue
 import run_flax_ner
 import run_mlm_flax
 import run_qa
 import run_summarization_flax
 import run_ta_mlm_flax


logging.basicConfig(level=logging.DEBUG)

lowerCamelCase       :	Tuple									=     logging.getLogger()

def     _SCREAMING_SNAKE_CASE    (							):


      '''simple docstring'''



      lowerCamelCase_	  =					argparse.ArgumentParser()
      parser.add_argument('-f'  )
      lowerCamelCase_	  =					parser.parse_args()
      return args.f

def     _SCREAMING_SNAKE_CASE    (							lowercase			: Optional[Any]			,							lowercase			: Dict="eval"  ):


      '''simple docstring'''



      lowerCamelCase_	  =					os.path.join(lowercase			,							f"""{split}_results.json"""  )
      if os.path.exists(lowercase  ):
            with open(lowercase			,							'r'  ) as f:
                  return json.load(lowercase  )
      raise ValueError(f"""can't find {path}"""  )


lowerCamelCase       :	str									=     logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)







class 						A(		UpperCamelCase     ):



 '''simple docstring'''


 def        a__				(       self      :						Union[str, Any]						)     ->			List[str]:

       """simple docstring"""






       lowerCamelCase_	  =					self.get_auto_remove_tmp_dir()
       lowerCamelCase_	  =					f"""
            run_glue.py
            --model_name_or_path distilbert-base-uncased
            --output_dir {tmp_dir}
            --train_file ./tests/fixtures/tests_samples/MRPC/train.csv
            --validation_file ./tests/fixtures/tests_samples/MRPC/dev.csv
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --learning_rate=1e-4
            --eval_steps=2
            --warmup_steps=2
            --seed=42
            --max_seq_length=128
            """.split()

       with patch.object(A_		,							'argv'		,							A_						):
             run_flax_glue.main()
             lowerCamelCase_	  =					get_results(A_						)
             self.assertGreaterEqual(result['eval_accuracy']		,							0.75						)

 @slow
 def        a__				(       self      :						List[str]						)     ->			Union[str, Any]:

       """simple docstring"""






       lowerCamelCase_	  =					self.get_auto_remove_tmp_dir()
       lowerCamelCase_	  =					f"""
            run_clm_flax.py
            --model_name_or_path distilgpt2
            --train_file ./tests/fixtures/sample_text.txt
            --validation_file ./tests/fixtures/sample_text.txt
            --do_train
            --do_eval
            --block_size 128
            --per_device_train_batch_size 4
            --per_device_eval_batch_size 4
            --num_train_epochs 2
            --logging_steps 2 --eval_steps 2
            --output_dir {tmp_dir}
            --overwrite_output_dir
            """.split()

       with patch.object(A_		,							'argv'		,							A_						):
             run_clm_flax.main()
             lowerCamelCase_	  =					get_results(A_						)
             self.assertLess(result['eval_perplexity']		,							100						)

 @slow
 def        a__				(       self      :						str						)     ->			Tuple:

       """simple docstring"""






       lowerCamelCase_	  =					self.get_auto_remove_tmp_dir()
       lowerCamelCase_	  =					f"""
            run_summarization.py
            --model_name_or_path t5-small
            --train_file tests/fixtures/tests_samples/xsum/sample.json
            --validation_file tests/fixtures/tests_samples/xsum/sample.json
            --test_file tests/fixtures/tests_samples/xsum/sample.json
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --num_train_epochs=3
            --warmup_steps=8
            --do_train
            --do_eval
            --do_predict
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --predict_with_generate
        """.split()

       with patch.object(A_		,							'argv'		,							A_						):
             run_summarization_flax.main()
             lowerCamelCase_	  =					get_results(A_		,							split='test'						)
             self.assertGreaterEqual(result['test_rouge1']		,							10						)
             self.assertGreaterEqual(result['test_rouge2']		,							2						)
             self.assertGreaterEqual(result['test_rougeL']		,							7						)
             self.assertGreaterEqual(result['test_rougeLsum']		,							7						)

 @slow
 def        a__				(       self      :						Optional[int]						)     ->			str:

       """simple docstring"""






       lowerCamelCase_	  =					self.get_auto_remove_tmp_dir()
       lowerCamelCase_	  =					f"""
            run_mlm.py
            --model_name_or_path distilroberta-base
            --train_file ./tests/fixtures/sample_text.txt
            --validation_file ./tests/fixtures/sample_text.txt
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --max_seq_length 128
            --per_device_train_batch_size 4
            --per_device_eval_batch_size 4
            --logging_steps 2 --eval_steps 2
            --do_train
            --do_eval
            --num_train_epochs=1
        """.split()

       with patch.object(A_		,							'argv'		,							A_						):
             run_mlm_flax.main()
             lowerCamelCase_	  =					get_results(A_						)
             self.assertLess(result['eval_perplexity']		,							42						)

 @slow
 def        a__				(       self      :						Optional[Any]						)     ->			Union[str, Any]:

       """simple docstring"""






       lowerCamelCase_	  =					self.get_auto_remove_tmp_dir()
       lowerCamelCase_	  =					f"""
            run_t5_mlm_flax.py
            --model_name_or_path t5-small
            --train_file ./tests/fixtures/sample_text.txt
            --validation_file ./tests/fixtures/sample_text.txt
            --do_train
            --do_eval
            --max_seq_length 128
            --per_device_train_batch_size 4
            --per_device_eval_batch_size 4
            --num_train_epochs 2
            --logging_steps 2 --eval_steps 2
            --output_dir {tmp_dir}
            --overwrite_output_dir
            """.split()

       with patch.object(A_		,							'argv'		,							A_						):
             run_ta_mlm_flax.main()
             lowerCamelCase_	  =					get_results(A_						)
             self.assertGreaterEqual(result['eval_accuracy']		,							0.42						)

 @slow
 def        a__				(       self      :						int						)     ->			Tuple:

       """simple docstring"""






       lowerCamelCase_	  =					7 if get_gpu_count() > 1 else 2

       lowerCamelCase_	  =					self.get_auto_remove_tmp_dir()
       lowerCamelCase_	  =					f"""
            run_flax_ner.py
            --model_name_or_path bert-base-uncased
            --train_file tests/fixtures/tests_samples/conll/sample.json
            --validation_file tests/fixtures/tests_samples/conll/sample.json
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --do_train
            --do_eval
            --warmup_steps=2
            --learning_rate=2e-4
            --logging_steps 2 --eval_steps 2
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=2
            --num_train_epochs={epochs}
            --seed 7
        """.split()

       with patch.object(A_		,							'argv'		,							A_						):
             run_flax_ner.main()
             lowerCamelCase_	  =					get_results(A_						)
             self.assertGreaterEqual(result['eval_accuracy']		,							0.75						)
             self.assertGreaterEqual(result['eval_f1']		,							0.3						)





 @slow
 def        a__				(       self      :						str						)     ->			int:

       """simple docstring"""






       lowerCamelCase_	  =					self.get_auto_remove_tmp_dir()
       lowerCamelCase_	  =					f"""
            run_qa.py
            --model_name_or_path bert-base-uncased
            --version_2_with_negative
            --train_file tests/fixtures/tests_samples/SQUAD/sample.json
            --validation_file tests/fixtures/tests_samples/SQUAD/sample.json
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --num_train_epochs=3
            --warmup_steps=2
            --do_train
            --do_eval
            --logging_steps 2 --eval_steps 2
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
        """.split()

       with patch.object(A_		,							'argv'		,							A_						):
             run_qa.main()
             lowerCamelCase_	  =					get_results(A_						)
             self.assertGreaterEqual(result['eval_f1']		,							30						)
             self.assertGreaterEqual(result['eval_exact']		,							30						)



