


'''simple docstring'''





import collections
import importlib.util
import os
import re
from pathlib import Path


__UpperCAmelCase : Any 			=				"src/transformers"


# Matches is_xxx_available()
__UpperCAmelCase : List[Any] 			=				re.compile(r"is\_([a-z_]*)_available()")
# Catches a one-line _import_struct = {xxx}
__UpperCAmelCase : List[str] 			=				re.compile(r"^_import_structure\s+=\s+\{([^\}]+)\}")
# Catches a line with a key-values pattern: "bla": ["foo", "bar"]
__UpperCAmelCase : Any 			=				re.compile(r"\s+\"\S*\":\s+\[([^\]]*)\]")
# Catches a line if not is_foo_available
__UpperCAmelCase : List[str] 			=				re.compile(r"^\s*if\s+not\s+is\_[a-z_]*\_available\(\)")
# Catches a line _import_struct["bla"].append("foo")
__UpperCAmelCase : Tuple 			=				re.compile(r"^\s*_import_structure\[\"\S*\"\]\.append\(\"(\S*)\"\)")
# Catches a line _import_struct["bla"].extend(["foo", "bar"]) or _import_struct["bla"] = ["foo", "bar"]
__UpperCAmelCase : Optional[int] 			=				re.compile(r"^\s*_import_structure\[\S*\](?:\.extend\(|\s*=\s+)\[([^\]]*)\]")
# Catches a line with an object between quotes and a comma:     "MyModel",
__UpperCAmelCase : str 			=				re.compile("^\s+\"([^\"]+)\",")
# Catches a line with objects between brackets only:    ["foo", "bar"],
__UpperCAmelCase : int 			=				re.compile("^\s+\[([^\]]+)\]")
# Catches a line with from foo import bar, bla, boo
__UpperCAmelCase : Any 			=				re.compile(r"\s+from\s+\S*\s+import\s+([^\(\s].*)\n")
# Catches a line with try:
__UpperCAmelCase : List[str] 			=				re.compile(r"^\s*try:")
# Catches a line with else:
__UpperCAmelCase : Any 			=				re.compile(r"^\s*else:")


def      a	(							SCREAMING_SNAKE_CASE_      :		List[str]       ):






       """simple docstring"""





       if _re_test_backend.search(SCREAMING_SNAKE_CASE_       ) is None:
              return None
       UpperCamelCase    :	Tuple   	=		[b[0] for b in _re_backend.findall(SCREAMING_SNAKE_CASE_       )]
       backends.sort()
       return "_and_".join(SCREAMING_SNAKE_CASE_       )


def      a	(							SCREAMING_SNAKE_CASE_      :		Dict       ):






       """simple docstring"""





       with open(SCREAMING_SNAKE_CASE_							,			'''r'''							,			encoding='''utf-8'''							,			newline='''\n'''       ) as f:
              UpperCamelCase    :	Any   	=		f.readlines()

       UpperCamelCase    :	Dict   	=		0
       while line_index < len(SCREAMING_SNAKE_CASE_       ) and not lines[line_index].startswith('''_import_structure = {'''       ):
              line_index += 1

       # If this is a traditional init, just return.
       if line_index >= len(SCREAMING_SNAKE_CASE_       ):
              return None

       # First grab the objects without a specific backend in _import_structure
       UpperCamelCase    :	List[Any]   	=		[]
       while not lines[line_index].startswith('''if TYPE_CHECKING'''       ) and find_backend(lines[line_index]       ) is None:
              UpperCamelCase    :	Any   	=		lines[line_index]
              # If we have everything on a single line, let's deal with it.
              if _re_one_line_import_struct.search(SCREAMING_SNAKE_CASE_       ):
                     UpperCamelCase    :	int   	=		_re_one_line_import_struct.search(SCREAMING_SNAKE_CASE_       ).groups()[0]
                     UpperCamelCase    :	int   	=		re.findall('''\[([^\]]+)\]'''							,			SCREAMING_SNAKE_CASE_       )
                     for imp in imports:
                            objects.extend([obj[1:-1] for obj in imp.split(''', '''       )]       )
                     line_index += 1
                     continue
              UpperCamelCase    :	Union[str, Any]   	=		_re_import_struct_key_value.search(SCREAMING_SNAKE_CASE_       )
              if single_line_import_search is not None:
                     UpperCamelCase    :	str   	=		[obj[1:-1] for obj in single_line_import_search.groups()[0].split(''', '''       ) if len(SCREAMING_SNAKE_CASE_       ) > 0]
                     objects.extend(SCREAMING_SNAKE_CASE_       )
              elif line.startswith(''' ''' * 8 + '''"'''       ):
                     objects.append(line[9:-3]       )
              line_index += 1

       UpperCamelCase    :	int   	=		{'''none''': objects}
       # Let's continue with backend-specific objects in _import_structure
       while not lines[line_index].startswith('''if TYPE_CHECKING'''       ):
              # If the line is an if not is_backend_available, we grab all objects associated.
              UpperCamelCase    :	int   	=		find_backend(lines[line_index]       )
              # Check if the backend declaration is inside a try block:
              if _re_try.search(lines[line_index - 1]       ) is None:
                     UpperCamelCase    :	List[str]   	=		None

              if backend is not None:
                     line_index += 1

                     # Scroll until we hit the else block of try-except-else
                     while _re_else.search(lines[line_index]       ) is None:
                            line_index += 1

                     line_index += 1

                     UpperCamelCase    :	List[str]   	=		[]
                     # Until we unindent, add backend objects to the list
                     while len(lines[line_index]       ) <= 1 or lines[line_index].startswith(''' ''' * 4       ):
                            UpperCamelCase    :	Dict   	=		lines[line_index]
                            if _re_import_struct_add_one.search(SCREAMING_SNAKE_CASE_       ) is not None:
                                   objects.append(_re_import_struct_add_one.search(SCREAMING_SNAKE_CASE_       ).groups()[0]       )
                            elif _re_import_struct_add_many.search(SCREAMING_SNAKE_CASE_       ) is not None:
                                   UpperCamelCase    :	List[Any]   	=		_re_import_struct_add_many.search(SCREAMING_SNAKE_CASE_       ).groups()[0].split(''', '''       )
                                   UpperCamelCase    :	Union[str, Any]   	=		[obj[1:-1] for obj in imports if len(SCREAMING_SNAKE_CASE_       ) > 0]
                                   objects.extend(SCREAMING_SNAKE_CASE_       )
                            elif _re_between_brackets.search(SCREAMING_SNAKE_CASE_       ) is not None:
                                   UpperCamelCase    :	Optional[Any]   	=		_re_between_brackets.search(SCREAMING_SNAKE_CASE_       ).groups()[0].split(''', '''       )
                                   UpperCamelCase    :	List[Any]   	=		[obj[1:-1] for obj in imports if len(SCREAMING_SNAKE_CASE_       ) > 0]
                                   objects.extend(SCREAMING_SNAKE_CASE_       )
                            elif _re_quote_object.search(SCREAMING_SNAKE_CASE_       ) is not None:
                                   objects.append(_re_quote_object.search(SCREAMING_SNAKE_CASE_       ).groups()[0]       )
                            elif line.startswith(''' ''' * 8 + '''"'''       ):
                                   objects.append(line[9:-3]       )
                            elif line.startswith(''' ''' * 1_2 + '''"'''       ):
                                   objects.append(line[1_3:-3]       )
                            line_index += 1

                     UpperCamelCase    :	Any   	=		objects
              else:
                     line_index += 1

    # At this stage we are in the TYPE_CHECKING part, first grab the objects without a specific backend
       UpperCamelCase    :	int   	=		[]
       while (
           line_index < len(SCREAMING_SNAKE_CASE_       )
           and find_backend(lines[line_index]       ) is None
           and not lines[line_index].startswith('''else'''       )
       ):
              UpperCamelCase    :	Union[str, Any]   	=		lines[line_index]
              UpperCamelCase    :	Union[str, Any]   	=		_re_import.search(SCREAMING_SNAKE_CASE_       )
              if single_line_import_search is not None:
                     objects.extend(single_line_import_search.groups()[0].split(''', '''       )       )
              elif line.startswith(''' ''' * 8       ):
                     objects.append(line[8:-2]       )
              line_index += 1

       UpperCamelCase    :	Optional[Any]   	=		{'''none''': objects}
       # Let's continue with backend-specific objects
       while line_index < len(SCREAMING_SNAKE_CASE_       ):
              # If the line is an if is_backend_available, we grab all objects associated.
              UpperCamelCase    :	Dict   	=		find_backend(lines[line_index]       )
              # Check if the backend declaration is inside a try block:
              if _re_try.search(lines[line_index - 1]       ) is None:
                     UpperCamelCase    :	str   	=		None

              if backend is not None:
                     line_index += 1

                     # Scroll until we hit the else block of try-except-else
                     while _re_else.search(lines[line_index]       ) is None:
                            line_index += 1

                     line_index += 1

                     UpperCamelCase    :	Union[str, Any]   	=		[]
                     # Until we unindent, add backend objects to the list
                     while len(lines[line_index]       ) <= 1 or lines[line_index].startswith(''' ''' * 8       ):
                            UpperCamelCase    :	Union[str, Any]   	=		lines[line_index]
                            UpperCamelCase    :	Optional[Any]   	=		_re_import.search(SCREAMING_SNAKE_CASE_       )
                            if single_line_import_search is not None:
                                   objects.extend(single_line_import_search.groups()[0].split(''', '''       )       )
                            elif line.startswith(''' ''' * 1_2       ):
                                   objects.append(line[1_2:-2]       )
                            line_index += 1

                     UpperCamelCase    :	Any   	=		objects
              else:
                     line_index += 1

       return import_dict_objects, type_hint_objects


def      a	(							SCREAMING_SNAKE_CASE_      :		int							,			SCREAMING_SNAKE_CASE_      :		Dict       ):






       """simple docstring"""





       def find_duplicates(SCREAMING_SNAKE_CASE_      :		List[Any]       ):
              return [k for k, v in collections.Counter(SCREAMING_SNAKE_CASE_       ).items() if v > 1]

       if list(import_dict_objects.keys()       ) != list(type_hint_objects.keys()       ):
              return ["Both sides of the init do not have the same backends!"]

       UpperCamelCase    :	Union[str, Any]   	=		[]
       for key in import_dict_objects.keys():
              UpperCamelCase    :	Optional[int]   	=		find_duplicates(import_dict_objects[key]       )
              if duplicate_imports:
                     errors.append(F"""Duplicate _import_structure definitions for: {duplicate_imports}"""       )
              UpperCamelCase    :	List[str]   	=		find_duplicates(type_hint_objects[key]       )
              if duplicate_type_hints:
                     errors.append(F"""Duplicate TYPE_CHECKING objects for: {duplicate_type_hints}"""       )

              if sorted(set(import_dict_objects[key]       )       ) != sorted(set(type_hint_objects[key]       )       ):
                     UpperCamelCase    :	Optional[Any]   	=		'''base imports''' if key == '''none''' else F"""{key} backend"""
                     errors.append(F"""Differences for {name}:"""       )
                     for a in type_hint_objects[key]:
                            if a not in import_dict_objects[key]:
                                   errors.append(F"""  {a} in TYPE_HINT but not in _import_structure."""       )
                     for a in import_dict_objects[key]:
                            if a not in type_hint_objects[key]:
                                   errors.append(F"""  {a} in _import_structure but not in TYPE_HINT."""       )
       return errors


def      a	(							):






       """simple docstring"""





       UpperCamelCase    :	List[Any]   	=		[]
       for root, _, files in os.walk(SCREAMING_SNAKE_CASE_       ):
              if "__init__.py" in files:
                     UpperCamelCase    :	str   	=		os.path.join(SCREAMING_SNAKE_CASE_							,			'''__init__.py'''       )
                     UpperCamelCase    :	List[str]   	=		parse_init(SCREAMING_SNAKE_CASE_       )
                     if objects is not None:
                            UpperCamelCase    :	Union[str, Any]   	=		analyze_results(*SCREAMING_SNAKE_CASE_       )
                            if len(SCREAMING_SNAKE_CASE_       ) > 0:
                                   UpperCamelCase    :	Union[str, Any]   	=		F"""Problem in {fname}, both halves do not define the same objects.\n{errors[0]}"""
                                   failures.append('''\n'''.join(SCREAMING_SNAKE_CASE_       )       )
       if len(SCREAMING_SNAKE_CASE_       ) > 0:
              raise ValueError('''\n\n'''.join(SCREAMING_SNAKE_CASE_       )       )


def      a	(							):






       """simple docstring"""





       UpperCamelCase    :	List[Any]   	=		[]
       for path, directories, files in os.walk(SCREAMING_SNAKE_CASE_       ):
              for folder in directories:
                     # Ignore private modules
                     if folder.startswith('''_'''       ):
                            directories.remove(SCREAMING_SNAKE_CASE_       )
                            continue
                     # Ignore leftovers from branches (empty folders apart from pycache)
                     if len(list((Path(SCREAMING_SNAKE_CASE_       ) / folder).glob('''*.py'''       )       )       ) == 0:
                            continue
                     UpperCamelCase    :	Tuple   	=		str((Path(SCREAMING_SNAKE_CASE_       ) / folder).relative_to(SCREAMING_SNAKE_CASE_       )       )
                     UpperCamelCase    :	Optional[int]   	=		short_path.replace(os.path.sep							,			'''.'''       )
                     submodules.append(SCREAMING_SNAKE_CASE_       )
              for fname in files:
                     if fname == "__init__.py":
                            continue
                     UpperCamelCase    :	List[str]   	=		str((Path(SCREAMING_SNAKE_CASE_       ) / fname).relative_to(SCREAMING_SNAKE_CASE_       )       )
                     UpperCamelCase    :	Union[str, Any]   	=		short_path.replace('''.py'''							,			''''''       ).replace(os.path.sep							,			'''.'''       )
                     if len(submodule.split('''.'''       )       ) == 1:
                            submodules.append(SCREAMING_SNAKE_CASE_       )
       return submodules


__UpperCAmelCase : Union[str, Any] 			=				[
    "convert_pytorch_checkpoint_to_tf2",
    "modeling_flax_pytorch_utils",
]


def      a	(							):






       """simple docstring"""





       UpperCamelCase    :	Optional[Any]   	=		importlib.util.spec_from_file_location(
           '''transformers'''							,			os.path.join(SCREAMING_SNAKE_CASE_							,			'''__init__.py'''       )							,			submodule_search_locations=[PATH_TO_TRANSFORMERS]							,			)
       UpperCamelCase    :	Union[str, Any]   	=		spec.loader.load_module()

       UpperCamelCase    :	Dict   	=		[
           module
           for module in get_transformers_submodules()
           if module not in IGNORE_SUBMODULES and module not in transformers._import_structure.keys()
       ]
       if len(SCREAMING_SNAKE_CASE_       ) > 0:
              UpperCamelCase    :	List[Any]   	=		'''\n'''.join(F"""- {module}""" for module in module_not_registered       )
              raise ValueError(
                  '''The following submodules are not properly registered in the main init of Transformers:\n'''
                  F"""{list_of_modules}\n"""
                  '''Make sure they appear somewhere in the keys of `_import_structure` with an empty list as value.'''       )


if __name__ == "__main__":
  check_all_inits()
  check_submodules()





from .imports import is_tqdm_available


if is_tqdm_available():
  from tqdm.auto import tqdm as _tqdm

from ..state import PartialState


def      a	(							SCREAMING_SNAKE_CASE_      :		bool = True							,			*SCREAMING_SNAKE_CASE_      :		List[str]							,			**SCREAMING_SNAKE_CASE_      :		Tuple       ):






       """simple docstring"""





       if not is_tqdm_available():
              raise ImportError('''Accelerate\'s `tqdm` module requires `tqdm` to be installed. Please run `pip install tqdm`.'''       )
       UpperCamelCase    :	int   	=		False
       if main_process_only:
              UpperCamelCase    :	int   	=		PartialState().local_process_index == 0
       return _tqdm(*SCREAMING_SNAKE_CASE_							,			**SCREAMING_SNAKE_CASE_							,			disable=SCREAMING_SNAKE_CASE_       )

