



import contextlib
from multiprocessing import Pool, RLock

from tqdm.auto import tqdm

from ..utils import experimental, logging


UpperCAmelCase_									=							logging.get_logger(__name__)





class 		UpperCAmelCase						:

   '''simple docstring'''





   SCREAMING_SNAKE_CASE__							=     None






@experimental
def 							UpperCAmelCase__		(				_SCREAMING_SNAKE_CASE   :	List[str]     ,				_SCREAMING_SNAKE_CASE   :	Dict     ,				_SCREAMING_SNAKE_CASE   :	Union[str, Any]     ,				_SCREAMING_SNAKE_CASE   :	Dict     ,				_SCREAMING_SNAKE_CASE   :	List[Any]     ,				_SCREAMING_SNAKE_CASE   :	Any     ,				_SCREAMING_SNAKE_CASE   :	Tuple				)->Tuple:







      if ParallelBackendConfig.backend_name is None:
            return _map_with_multiprocessing_pool(
                __snake_case     ,				__snake_case     ,				__snake_case     ,				__snake_case     ,				__snake_case     ,				__snake_case     ,				__snake_case				)

      return _map_with_joblib(__snake_case     ,				__snake_case     ,				__snake_case     ,				__snake_case     ,				__snake_case     ,				__snake_case     ,				__snake_case				)
def 							UpperCAmelCase__		(				_SCREAMING_SNAKE_CASE   :	List[str]     ,				_SCREAMING_SNAKE_CASE   :	List[str]     ,				_SCREAMING_SNAKE_CASE   :	List[Any]     ,				_SCREAMING_SNAKE_CASE   :	Tuple     ,				_SCREAMING_SNAKE_CASE   :	Tuple     ,				_SCREAMING_SNAKE_CASE   :	str     ,				_SCREAMING_SNAKE_CASE   :	List[Any]				)->int:
      _lowerCAmelCase 			=  num_proc if num_proc <= len(__snake_case				) else len(__snake_case				)
      _lowerCAmelCase 			=  []  # We organize the splits ourselve (contiguous splits)
      for index in range(__snake_case				):
            _lowerCAmelCase 			=  len(__snake_case				) // num_proc
            _lowerCAmelCase 			=  len(__snake_case				) % num_proc
            _lowerCAmelCase 			=  div * index + min(__snake_case     ,				__snake_case				)
            _lowerCAmelCase 			=  start + div + (1 if index < mod else 0)
            split_kwds.append((function, iterable[start:end], types, index, disable_tqdm, desc)				)

      if len(__snake_case				) != sum(len(i[1]				) for i in split_kwds				):
            raise ValueError(
                f'''Error dividing inputs iterable among processes. '''
                f'''Total number of objects {len(__snake_case				)}, '''
                f'''length: {sum(len(i[1]				) for i in split_kwds				)}'''				)

      logger.info(
          f'''Spawning {num_proc} processes for {len(__snake_case				)} objects in slices of {[len(i[1]				) for i in split_kwds]}'''				)
      _lowerCAmelCase			,   _lowerCAmelCase 			=  None, None
      if not disable_tqdm:
            _lowerCAmelCase			,   _lowerCAmelCase 			=  (RLock(),), tqdm.set_lock
      with Pool(__snake_case     ,				initargs=__snake_case     ,				initializer=__snake_case				) as pool:
            _lowerCAmelCase 			=  pool.map(__snake_case     ,				__snake_case				)
      logger.info(f'''Finished {num_proc} processes'''				)
      _lowerCAmelCase 			=  [obj for proc_res in mapped for obj in proc_res]
      logger.info(f'''Unpacked {len(__snake_case				)} objects'''				)

      return mapped
def 							UpperCAmelCase__		(				_SCREAMING_SNAKE_CASE   :	List[Any]     ,				_SCREAMING_SNAKE_CASE   :	str     ,				_SCREAMING_SNAKE_CASE   :	List[str]     ,				_SCREAMING_SNAKE_CASE   :	int     ,				_SCREAMING_SNAKE_CASE   :	Tuple     ,				_SCREAMING_SNAKE_CASE   :	Dict     ,				_SCREAMING_SNAKE_CASE   :	int				)->List[str]:
      import joblib

      with joblib.parallel_backend(ParallelBackendConfig.backend_name     ,				n_jobs=__snake_case				):
            return joblib.Parallel()(
                joblib.delayed(__snake_case				)((function, obj, types, None, True, None)				) for obj in iterable				)


@experimental
@contextlib.contextmanager
def 							UpperCAmelCase__		(				_SCREAMING_SNAKE_CASE   :	Tuple				)->int:
      _lowerCAmelCase 			=  backend_name

      if backend_name == "spark":
            from joblibspark import register_spark

            register_spark()

            # TODO: call create_cache_and_write_probe if "download" in steps
            # TODO: raise NotImplementedError when Dataset.map etc is called

      try:
            yield
      finally:
            _lowerCAmelCase 			=  None




# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os

from accelerate.utils import ComputeEnvironment

from .cluster import get_cluster_input
from .config_args import cache_dir, default_config_file, default_yaml_config_file, load_config_from_file  # noqa: F401
from .config_utils import _ask_field, _ask_options, _convert_compute_environment  # noqa: F401
from .sagemaker import get_sagemaker_input


UpperCAmelCase_									=							"Launches a series of prompts to create and save a `default_config.yaml` configuration file for your training system. Should always be ran first on your machine"
def 							UpperCAmelCase__		(				)->Any:
      _lowerCAmelCase 			=  _ask_options(
          '''In which compute environment are you running?'''     ,				['''This machine''', '''AWS (Amazon SageMaker)''']     ,				_convert_compute_environment     ,				)
      if compute_environment == ComputeEnvironment.AMAZON_SAGEMAKER:
            _lowerCAmelCase 			=  get_sagemaker_input()
      else:
            _lowerCAmelCase 			=  get_cluster_input()
      return config
def 							UpperCAmelCase__		(				_SCREAMING_SNAKE_CASE   :	int=None				)->str:
      if subparsers is not None:
            _lowerCAmelCase 			=  subparsers.add_parser('''config'''     ,				description=_SCREAMING_SNAKE_CASE				)
      else:
            _lowerCAmelCase 			=  argparse.ArgumentParser('''Accelerate config command'''     ,				description=_SCREAMING_SNAKE_CASE				)

      parser.add_argument(
          '''--config_file'''     ,				default=_SCREAMING_SNAKE_CASE     ,				help=(
              '''The path to use to store the config file. Will default to a file named default_config.yaml in the cache '''
              '''location, which is the content of the environment `HF_HOME` suffixed with \'accelerate\', or if you don\'t have '''
              '''such an environment variable, your cache directory (\'~/.cache\' or the content of `XDG_CACHE_HOME`) suffixed '''
              '''with \'huggingface\'.'''
          )     ,				)

      if subparsers is not None:
            parser.set_defaults(func=_SCREAMING_SNAKE_CASE				)
      return parser
def 							UpperCAmelCase__		(				_SCREAMING_SNAKE_CASE   :	Dict				)->str:
      _lowerCAmelCase 			=  get_user_input()
      if args.config_file is not None:
            _lowerCAmelCase 			=  args.config_file
      else:
            if not os.path.isdir(_SCREAMING_SNAKE_CASE				):
                  os.makedirs(_SCREAMING_SNAKE_CASE				)
            _lowerCAmelCase 			=  default_yaml_config_file

      if config_file.endswith('''.json'''				):
            config.to_json_file(_SCREAMING_SNAKE_CASE				)
      else:
            config.to_yaml_file(_SCREAMING_SNAKE_CASE				)
      print(f'''accelerate configuration saved at {config_file}'''				)
def 							UpperCAmelCase__		(				)->List[Any]:
      _lowerCAmelCase 			=  config_command_parser()
      _lowerCAmelCase 			=  parser.parse_args()
      config_command(_SCREAMING_SNAKE_CASE				)


if __name__ == "__main__":
    main()