


'''simple docstring'''







import inspect
import os

import torch
from transformers import AutoModel
from transformers.testing_utils import mockenv_context
from transformers.trainer_utils import set_seed

import accelerate
from accelerate.accelerator import Accelerator
from accelerate.state import AcceleratorState
from accelerate.test_utils.testing import (
    AccelerateTestCase,
    TempDirTestCase,
    execute_subprocess_async,
    require_cuda,
    require_fsdp,
    require_multi_gpu,
    slow,
)
from accelerate.utils.constants import (
    FSDP_AUTO_WRAP_POLICY,
    FSDP_BACKWARD_PREFETCH,
    FSDP_SHARDING_STRATEGY,
    FSDP_STATE_DICT_TYPE,
)
from accelerate.utils.dataclasses import FullyShardedDataParallelPlugin
from accelerate.utils.other import patch_environment


set_seed(42)

lowerCAmelCase    :     List[str]					=						"""bert-base-cased"""
lowerCAmelCase    :     Union[str, Any]					=						"""fp16"""
lowerCAmelCase    :     Dict					=						"""bf16"""
lowerCAmelCase    :     Any					=						[FPaa, BFaa]
@require_fsdp
@require_cuda
class 	UpperCamelCase__					( SCREAMING_SNAKE_CASE_	):



     """simple docstring"""






     def a	( self   ):

            '''simple docstring'''

            super().setUp()

            _lowerCAmelCase    :							Optional[Any]   			=       dict(
                ACCELERATE_USE_FSDP='true'      ,	MASTER_ADDR='localhost'      ,	MASTER_PORT='10999'      ,	RANK='0'      ,	LOCAL_RANK='0'      ,	WORLD_SIZE='1'      ,	)







     def a	( self   ):

            '''simple docstring'''

            from torch.distributed.fsdp.fully_sharded_data_parallel import ShardingStrategy

            for i, strategy in enumerate(snake_case__   ):
                   _lowerCAmelCase    :							Optional[int]   			=       self.dist_env.copy()
                   _lowerCAmelCase    :							Dict   			=       F'{i + 1}'
                   _lowerCAmelCase    :							Optional[Any]   			=       strategy
                   with mockenv_context(**snake_case__   ):
                          _lowerCAmelCase    :							List[Any]   			=       FullyShardedDataParallelPlugin()
                          self.assertEqual(fsdp_plugin.sharding_strategy      ,	ShardingStrategy(i + 1   )   )







     def a	( self   ):

            '''simple docstring'''

            from torch.distributed.fsdp.fully_sharded_data_parallel import BackwardPrefetch

            for i, prefetch_policy in enumerate(snake_case__   ):
                   _lowerCAmelCase    :							Union[str, Any]   			=       self.dist_env.copy()
                   _lowerCAmelCase    :							List[Any]   			=       prefetch_policy
                   with mockenv_context(**snake_case__   ):
                          _lowerCAmelCase    :							Any   			=       FullyShardedDataParallelPlugin()
                          if prefetch_policy == "NO_PREFETCH":
                                 self.assertIsNone(fsdp_plugin.backward_prefetch   )
                          else:
                                 self.assertEqual(fsdp_plugin.backward_prefetch      ,	BackwardPrefetch(i + 1   )   )







     def a	( self   ):

            '''simple docstring'''

            from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType

            for i, state_dict_type in enumerate(snake_case__   ):
                   _lowerCAmelCase    :							Any   			=       self.dist_env.copy()
                   _lowerCAmelCase    :							Any   			=       state_dict_type
                   with mockenv_context(**snake_case__   ):
                          _lowerCAmelCase    :							List[Any]   			=       FullyShardedDataParallelPlugin()
                          self.assertEqual(fsdp_plugin.state_dict_type      ,	StateDictType(i + 1   )   )
                          if state_dict_type == "FULL_STATE_DICT":
                                 self.assertTrue(fsdp_plugin.state_dict_config.offload_to_cpu   )
                                 self.assertTrue(fsdp_plugin.state_dict_config.ranka_only   )







     def a	( self   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Optional[int]   			=       AutoModel.from_pretrained(snake_case__   )
            for policy in FSDP_AUTO_WRAP_POLICY:
                   _lowerCAmelCase    :							str   			=       self.dist_env.copy()
                   _lowerCAmelCase    :							List[Any]   			=       policy
                   if policy == "TRANSFORMER_BASED_WRAP":
                          _lowerCAmelCase    :							List[str]   			=       'BertLayer'
                   elif policy == "SIZE_BASED_WRAP":
                          _lowerCAmelCase    :							Union[str, Any]   			=       '2000'
                   with mockenv_context(**snake_case__   ):
                          _lowerCAmelCase    :							Dict   			=       FullyShardedDataParallelPlugin()
                          fsdp_plugin.set_auto_wrap_policy(snake_case__   )
                          if policy == "NO_WRAP":
                                 self.assertIsNone(fsdp_plugin.auto_wrap_policy   )
                          else:
                                 self.assertIsNotNone(fsdp_plugin.auto_wrap_policy   )

            _lowerCAmelCase    :							int   			=       self.dist_env.copy()
            _lowerCAmelCase    :							Union[str, Any]   			=       'TRANSFORMER_BASED_WRAP'
            _lowerCAmelCase    :							List[str]   			=       'T5Layer'
            with mockenv_context(**snake_case__   ):
                   _lowerCAmelCase    :							Optional[Any]   			=       FullyShardedDataParallelPlugin()
                   with self.assertRaises(snake_case__   ) as cm:
                          fsdp_plugin.set_auto_wrap_policy(snake_case__   )
                   self.assertTrue('Could not find the transformer layer class to wrap in the model.' in str(cm.exception   )   )

            _lowerCAmelCase    :							str   			=       self.dist_env.copy()
            _lowerCAmelCase    :							List[Any]   			=       'SIZE_BASED_WRAP'
            _lowerCAmelCase    :							Tuple   			=       '0'
            with mockenv_context(**snake_case__   ):
                   _lowerCAmelCase    :							Optional[Any]   			=       FullyShardedDataParallelPlugin()
                   fsdp_plugin.set_auto_wrap_policy(snake_case__   )
                   self.assertIsNone(fsdp_plugin.auto_wrap_policy   )







     def a	( self   ):

            '''simple docstring'''

            from torch.distributed.fsdp.fully_sharded_data_parallel import MixedPrecision
            from torch.distributed.fsdp.sharded_grad_scaler import ShardedGradScaler

            for mp_dtype in dtypes:
                   _lowerCAmelCase    :							Tuple   			=       self.dist_env.copy()
                   _lowerCAmelCase    :							List[Any]   			=       mp_dtype
                   with mockenv_context(**snake_case__   ):
                          _lowerCAmelCase    :							Tuple   			=       Accelerator()
                          if mp_dtype == "fp16":
                                 _lowerCAmelCase    :							Dict   			=       torch.floataa
                          elif mp_dtype == "bf16":
                                 _lowerCAmelCase    :							List[str]   			=       torch.bfloataa
                          _lowerCAmelCase    :							Optional[Any]   			=       MixedPrecision(param_dtype=snake_case__      ,	reduce_dtype=snake_case__      ,	buffer_dtype=snake_case__   )
                          self.assertEqual(accelerator.state.fsdp_plugin.mixed_precision_policy      ,	snake_case__   )
                          if mp_dtype == FPaa:
                                 self.assertTrue(isinstance(accelerator.scaler      ,	snake_case__   )   )
                          elif mp_dtype == BFaa:
                                 self.assertIsNone(accelerator.scaler   )
                          AcceleratorState._reset_state(snake_case__   )







     def a	( self   ):

            '''simple docstring'''

            from torch.distributed.fsdp.fully_sharded_data_parallel import CPUOffload

            for flag in [True, False]:
                   _lowerCAmelCase    :							List[Any]   			=       self.dist_env.copy()
                   _lowerCAmelCase    :							Union[str, Any]   			=       str(snake_case__   ).lower()
                   with mockenv_context(**snake_case__   ):
                          _lowerCAmelCase    :							Tuple   			=       FullyShardedDataParallelPlugin()
                          self.assertEqual(fsdp_plugin.cpu_offload      ,	CPUOffload(offload_params=snake_case__   )   )
@require_fsdp
@require_multi_gpu
@slow
class 	UpperCamelCase__					( SCREAMING_SNAKE_CASE_	):



     """simple docstring"""






     def a	( self   ):

            '''simple docstring'''

            super().setUp()
            _lowerCAmelCase    :							Any   			=       0.82
            _lowerCAmelCase    :							Optional[int]   			=       [
                'fsdp_shard_grad_op_transformer_based_wrap',
                'fsdp_full_shard_transformer_based_wrap',
            ]
            _lowerCAmelCase    :							Any   			=       {
                'multi_gpu_fp16': 3200,
                'fsdp_shard_grad_op_transformer_based_wrap_fp16': 2000,
                'fsdp_full_shard_transformer_based_wrap_fp16': 1900,
                # Disabling below test as it overwhelms the RAM memory usage
                # on CI self-hosted runner leading to tests getting killed.
                # "fsdp_full_shard_cpu_offload_transformer_based_wrap_fp32": 1500,  # fp16 was leading to indefinite hang
            }
            _lowerCAmelCase    :							List[Any]   			=       160
            _lowerCAmelCase    :							int   			=       160

            _lowerCAmelCase    :							List[str]   			=       inspect.getfile(accelerate.test_utils   )
            _lowerCAmelCase    :							List[Any]   			=       os.path.sep.join(mod_file.split(os.path.sep   )[:-1] + ['scripts', 'external_deps']   )







     def a	( self   ):

            '''simple docstring'''

            _lowerCAmelCase    :							int   			=       os.path.join(self.test_scripts_folder      ,	'test_performance.py'   )
            _lowerCAmelCase    :							str   			=       ['accelerate', 'launch', '--num_processes=2', '--num_machines=1', '--machine_rank=0', '--use_fsdp']
            for config in self.performance_configs:
                   _lowerCAmelCase    :							Tuple   			=       cmd.copy()
                   for i, strategy in enumerate(snake_case__   ):
                          if strategy.lower() in config:
                                 cmd_config.append(F'--fsdp_sharding_strategy={i+1}'   )
                                 break

                   if "fp32" in config:
                          cmd_config.append('--mixed_precision=no'   )
                   else:
                          cmd_config.append('--mixed_precision=fp16'   )

                   if "cpu_offload" in config:
                          cmd_config.append('--fsdp_offload_params=True'   )

                   for policy in FSDP_AUTO_WRAP_POLICY:
                          if policy.lower() in config:
                                 cmd_config.append(F'--fsdp_auto_wrap_policy={policy}'   )
                                 break

                   if policy == "TRANSFORMER_BASED_WRAP":
                          cmd_config.append('--fsdp_transformer_layer_cls_to_wrap=BertLayer'   )
                   elif policy == "SIZE_BASED_WRAP":
                          cmd_config.append('--fsdp_min_num_params=2000'   )

                   cmd_config.extend(
                       [
                           self.test_file_path,
                           F'--output_dir={self.tmpdir}',
                           F'--performance_lower_bound={self.performance_lower_bound}',
                       ]   )
                   with patch_environment(omp_num_threads=1   ):
                          execute_subprocess_async(snake_case__      ,	env=os.environ.copy()   )







     def a	( self   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Dict   			=       os.path.join(self.test_scripts_folder      ,	'test_checkpointing.py'   )
            _lowerCAmelCase    :							Tuple   			=       [
                'accelerate',
                'launch',
                '--num_processes=2',
                '--num_machines=1',
                '--machine_rank=0',
                '--use_fsdp',
                '--mixed_precision=fp16',
                '--fsdp_transformer_layer_cls_to_wrap=BertLayer',
            ]

            for i, strategy in enumerate(snake_case__   ):
                   _lowerCAmelCase    :							Any   			=       cmd.copy()
                   cmd_config.append(F'--fsdp_sharding_strategy={i+1}'   )
                   if strategy != "FULL_SHARD":
                          continue
                   _lowerCAmelCase    :							str   			=       len(snake_case__   )
                   for state_dict_type in FSDP_STATE_DICT_TYPE:
                          _lowerCAmelCase    :							List[str]   			=       cmd_config[:state_dict_config_index]
                          cmd_config.append(F'--fsdp_state_dict_type={state_dict_type}'   )
                          cmd_config.extend(
                              [
                                  self.test_file_path,
                                  F'--output_dir={self.tmpdir}',
                                  '--partial_train_epoch=1',
                              ]   )
                          with patch_environment(omp_num_threads=1   ):
                                 execute_subprocess_async(snake_case__      ,	env=os.environ.copy()   )

                          _lowerCAmelCase    :							List[str]   			=       cmd_config[:-1]
                          _lowerCAmelCase    :							List[str]   			=       os.path.join(self.tmpdir      ,	'epoch_0'   )
                          cmd_config.extend(
                              [
                                  F'--resume_from_checkpoint={resume_from_checkpoint}',
                              ]   )
                          with patch_environment(omp_num_threads=1   ):
                                 execute_subprocess_async(snake_case__      ,	env=os.environ.copy()   )







     def a	( self   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Optional[int]   			=       os.path.join(self.test_scripts_folder      ,	'test_peak_memory_usage.py'   )
            _lowerCAmelCase    :							List[Any]   			=       [
                'accelerate',
                'launch',
                '--num_processes=2',
                '--num_machines=1',
                '--machine_rank=0',
            ]
            for spec, peak_mem_upper_bound in self.peak_memory_usage_upper_bound.items():
                   _lowerCAmelCase    :							Dict   			=       cmd.copy()
                   if "fp16" in spec:
                          cmd_config.extend(['--mixed_precision=fp16']   )
                   else:
                          cmd_config.extend(['--mixed_precision=no']   )

                   if "multi_gpu" in spec:
                          continue
                   else:
                          cmd_config.extend(['--use_fsdp']   )
                          for i, strategy in enumerate(snake_case__   ):
                                 if strategy.lower() in spec:
                                        cmd_config.append(F'--fsdp_sharding_strategy={i+1}'   )
                                        break

                          if "cpu_offload" in spec:
                                 cmd_config.append('--fsdp_offload_params=True'   )

                          for policy in FSDP_AUTO_WRAP_POLICY:
                                 if policy.lower() in spec:
                                        cmd_config.append(F'--fsdp_auto_wrap_policy={policy}'   )
                                        break

                          if policy == "TRANSFORMER_BASED_WRAP":
                                 cmd_config.append('--fsdp_transformer_layer_cls_to_wrap=BertLayer'   )
                          elif policy == "SIZE_BASED_WRAP":
                                 cmd_config.append('--fsdp_min_num_params=2000'   )

                   cmd_config.extend(
                       [
                           self.test_file_path,
                           F'--output_dir={self.tmpdir}',
                           F'--peak_memory_upper_bound={peak_mem_upper_bound}',
                           F'--n_train={self.n_train}',
                           F'--n_val={self.n_val}',
                       ]   )
                   with patch_environment(omp_num_threads=1   ):
                          execute_subprocess_async(snake_case__      ,	env=os.environ.copy()   )





'''simple docstring'''







from typing import Any






def 			lowercase				(_A					):







       """simple docstring"""







       if not input_list:
              return []
       _lowerCAmelCase    :							Optional[int]   			=       [input_list.count(_A					) for value in input_list]
       _lowerCAmelCase    :							int   			=       max(_A					)  # Gets the maximum count in the input list.
       # Gets values of modes
       return sorted({input_list[i] for i, value in enumerate(_A					) if value == y}					)


if __name__ == "__main__":
       import doctest

       doctest.testmod()

