# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import torch

from accelerate import PartialState
from accelerate.utils.operations import broadcast, gather, gather_object, pad_across_processes, reduce







def     _a				(      lowercase__  :			Any ):







     '''simple docstring'''


     return (torch.arange(state.num_processes ) + 1.0 + (state.num_processes * state.process_index)).to(state.device )







def     _a				(      lowercase__  :			Tuple ):







     '''simple docstring'''


     SCREAMING_SNAKE_CASE__				:				int			=		create_tensor(lowercase__ )
     SCREAMING_SNAKE_CASE__				:				Optional[Any]			=		gather(lowercase__ )
     assert gathered_tensor.tolist() == list(range(1     ,  state.num_processes**2 + 1 ) )







def     _a				(      lowercase__  :			List[Any] ):







     '''simple docstring'''


     SCREAMING_SNAKE_CASE__				:				str			=		[state.process_index]
     SCREAMING_SNAKE_CASE__				:				Any			=		gather_object(lowercase__ )
     assert len(lowercase__ ) == state.num_processes, f'''{gathered_obj}, {len(lowercase__ )} != {state.num_processes}'''
     assert gathered_obj == list(range(state.num_processes ) ), f'''{gathered_obj} != {list(range(state.num_processes ) )}'''







def     _a				(      lowercase__  :			str ):







     '''simple docstring'''


     SCREAMING_SNAKE_CASE__				:				Optional[Any]			=		create_tensor(lowercase__ )
     SCREAMING_SNAKE_CASE__				:				Optional[Any]			=		broadcast(lowercase__ )
     assert broadcasted_tensor.shape == torch.Size([state.num_processes] )
     assert broadcasted_tensor.tolist() == list(range(1     ,  state.num_processes + 1 ) )







def     _a				(      lowercase__  :			int ):







     '''simple docstring'''


     if state.is_main_process:
          SCREAMING_SNAKE_CASE__				:				Optional[int]			=		torch.arange(state.num_processes + 1 ).to(state.device )
     else:
          SCREAMING_SNAKE_CASE__				:				List[Any]			=		torch.arange(state.num_processes ).to(state.device )
     SCREAMING_SNAKE_CASE__				:				Any			=		pad_across_processes(lowercase__ )
     assert padded_tensor.shape == torch.Size([state.num_processes + 1] )
     if not state.is_main_process:
          assert padded_tensor.tolist() == list(range(0     ,  state.num_processes ) ) + [0]







def     _a				(      lowercase__  :			Optional[Any] ):







     '''simple docstring'''


     if state.num_processes != 2:
          return
     SCREAMING_SNAKE_CASE__				:				List[Any]			=		create_tensor(lowercase__ )
     SCREAMING_SNAKE_CASE__				:				str			=		reduce(lowercase__     ,  'sum' )
     SCREAMING_SNAKE_CASE__				:				Union[str, Any]			=		torch.tensor([4.0, 6] ).to(state.device )
     assert torch.allclose(lowercase__     ,  lowercase__ ), f'''{reduced_tensor} != {truth_tensor}'''







def     _a				(      lowercase__  :			int ):







     '''simple docstring'''


     if state.num_processes != 2:
          return
     SCREAMING_SNAKE_CASE__				:				Any			=		create_tensor(lowercase__ )
     SCREAMING_SNAKE_CASE__				:				List[Any]			=		reduce(lowercase__     ,  'mean' )
     SCREAMING_SNAKE_CASE__				:				Optional[Any]			=		torch.tensor([2.0, 3] ).to(state.device )
     assert torch.allclose(lowercase__     ,  lowercase__ ), f'''{reduced_tensor} != {truth_tensor}'''







def     _a				(      lowercase__  :			int ):







     '''simple docstring'''


     main()







def     _a				(      ):







     '''simple docstring'''


     SCREAMING_SNAKE_CASE__				:				int			=		PartialState()
     state.print(f'''State: {state}''' )
     state.print('testing gather' )
     test_gather(lowercase__ )
     state.print('testing gather_object' )
     test_gather_object(lowercase__ )
     state.print('testing broadcast' )
     test_broadcast(lowercase__ )
     state.print('testing pad_across_processes' )
     test_pad_across_processes(lowercase__ )
     state.print('testing reduce_sum' )
     test_reduce_sum(lowercase__ )
     state.print('testing reduce_mean' )
     test_reduce_mean(lowercase__ )


if __name__ == "__main__":
      main()




import os
import socket
from contextlib import contextmanager

import torch

from ..commands.config.default import write_basic_config  # noqa: F401
from ..state import PartialState
from .dataclasses import DistributedType
from .imports import is_deepspeed_available, is_tpu_available
from .transformer_engine import convert_model
from .versions import is_torch_version


if is_deepspeed_available():
      from deepspeed import DeepSpeedEngine

if is_tpu_available(check_device=False):
      import torch_xla.core.xla_model as xm







def     _a				(      lowercase__  :			int ):







     '''simple docstring'''


     if is_torch_version('<'     ,  '2.0.0' ) or not hasattr(lowercase__     ,  '_dynamo' ):
          return False
     return isinstance(lowercase__     ,  torch._dynamo.eval_frame.OptimizedModule )







def     _a				(      lowercase__  :			Optional[Any]     ,  lowercase__  :			bool = True ):







     '''simple docstring'''


     SCREAMING_SNAKE_CASE__				:				Union[str, Any]			=		(torch.nn.parallel.DistributedDataParallel, torch.nn.DataParallel)

     SCREAMING_SNAKE_CASE__				:				Dict			=		is_compiled_module(lowercase__ )
     if is_compiled:
          SCREAMING_SNAKE_CASE__				:				Tuple			=		model
          SCREAMING_SNAKE_CASE__				:				int			=		model._orig_mod

     if is_deepspeed_available():
          options += (DeepSpeedEngine,)

     while isinstance(lowercase__     ,  lowercase__ ):
          SCREAMING_SNAKE_CASE__				:				Any			=		model.module

     if not keep_fpaa_wrapper:
          SCREAMING_SNAKE_CASE__				:				List[Any]			=		getattr(lowercase__     ,  'forward' )
          SCREAMING_SNAKE_CASE__				:				str			=		model.__dict__.pop('_original_forward'     ,  lowercase__ )
          if original_forward is not None:
               while hasattr(lowercase__     ,  '__wrapped__' ):
                    SCREAMING_SNAKE_CASE__				:				Dict			=		forward.__wrapped__
                    if forward == original_forward:
                         break
               SCREAMING_SNAKE_CASE__				:				Dict			=		forward
          if getattr(lowercase__     ,  '_converted_to_transformer_engine'     ,  lowercase__ ):
               convert_model(lowercase__     ,  to_transformer_engine=lowercase__ )

     if is_compiled:
          SCREAMING_SNAKE_CASE__				:				List[Any]			=		model
          SCREAMING_SNAKE_CASE__				:				Optional[Any]			=		compiled_model

     return model







def     _a				(      ):







     '''simple docstring'''


     PartialState().wait_for_everyone()







def     _a				(      lowercase__  :			str     ,  lowercase__  :			Optional[Any] ):







     '''simple docstring'''


     if PartialState().distributed_type == DistributedType.TPU:
          xm.save(lowercase__     ,  lowercase__ )
     elif PartialState().local_process_index == 0:
          torch.save(lowercase__     ,  lowercase__ )







@contextmanager
def     _a				(      **lowercase__  :			str ):







     '''simple docstring'''


     for key, value in kwargs.items():
          SCREAMING_SNAKE_CASE__				:				int			=		str(lowercase__ )

     yield

     for key in kwargs:
          if key.upper() in os.environ:
               del os.environ[key.upper()]







def     _a				(      lowercase__  :			Optional[Any] ):







     '''simple docstring'''


     if not hasattr(lowercase__     ,  '__qualname__' ) and not hasattr(lowercase__     ,  '__name__' ):
          SCREAMING_SNAKE_CASE__				:				Any			=		getattr(lowercase__     ,  '__class__'     ,  lowercase__ )
     if hasattr(lowercase__     ,  '__qualname__' ):
          return obj.__qualname__
     if hasattr(lowercase__     ,  '__name__' ):
          return obj.__name__
     return str(lowercase__ )







def     _a				(      lowercase__  :			List[str]     ,  lowercase__  :			List[Any] ):







     '''simple docstring'''


     for key, value in source.items():
          if isinstance(lowercase__     ,  lowercase__ ):
               SCREAMING_SNAKE_CASE__				:				List[str]			=		destination.setdefault(lowercase__     ,  {} )
               merge_dicts(lowercase__     ,  lowercase__ )
          else:
               SCREAMING_SNAKE_CASE__				:				List[Any]			=		value

     return destination







def     _a				(      lowercase__  :			int = None ):







     '''simple docstring'''


     if port is None:
          SCREAMING_SNAKE_CASE__				:				int			=		2_95_00
     with socket.socket(socket.AF_INET     ,  socket.SOCK_STREAM ) as s:
          return s.connect_ex(('localhost', port) ) == 0



