


"""simple docstring"""






import torch

from accelerate import PartialState
from accelerate.utils.operations import broadcast, gather, gather_object, pad_across_processes, reduce


def     _lowercase       (						__snake_case			)		->     Union[str, Any]:
			return (torch.arange(state.num_processes			) + 1.0 + (state.num_processes * state.process_index)).to(state.device			)


def     _lowercase       (						__snake_case			)		->     int:
			__lowerCAmelCase   :						List[str]				= create_tensor(__snake_case			)
			__lowerCAmelCase   :						Union[str, Any]				= gather(__snake_case			)
			assert gathered_tensor.tolist() == list(range(1      ,state.num_processes**2 + 1			)			)


def     _lowercase       (						__snake_case			)		->     Any:
			__lowerCAmelCase   :						Optional[int]				= [state.process_index]
			__lowerCAmelCase   :						Dict				= gather_object(__snake_case			)
			assert len(__snake_case			) == state.num_processes, F"""{gathered_obj}, {len(__snake_case			)} != {state.num_processes}"""
			assert gathered_obj == list(range(state.num_processes			)			), F"""{gathered_obj} != {list(range(state.num_processes			)			)}"""


def     _lowercase       (						__snake_case			)		->     Any:
			__lowerCAmelCase   :						List[str]				= create_tensor(__snake_case			)
			__lowerCAmelCase   :						List[str]				= broadcast(__snake_case			)
			assert broadcasted_tensor.shape == torch.Size([state.num_processes]			)
			assert broadcasted_tensor.tolist() == list(range(1      ,state.num_processes + 1			)			)


def     _lowercase       (						__snake_case			)		->     str:
			# We need to pad the tensor with one more element if we are the main process
			# to ensure that we can pad
			if state.is_main_process:
						__lowerCAmelCase   :						List[str]				= torch.arange(state.num_processes + 1			).to(state.device			)
			else:
						__lowerCAmelCase   :						Dict				= torch.arange(state.num_processes			).to(state.device			)
			__lowerCAmelCase   :						Optional[Any]				= pad_across_processes(__snake_case			)
			assert padded_tensor.shape == torch.Size([state.num_processes + 1]			)
			if not state.is_main_process:
						assert padded_tensor.tolist() == list(range(0      ,state.num_processes			)			) + [0]


def     _lowercase       (						__snake_case			)		->     str:
			# For now runs on only two processes
			if state.num_processes != 2:
						return
			__lowerCAmelCase   :						Dict				= create_tensor(__snake_case			)
			__lowerCAmelCase   :						Optional[int]				= reduce(__snake_case      ,"sum"			)
			__lowerCAmelCase   :						Optional[int]				= torch.tensor([4.0, 6]			).to(state.device			)
			assert torch.allclose(__snake_case      ,__snake_case			), F"""{reduced_tensor} != {truth_tensor}"""


def     _lowercase       (						__snake_case			)		->     Tuple:
			# For now runs on only two processes
			if state.num_processes != 2:
						return
			__lowerCAmelCase   :						List[str]				= create_tensor(__snake_case			)
			__lowerCAmelCase   :						Any				= reduce(__snake_case      ,"mean"			)
			__lowerCAmelCase   :						Tuple				= torch.tensor([2.0, 3]			).to(state.device			)
			assert torch.allclose(__snake_case      ,__snake_case			), F"""{reduced_tensor} != {truth_tensor}"""


def     _lowercase       (						__snake_case			)		->     Any:
			# For xla_spawn (TPUs)
			main()


def     _lowercase       (						)		->     Any:
			__lowerCAmelCase   :						List[str]				= PartialState()
			state.print(F"""State: {state}"""			)
			state.print("testing gather"			)
			test_gather(__snake_case			)
			state.print("testing gather_object"			)
			test_gather_object(__snake_case			)
			state.print("testing broadcast"			)
			test_broadcast(__snake_case			)
			state.print("testing pad_across_processes"			)
			test_pad_across_processes(__snake_case			)
			state.print("testing reduce_sum"			)
			test_reduce_sum(__snake_case			)
			state.print("testing reduce_mean"			)
			test_reduce_mean(__snake_case			)


if __name__ == "__main__":
	main()




"""simple docstring"""






import argparse

import torch

from ...utils import logging
from . import AlbertConfig, AlbertForPreTraining, load_tf_weights_in_albert


logging.set_verbosity_info()


def     _lowercase       (						__snake_case      ,__snake_case      ,__snake_case			)		->     Tuple:
			# Initialise PyTorch model
			__lowerCAmelCase   :						Any				= AlbertConfig.from_json_file(__snake_case			)
			print(F"""Building PyTorch model from configuration: {config}"""			)
			__lowerCAmelCase   :						str				= AlbertForPreTraining(__snake_case			)

			# Load weights from tf checkpoint
			load_tf_weights_in_albert(__snake_case      ,__snake_case      ,__snake_case			)

			# Save pytorch-model
			print(F"""Save PyTorch model to {pytorch_dump_path}"""			)
			torch.save(model.state_dict()      ,__snake_case			)


if __name__ == "__main__":
	__snake_case							:     Union[str, Any]   =					argparse.ArgumentParser()
	# Required parameters
	parser.add_argument(
	    '--tf_checkpoint_path', default=None, type=str, required=True, help='Path to the TensorFlow checkpoint path.'
	)
	parser.add_argument(
	    '--albert_config_file',
	    default=None,
	    type=str,
	    required=True,
	    help=(
	        'The config json file corresponding to the pre-trained ALBERT model. \n'
	        'This specifies the model architecture.'
	    ),
	)
	parser.add_argument(
	    '--pytorch_dump_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
	)
	__snake_case							:     str   =					parser.parse_args()
	convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, args.albert_config_file, args.pytorch_dump_path)