






import argparse
import os

# New Code #
import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType
from accelerate.utils import find_executable_batch_size


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to ensure out-of-memory errors never
# interrupt training, and builds off the `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


lowerCamelCase : Tuple						   =       16
lowerCamelCase : List[str]						   =       32
def       snake_case_		(     lowerCAmelCase_       :						Accelerator       ,       lowerCAmelCase_       :						int = 16      ):
							__lowercase      :							Optional[int]				=     AutoTokenizer.from_pretrained("""bert-base-cased"""      )
							__lowercase      :							str				=     load_dataset("""glue"""       ,       """mrpc"""      )

							def tokenize_function(lowerCAmelCase_       :						str      ):
														# max_length=None => use the model max length (it's actually the default)
														__lowercase      :							Tuple				=     tokenizer(examples["""sentence1"""]       ,       examples["""sentence2"""]       ,       truncation=__A       ,       max_length=__A      )
														return outputs

							# Apply the method we just defined to all the examples in all the splits of the dataset
							# starting with the main process first:
							with accelerator.main_process_first():
														__lowercase      :							Union[str, Any]				=     datasets.map(
														    __A       ,       batched=__A       ,       remove_columns=["""idx""", """sentence1""", """sentence2"""]       ,       )

							# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
							# transformers library
							__lowercase      :							Optional[Any]				=     tokenized_datasets.rename_column("""label"""       ,       """labels"""      )

							def collate_fn(lowerCAmelCase_       :						int      ):
														# On TPU it's best to pad everything to the same length or training will be very slow.
														__lowercase      :							List[Any]				=     128 if accelerator.distributed_type == DistributedType.TPU else None
														# When using mixed precision we want round multiples of 8/16
														if accelerator.mixed_precision == "fp8":
																					__lowercase      :							Optional[Any]				=     16
														elif accelerator.mixed_precision != "no":
																					__lowercase      :							str				=     8
														else:
																					__lowercase      :							Optional[Any]				=     None

														return tokenizer.pad(
														    __A       ,       padding="""longest"""       ,       max_length=__A       ,       pad_to_multiple_of=__A       ,       return_tensors="""pt"""       ,       )

							# Instantiate dataloaders.
							__lowercase      :							Dict				=     DataLoader(
							    tokenized_datasets["""train"""]       ,       shuffle=__A       ,       collate_fn=__A       ,       batch_size=__A      )
							__lowercase      :							Optional[int]				=     DataLoader(
							    tokenized_datasets["""validation"""]       ,       shuffle=__A       ,       collate_fn=__A       ,       batch_size=__A      )

							return train_dataloader, eval_dataloader


# For testing only
if os.environ.get('''TESTING_MOCKED_DATALOADERS''', None) == "1":
	from accelerate.test_utils.training import mocked_dataloaders

	lowerCamelCase : Tuple						   =       mocked_dataloaders  # noqa: F811
def       snake_case_		(     lowerCAmelCase_       :						int       ,       lowerCAmelCase_       :						int      ):




							if os.environ.get("""TESTING_MOCKED_DATALOADERS"""       ,       __A      ) == "1":
														__lowercase      :							Union[str, Any]				=     2
							# Initialize accelerator
							__lowercase      :							str				=     Accelerator(cpu=args.cpu       ,       mixed_precision=args.mixed_precision      )
							# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
							__lowercase      :							Dict				=     config['''lr''']
							__lowercase      :							Tuple				=     int(config["""num_epochs"""]      )
							__lowercase      :							List[Any]				=     int(config["""seed"""]      )
							__lowercase      :							Tuple				=     int(config["""batch_size"""]      )

							__lowercase      :							Union[str, Any]				=     evaluate.load("""glue"""       ,       """mrpc"""      )

							# New Code #
							# We now can define an inner training loop function. It should take a batch size as the only parameter,
							# and build the dataloaders in there.
							# It also gets our decorator
							@find_executable_batch_size(starting_batch_size=__A      )
							def inner_training_loop(lowerCAmelCase_       :						Tuple      ):
														# And now just move everything below under this function
														# We need to bring in the Accelerator object from earlier
														nonlocal accelerator
														# And reset all of its attributes that could hold onto any memory:
														accelerator.free_memory()

														# Then we can declare the model, optimizer, and everything else:
														set_seed(__A      )

														# Instantiate the model (we build the model here so that the seed also control new weights initialization)
														__lowercase      :							List[Any]				=     AutoModelForSequenceClassification.from_pretrained("""bert-base-cased"""       ,       return_dict=__A      )

														# We could avoid this line since the accelerator is set with `device_placement=True` (default value).
														# Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
														# creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
														__lowercase      :							Optional[Any]				=     model.to(accelerator.device      )

														# Instantiate optimizer
														__lowercase      :							Any				=     AdamW(params=model.parameters()       ,       lr=__A      )
														__lowercase      :							Tuple				=     get_dataloaders(__A       ,       __A      )

														# Instantiate scheduler
														__lowercase      :							Any				=     get_linear_schedule_with_warmup(
														    optimizer=__A       ,       num_warmup_steps=100       ,       num_training_steps=(len(__A      ) * num_epochs)       ,       )

														# Prepare everything
														# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
														# prepare method.
														__lowercase      :							List[Any]				=     accelerator.prepare(
														    __A       ,       __A       ,       __A       ,       __A       ,       __A      )

														# Now we train the model
														for epoch in range(__A      ):
																					model.train()
																					for step, batch in enumerate(__A      ):
																												# We could avoid this line since we set the accelerator with `device_placement=True`.
																												batch.to(accelerator.device      )
																												__lowercase      :							List[str]				=     model(**__A      )
																												__lowercase      :							int				=     outputs.loss
																												accelerator.backward(__A      )
																												optimizer.step()
																												lr_scheduler.step()
																												optimizer.zero_grad()

																					model.eval()
																					for step, batch in enumerate(__A      ):
																												# We could avoid this line since we set the accelerator with `device_placement=True`.
																												batch.to(accelerator.device      )
																												with torch.no_grad():
																																			__lowercase      :							List[Any]				=     model(**__A      )
																												__lowercase      :							Optional[Any]				=     outputs.logits.argmax(dim=-1      )
																												__lowercase      :							List[Any]				=     accelerator.gather_for_metrics((predictions, batch["""labels"""])      )
																												metric.add_batch(
																												    predictions=__A       ,       references=__A       ,       )

																					__lowercase      :							Optional[Any]				=     metric.compute()
																					# Use accelerator.print to print only on the main process.
																					accelerator.print(F"epoch {epoch}:"       ,       __A      )

																					 # New Code #
																					 # And call it at the end with no arguments
																					 # Note: You could also refactor this outside of your training loop function
							inner_training_loop()
def       snake_case_		(     ):
							__lowercase      :							Any				=     argparse.ArgumentParser(description="""Simple example of training script."""      )
							parser.add_argument(
							    """--mixed_precision"""       ,       type=__A       ,       default=__A       ,       choices=["""no""", """fp16""", """bf16""", """fp8"""]       ,       help="""Whether to use mixed precision. Choose"""
							    """between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."""
							    """and an Nvidia Ampere GPU."""       ,       )
							parser.add_argument("""--cpu"""       ,       action="""store_true"""       ,       help="""If passed, will train on the CPU."""      )
							__lowercase      :							Tuple				=     parser.parse_args()
							__lowercase      :							Optional[Any]				=     {'''lr''': 2e-5, '''num_epochs''': 3, '''seed''': 42, '''batch_size''': 16}
							training_function(__A       ,       __A      )


if __name__ == "__main__":
	main()







# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os

from accelerate.utils import ComputeEnvironment

from .cluster import get_cluster_input
from .config_args import cache_dir, default_config_file, default_yaml_config_file, load_config_from_file  # noqa: F401
from .config_utils import _ask_field, _ask_options, _convert_compute_environment  # noqa: F401
from .sagemaker import get_sagemaker_input


lowerCamelCase : int						   =       '''Launches a series of prompts to create and save a `default_config.yaml` configuration file for your training system. Should always be ran first on your machine'''
def       snake_case_		(     ):
							__lowercase      :							List[Any]				=     _ask_options(
							    """In which compute environment are you running?"""       ,       ["""This machine""", """AWS (Amazon SageMaker)"""]       ,       _convert_compute_environment       ,       )
							if compute_environment == ComputeEnvironment.AMAZON_SAGEMAKER:
														__lowercase      :							Union[str, Any]				=     get_sagemaker_input()
							else:
														__lowercase      :							str				=     get_cluster_input()
							return config
def       snake_case_		(     lowerCAmelCase_       :						List[str]=None      ):
							if subparsers is not None:
														__lowercase      :							Optional[int]				=     subparsers.add_parser("""config"""       ,       description=lowerCAmelCase_      )
							else:
														__lowercase      :							List[str]				=     argparse.ArgumentParser("""Accelerate config command"""       ,       description=lowerCAmelCase_      )

							parser.add_argument(
							    """--config_file"""       ,       default=lowerCAmelCase_       ,       help=(
							        """The path to use to store the config file. Will default to a file named default_config.yaml in the cache """
							        """location, which is the content of the environment `HF_HOME` suffixed with 'accelerate', or if you don't have """
							        """such an environment variable, your cache directory ('~/.cache' or the content of `XDG_CACHE_HOME`) suffixed """
							        """with 'huggingface'."""
							    )       ,       )

							if subparsers is not None:
														parser.set_defaults(func=lowerCAmelCase_      )
							return parser
def       snake_case_		(     lowerCAmelCase_       :						Tuple      ):
							__lowercase      :							Union[str, Any]				=     get_user_input()
							if args.config_file is not None:
														__lowercase      :							List[Any]				=     args.config_file
							else:
														if not os.path.isdir(lowerCAmelCase_      ):
																					os.makedirs(lowerCAmelCase_      )
														__lowercase      :							Any				=     default_yaml_config_file

							if config_file.endswith(""".json"""      ):
														config.to_json_file(lowerCAmelCase_      )
							else:
														config.to_yaml_file(lowerCAmelCase_      )
							print(F"accelerate configuration saved at {config_file}"      )
def       snake_case_		(     ):
							__lowercase      :							str				=     config_command_parser()
							__lowercase      :							str				=     parser.parse_args()
							config_command(lowerCAmelCase_      )


if __name__ == "__main__":
	main()