

'''simple docstring'''







import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing the experiment tracking capability,
# and builds off the `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To help focus on the differences in the code, building `DataLoaders`
# was refactored into its own function.
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################

__lowerCamelCase		       =    16
__lowerCamelCase		       =    32
def       UpperCAmelCase__    (      UpperCAmelCase__,       UpperCAmelCase__ = 16				)			->			Optional[int]:
       A_						=      AutoTokenizer.from_pretrained("""bert-base-cased"""				)
       A_						=      load_dataset("""glue""",       """mrpc"""				)

       def tokenize_function(UpperCAmelCase__				):
              # max_length=None => use the model max length (it's actually the default)
              A_						=      tokenizer(examples["""sentence1"""],       examples["""sentence2"""],       truncation=UpperCAmelCase__,       max_length=UpperCAmelCase__				)
              return outputs

       # Apply the method we just defined to all the examples in all the splits of the dataset
       # starting with the main process first:
       with accelerator.main_process_first():
              A_						=      datasets.map(
                  UpperCAmelCase__,       batched=UpperCAmelCase__,       remove_columns=["""idx""", """sentence1""", """sentence2"""],       )

       # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
       # transformers library
       A_						=      tokenized_datasets.rename_column("""label""",       """labels"""				)

       def collate_fn(UpperCAmelCase__				):
              # On TPU it's best to pad everything to the same length or training will be very slow.
              A_						=      1_28 if accelerator.distributed_type == DistributedType.TPU else None
              # When using mixed precision we want round multiples of 8/16
              if accelerator.mixed_precision == "fp8":
                     A_						=      16
              elif accelerator.mixed_precision != "no":
                     A_						=      8
              else:
                     A_						=      None

              return tokenizer.pad(
                  UpperCAmelCase__,       padding="""longest""",       max_length=UpperCAmelCase__,       pad_to_multiple_of=UpperCAmelCase__,       return_tensors="""pt""",       )

       # Instantiate dataloaders.
       A_						=      DataLoader(
           tokenized_datasets["""train"""],       shuffle=UpperCAmelCase__,       collate_fn=UpperCAmelCase__,       batch_size=UpperCAmelCase__				)
       A_						=      DataLoader(
           tokenized_datasets["""validation"""],       shuffle=UpperCAmelCase__,       collate_fn=UpperCAmelCase__,       batch_size=UpperCAmelCase__				)

       return train_dataloader, eval_dataloader


# For testing only
if os.environ.get('''TESTING_MOCKED_DATALOADERS''', None) == "1":
      from accelerate.test_utils.training import mocked_dataloaders

      __lowerCamelCase		       =    mocked_dataloaders  # noqa: F811
def       UpperCAmelCase__    (      UpperCAmelCase__,       UpperCAmelCase__				)			->			Optional[Any]:
       # For testing only
       if os.environ.get("""TESTING_MOCKED_DATALOADERS""",       UpperCAmelCase__				) == "1":
              A_						=      2
       # Initialize Accelerator

       # New Code #
       # We pass in "all" to `log_with` to grab all available trackers in the environment
       # Note: If using a custom `Tracker` class, should be passed in here such as:
       # >>> log_with = ["all", MyCustomTrackerClassInstance()]
       if args.with_tracking:
              A_						=      Accelerator(
                  cpu=args.cpu,       mixed_precision=args.mixed_precision,       log_with="""all""",       project_dir=args.project_dir				)
       else:
              A_						=      Accelerator(cpu=args.cpu,       mixed_precision=args.mixed_precision				)
       # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
       A_						=      config["""lr"""]
       A_						=      int(config["""num_epochs"""]				)
       A_						=      int(config["""seed"""]				)
       A_						=      int(config["""batch_size"""]				)
       set_seed(UpperCAmelCase__				)

       A_					,			A_						=      get_dataloaders(UpperCAmelCase__,       UpperCAmelCase__				)
       A_						=      evaluate.load("""glue""",       """mrpc"""				)

       # If the batch size is too big we use gradient accumulation
       A_						=      1
       if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
              A_						=      batch_size // MAX_GPU_BATCH_SIZE
              A_						=      MAX_GPU_BATCH_SIZE

       # Instantiate the model (we build the model here so that the seed also control new weights initialization)
       A_						=      AutoModelForSequenceClassification.from_pretrained("""bert-base-cased""",       return_dict=UpperCAmelCase__				)

       # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
       # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
       # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
       A_						=      model.to(accelerator.device				)

       # Instantiate optimizer
       A_						=      AdamW(params=model.parameters(),       lr=UpperCAmelCase__				)

       # Instantiate scheduler
       A_						=      get_linear_schedule_with_warmup(
           optimizer=UpperCAmelCase__,       num_warmup_steps=1_00,       num_training_steps=(len(UpperCAmelCase__				) * num_epochs) // gradient_accumulation_steps,       )

       # Prepare everything
       # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
       # prepare method.
       A_					,			A_					,			A_					,			A_					,			A_						=      accelerator.prepare(
           UpperCAmelCase__,       UpperCAmelCase__,       UpperCAmelCase__,       UpperCAmelCase__,       UpperCAmelCase__				)

       # New Code #
       # We need to initialize the trackers we use. Overall configurations can also be stored
       if args.with_tracking:
              A_						=      os.path.split(UpperCAmelCase__				)[-1].split("""."""				)[0]
              accelerator.init_trackers(UpperCAmelCase__,       UpperCAmelCase__				)

       # Now we train the model
       for epoch in range(UpperCAmelCase__				):
              model.train()
              # New Code #
              # For our tracking example, we will log the total loss of each epoch
              if args.with_tracking:
                     A_						=      0
              for step, batch in enumerate(UpperCAmelCase__				):
                     # We could avoid this line since we set the accelerator with `device_placement=True`.
                     batch.to(accelerator.device				)
                     A_						=      model(**UpperCAmelCase__				)
                     A_						=      outputs.loss
                     # New Code #
                     if args.with_tracking:
                            total_loss += loss.detach().float()
                     A_						=      loss / gradient_accumulation_steps
                     accelerator.backward(UpperCAmelCase__				)
                     if step % gradient_accumulation_steps == 0:
                            optimizer.step()
                            lr_scheduler.step()
                            optimizer.zero_grad()

              model.eval()
              for step, batch in enumerate(UpperCAmelCase__				):
                     # We could avoid this line since we set the accelerator with `device_placement=True` (the default).
                     batch.to(accelerator.device				)
                     with torch.no_grad():
                            A_						=      model(**UpperCAmelCase__				)
                     A_						=      outputs.logits.argmax(dim=-1				)
                     A_					,			A_						=      accelerator.gather_for_metrics((predictions, batch["""labels"""])				)
                     metric.add_batch(
                         predictions=UpperCAmelCase__,       references=UpperCAmelCase__,       )

              A_						=      metric.compute()
              # Use accelerator.print to print only on the main process.
              accelerator.print(F'''epoch {epoch}:''',       UpperCAmelCase__				)

              # New Code #
              # To actually log, we call `Accelerator.log`
              # The values passed can be of `str`, `int`, `float` or `dict` of `str` to `float`/`int`
              if args.with_tracking:
                     accelerator.log(
                         {
                             """accuracy""": eval_metric["""accuracy"""],
                             """f1""": eval_metric["""f1"""],
                             """train_loss""": total_loss.item() / len(UpperCAmelCase__				),
                             """epoch""": epoch,
                         },       step=UpperCAmelCase__,       )

    # New Code #
    # When a run is finished, you should call `accelerator.end_training()`
    # to close all of the open trackers
       if args.with_tracking:
              accelerator.end_training()
def       UpperCAmelCase__    (      )			->			Dict:
       A_						=      argparse.ArgumentParser(description="""Simple example of training script."""				)
       parser.add_argument(
           """--mixed_precision""",       type=UpperCAmelCase__,       default=UpperCAmelCase__,       choices=["""no""", """fp16""", """bf16""", """fp8"""],       help="""Whether to use mixed precision. Choose"""
           """between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."""
           """and an Nvidia Ampere GPU.""",       )
       parser.add_argument("""--cpu""",       action="""store_true""",       help="""If passed, will train on the CPU."""				)
       parser.add_argument(
           """--with_tracking""",       action="""store_true""",       help="""Whether to load in all available experiment trackers from the environment and use them for logging.""",       )
       parser.add_argument(
           """--project_dir""",       type=UpperCAmelCase__,       default="""logs""",       help="""Location on where to store experiment tracking logs` and relevent project information""",       )
       A_						=      parser.parse_args()
       A_						=      {"""lr""": 2e-5, """num_epochs""": 3, """seed""": 42, """batch_size""": 16}
       training_function(UpperCAmelCase__,       UpperCAmelCase__				)


if __name__ == "__main__":
      main()



'''simple docstring'''







def       UpperCAmelCase__    (      UpperCAmelCase__				)			->			bool:
       if num < 0:
              return False

       A_						=      num
       A_						=      0
       while num > 0:
              A_						=      rev_num * 10 + (num % 10)
              num //= 10

       return num_copy == rev_num


if __name__ == "__main__":
      import doctest

      doctest.testmod()
