





'''simple docstring'''







import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
snake_case_     :       Union[str, Any]						=  data_utils.TransfoXLTokenizer
snake_case_     :       List[Any]						=  data_utils.TransfoXLCorpus
snake_case_     :       Union[str, Any]						=  data_utils
snake_case_     :       Any						=  data_utils





def        lowerCamelCase_      (  SCREAMING_SNAKE_CASE__      :			str,			SCREAMING_SNAKE_CASE__      :			List[Any],			SCREAMING_SNAKE_CASE__      :			int,			SCREAMING_SNAKE_CASE__      :			Tuple   ) ->	Optional[Any]:
						if transfo_xl_dataset_file:
												# Convert a pre-processed corpus (see original TensorFlow repo)
												with open(SCREAMING_SNAKE_CASE__,			'''rb'''   ) as fp:
																		UpperCAmelCase_			:      Dict     						=  pickle.load(SCREAMING_SNAKE_CASE__,			encoding='''latin1'''   )
												# Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
												UpperCAmelCase_			:      str     						=  pytorch_dump_folder_path + '''/''' + VOCAB_FILES_NAMES['''pretrained_vocab_file''']
												print(F"""Save vocabulary to {pytorch_vocab_dump_path}"""   )
												UpperCAmelCase_			:      Optional[Any]     						=  corpus.vocab.__dict__
												torch.save(SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__   )

												UpperCAmelCase_			:      Optional[Any]     						=  corpus.__dict__
												corpus_dict_no_vocab.pop('''vocab''',			SCREAMING_SNAKE_CASE__   )
												UpperCAmelCase_			:      Optional[Any]     						=  pytorch_dump_folder_path + '''/''' + CORPUS_NAME
												print(F"""Save dataset to {pytorch_dataset_dump_path}"""   )
												torch.save(SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__   )

						if tf_checkpoint_path:
												# Convert a pre-trained TensorFlow model
												UpperCAmelCase_			:      Optional[int]     						=  os.path.abspath(SCREAMING_SNAKE_CASE__   )
												UpperCAmelCase_			:      List[Any]     						=  os.path.abspath(SCREAMING_SNAKE_CASE__   )

												print(F"""Converting Transformer XL checkpoint from {tf_path} with config at {config_path}."""   )
												# Initialise PyTorch model
												if transfo_xl_config_file == "":
																		UpperCAmelCase_			:      Any     						=  TransfoXLConfig()
												else:
																		UpperCAmelCase_			:      Tuple     						=  TransfoXLConfig.from_json_file(SCREAMING_SNAKE_CASE__   )
												print(F"""Building PyTorch model from configuration: {config}"""   )
												UpperCAmelCase_			:      Union[str, Any]     						=  TransfoXLLMHeadModel(SCREAMING_SNAKE_CASE__   )

												UpperCAmelCase_			:      Any     						=  load_tf_weights_in_transfo_xl(SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__   )
												# Save pytorch-model
												UpperCAmelCase_			:      List[Any]     						=  os.path.join(SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__   )
												UpperCAmelCase_			:      List[str]     						=  os.path.join(SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__   )
												print(F"""Save PyTorch model to {os.path.abspath(SCREAMING_SNAKE_CASE__   )}"""   )
												torch.save(model.state_dict(),			SCREAMING_SNAKE_CASE__   )
												print(F"""Save configuration file to {os.path.abspath(SCREAMING_SNAKE_CASE__   )}"""   )
												with open(SCREAMING_SNAKE_CASE__,			'''w''',			encoding='''utf-8'''   ) as f:
																		f.write(config.to_json_string()   )


if __name__ == "__main__":
					snake_case_     :       List[str]						=  argparse.ArgumentParser()
					parser.add_argument(
					    "--pytorch_dump_folder_path",
					    default=None,
					    type=str,
					    required=True,
					    help="Path to the folder to store the PyTorch model or dataset/vocab.",
					)
					parser.add_argument(
					    "--tf_checkpoint_path",
					    default="",
					    type=str,
					    help="An optional path to a TensorFlow checkpoint path to be converted.",
					)
					parser.add_argument(
					    "--transfo_xl_config_file",
					    default="",
					    type=str,
					    help=(
					        "An optional config json file corresponding to the pre-trained BERT model. \n"
					        "This specifies the model architecture."
					    ),
					)
					parser.add_argument(
					    "--transfo_xl_dataset_file",
					    default="",
					    type=str,
					    help="An optional dataset file to be converted in a vocabulary.",
					)
					snake_case_     :       List[Any]						=  parser.parse_args()
					convert_transfo_xl_checkpoint_to_pytorch(
					    args.tf_checkpoint_path,
					    args.transfo_xl_config_file,
					    args.pytorch_dump_folder_path,
					    args.transfo_xl_dataset_file,
					)












'''simple docstring'''







import argparse
import json
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType
from accelerate.utils.deepspeed import DummyOptim, DummyScheduler


snake_case_     :       Optional[int]						=  16
snake_case_     :       Tuple						=  32





def        lowerCamelCase_      (  SCREAMING_SNAKE_CASE__      :			Accelerator,			SCREAMING_SNAKE_CASE__      :			int = 16,			SCREAMING_SNAKE_CASE__      :			str = "bert-base-cased"   ) ->	Dict:
						UpperCAmelCase_			:      Dict     						=  AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__   )
						UpperCAmelCase_			:      int     						=  load_dataset('''glue''',			'''mrpc'''   )

						def tokenize_function(SCREAMING_SNAKE_CASE__      :			Optional[int]   ):
												# max_length=None => use the model max length (it's actually the default)
												UpperCAmelCase_			:      Union[str, Any]     						=  tokenizer(examples['''sentence1'''],			examples['''sentence2'''],			truncation=SCREAMING_SNAKE_CASE__,			max_length=SCREAMING_SNAKE_CASE__   )
												return outputs

						# Apply the method we just defined to all the examples in all the splits of the dataset
						UpperCAmelCase_			:      Tuple     						=  datasets.map(
						    SCREAMING_SNAKE_CASE__,			batched=SCREAMING_SNAKE_CASE__,			remove_columns=['''idx''', '''sentence1''', '''sentence2'''],			load_from_cache_file=SCREAMING_SNAKE_CASE__   )

						# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
						# transformers library
						UpperCAmelCase_			:      Optional[Any]     						=  tokenized_datasets.rename_column('''label''',			'''labels'''   )

						def collate_fn(SCREAMING_SNAKE_CASE__      :			str   ):
												# On TPU it's best to pad everything to the same length or training will be very slow.
												if accelerator.distributed_type == DistributedType.TPU:
																		return tokenizer.pad(SCREAMING_SNAKE_CASE__,			padding='''max_length''',			max_length=128,			return_tensors='''pt'''   )
												return tokenizer.pad(SCREAMING_SNAKE_CASE__,			padding='''longest''',			return_tensors='''pt'''   )

						# Instantiate dataloaders.
						UpperCAmelCase_			:      str     						=  DataLoader(
						    tokenized_datasets['''train'''],			shuffle=SCREAMING_SNAKE_CASE__,			collate_fn=SCREAMING_SNAKE_CASE__,			batch_size=SCREAMING_SNAKE_CASE__   )
						UpperCAmelCase_			:      int     						=  DataLoader(
						    tokenized_datasets['''validation'''],			shuffle=SCREAMING_SNAKE_CASE__,			collate_fn=SCREAMING_SNAKE_CASE__,			batch_size=SCREAMING_SNAKE_CASE__   )

						return train_dataloader, eval_dataloader





def        lowerCamelCase_      (  SCREAMING_SNAKE_CASE__      :			Optional[Any],			SCREAMING_SNAKE_CASE__      :			Optional[int],			SCREAMING_SNAKE_CASE__      :			Tuple,			SCREAMING_SNAKE_CASE__      :			Any   ) ->	Any:
						model.eval()
						UpperCAmelCase_			:      List[str]     						=  0
						for step, batch in enumerate(SCREAMING_SNAKE_CASE__   ):
												# We could avoid this line since we set the accelerator with `device_placement=True`.
												batch.to(accelerator.device   )
												with torch.no_grad():
																		UpperCAmelCase_			:      Dict     						=  model(**SCREAMING_SNAKE_CASE__   )
												UpperCAmelCase_			:      str     						=  outputs.logits.argmax(dim=-1   )
												# It is slightly faster to call this once, than multiple times
												UpperCAmelCase_						,       UpperCAmelCase_			:      List[str]     						=  accelerator.gather(
												    (predictions, batch['''labels'''])   )  # If we are in a multiprocess environment, the last batch has duplicates
												if accelerator.use_distributed:
																		if step == len(SCREAMING_SNAKE_CASE__   ) - 1:
																								UpperCAmelCase_			:      Tuple     						=  predictions[: len(eval_dataloader.dataset   ) - samples_seen]
																								UpperCAmelCase_			:      int     						=  references[: len(eval_dataloader.dataset   ) - samples_seen]
																		else:
																								samples_seen += references.shape[0]
												metric.add_batch(
												    predictions=SCREAMING_SNAKE_CASE__,			references=SCREAMING_SNAKE_CASE__,			)

						UpperCAmelCase_			:      List[str]     						=  metric.compute()
						return eval_metric["accuracy"]





def        lowerCamelCase_      (  SCREAMING_SNAKE_CASE__      :			Tuple,			SCREAMING_SNAKE_CASE__      :			int   ) ->	Tuple:
						# Initialize accelerator
						UpperCAmelCase_			:      Union[str, Any]     						=  Accelerator()

						# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
						UpperCAmelCase_			:      int     						=  config['''lr''']
						UpperCAmelCase_			:      Optional[int]     						=  int(config['''num_epochs''']   )
						UpperCAmelCase_			:      Optional[int]     						=  int(config['''seed''']   )
						UpperCAmelCase_			:      List[str]     						=  int(config['''batch_size''']   )
						UpperCAmelCase_			:      Optional[int]     						=  args.model_name_or_path

						set_seed(SCREAMING_SNAKE_CASE__   )
						UpperCAmelCase_						,       UpperCAmelCase_			:      Union[str, Any]     						=  get_dataloaders(SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__   )

						# Instantiate the model (we build the model here so that the seed also control new weights initialization)
						UpperCAmelCase_			:      List[Any]     						=  AutoModelForSequenceClassification.from_pretrained(SCREAMING_SNAKE_CASE__,			return_dict=SCREAMING_SNAKE_CASE__   )

						# Instantiate optimizer
						UpperCAmelCase_			:      str     						=  (
						    AdamW
						    if accelerator.state.deepspeed_plugin is None
						    or '''optimizer''' not in accelerator.state.deepspeed_plugin.deepspeed_config
						    else DummyOptim
						)
						UpperCAmelCase_			:      List[str]     						=  optimizer_cls(params=model.parameters(),			lr=SCREAMING_SNAKE_CASE__   )

						if accelerator.state.deepspeed_plugin is not None:
												UpperCAmelCase_			:      List[Any]     						=  accelerator.state.deepspeed_plugin.deepspeed_config[
												    '''gradient_accumulation_steps'''
												]
						else:
												UpperCAmelCase_			:      Tuple     						=  1
						UpperCAmelCase_			:      int     						=  (len(SCREAMING_SNAKE_CASE__   ) * num_epochs) // gradient_accumulation_steps

						# Instantiate scheduler
						if (
						    accelerator.state.deepspeed_plugin is None
						    or "scheduler" not in accelerator.state.deepspeed_plugin.deepspeed_config
						):
												UpperCAmelCase_			:      Tuple     						=  get_linear_schedule_with_warmup(
												    optimizer=SCREAMING_SNAKE_CASE__,			num_warmup_steps=0,			num_training_steps=SCREAMING_SNAKE_CASE__,			)
						else:
												UpperCAmelCase_			:      Any     						=  DummyScheduler(SCREAMING_SNAKE_CASE__,			total_num_steps=SCREAMING_SNAKE_CASE__,			warmup_num_steps=0   )

						# Prepare everything
						# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
						# prepare method.
						UpperCAmelCase_						,       UpperCAmelCase_						,       UpperCAmelCase_						,       UpperCAmelCase_						,       UpperCAmelCase_			:      Union[str, Any]     						=  accelerator.prepare(
						    SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__   )

						# We need to keep track of how many total steps we have iterated over
						UpperCAmelCase_			:      Union[str, Any]     						=  0
						# We also need to keep track of the stating epoch so files are named properly
						UpperCAmelCase_			:      Dict     						=  0
						UpperCAmelCase_			:      int     						=  evaluate.load('''glue''',			'''mrpc'''   )
						UpperCAmelCase_			:      Optional[Any]     						=  num_epochs

						if args.partial_train_epoch is not None:
												UpperCAmelCase_			:      List[Any]     						=  args.partial_train_epoch

						if args.resume_from_checkpoint:
												accelerator.load_state(args.resume_from_checkpoint   )
												UpperCAmelCase_			:      Tuple     						=  args.resume_from_checkpoint.split('''epoch_'''   )[1]
												UpperCAmelCase_			:      int     						=  ''''''
												for char in epoch_string:
																		if char.isdigit():
																								state_epoch_num += char
																		else:
																								break
												UpperCAmelCase_			:      Union[str, Any]     						=  int(SCREAMING_SNAKE_CASE__   ) + 1
												UpperCAmelCase_			:      Dict     						=  evaluation_loop(SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__   )
												accelerator.print('''resumed checkpoint performance:''',			SCREAMING_SNAKE_CASE__   )
												accelerator.print('''resumed checkpoint\'s scheduler\'s lr:''',			lr_scheduler.get_lr()[0]   )
												accelerator.print('''resumed optimizers\'s lr:''',			optimizer.param_groups[0]['''lr''']   )
												with open(os.path.join(args.output_dir,			F"""state_{starting_epoch-1}.json"""   ),			'''r'''   ) as f:
																		UpperCAmelCase_			:      Optional[int]     						=  json.load(SCREAMING_SNAKE_CASE__   )
																		assert resumed_state["accuracy"] == accuracy, "Accuracy mismatch, loading from checkpoint failed"
																		assert (
																		    resumed_state["lr"] == lr_scheduler.get_lr()[0]
																		), "Scheduler learning rate mismatch, loading from checkpoint failed"
																		assert (
																		    resumed_state["optimizer_lr"] == optimizer.param_groups[0]["lr"]
																		), "Optimizer learning rate mismatch, loading from checkpoint failed"
																		assert resumed_state["epoch"] == starting_epoch - 1, "Epoch mismatch, loading from checkpoint failed"
																		return

    # Now we train the model
						UpperCAmelCase_			:      int     						=  {}
						for epoch in range(SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__   ):
												model.train()
												for step, batch in enumerate(SCREAMING_SNAKE_CASE__   ):
																		UpperCAmelCase_			:      Optional[int]     						=  model(**SCREAMING_SNAKE_CASE__   )
																		UpperCAmelCase_			:      Any     						=  outputs.loss
																		UpperCAmelCase_			:      Tuple     						=  loss / gradient_accumulation_steps
																		accelerator.backward(SCREAMING_SNAKE_CASE__   )
																		if step % gradient_accumulation_steps == 0:
																								optimizer.step()
																								lr_scheduler.step()
																								optimizer.zero_grad()

																		overall_step += 1
												UpperCAmelCase_			:      Tuple     						=  F"""epoch_{epoch}"""
												UpperCAmelCase_			:      Optional[int]     						=  os.path.join(args.output_dir,			SCREAMING_SNAKE_CASE__   )
												accelerator.save_state(SCREAMING_SNAKE_CASE__   )
												UpperCAmelCase_			:      int     						=  evaluation_loop(SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__   )
												UpperCAmelCase_			:      Optional[Any]     						=  accuracy
												UpperCAmelCase_			:      Any     						=  lr_scheduler.get_lr()[0]
												UpperCAmelCase_			:      List[str]     						=  optimizer.param_groups[0]['''lr''']
												UpperCAmelCase_			:      Tuple     						=  epoch
												UpperCAmelCase_			:      Dict     						=  overall_step
												accelerator.print(F"""epoch {epoch}:""",			SCREAMING_SNAKE_CASE__   )

												accelerator.wait_for_everyone()
												if accelerator.is_main_process:
																		with open(os.path.join(args.output_dir,			F"""state_{epoch}.json"""   ),			'''w'''   ) as f:
																								json.dump(SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__   )





def        lowerCamelCase_      (  ) ->	List[str]:
						UpperCAmelCase_			:      Optional[int]     						=  argparse.ArgumentParser(description='''Simple example of training script tracking peak GPU memory usage.'''   )
						parser.add_argument(
						    '''--model_name_or_path''',			type=SCREAMING_SNAKE_CASE__,			default='''bert-base-cased''',			help='''Path to pretrained model or model identifier from huggingface.co/models.''',			required=SCREAMING_SNAKE_CASE__,			)
						parser.add_argument(
						    '''--output_dir''',			type=SCREAMING_SNAKE_CASE__,			default='''.''',			help='''Optional save directory where all checkpoint folders will be stored. Default is the current working directory.''',			)
						parser.add_argument(
						    '''--resume_from_checkpoint''',			type=SCREAMING_SNAKE_CASE__,			default=SCREAMING_SNAKE_CASE__,			help='''If the training should continue from a checkpoint folder.''',			)
						parser.add_argument(
						    '''--partial_train_epoch''',			type=SCREAMING_SNAKE_CASE__,			default=SCREAMING_SNAKE_CASE__,			help='''If passed, the training will stop after this number of epochs.''',			)
						parser.add_argument(
						    '''--num_epochs''',			type=SCREAMING_SNAKE_CASE__,			default=2,			help='''Number of train epochs.''',			)
						UpperCAmelCase_			:      Optional[int]     						=  parser.parse_args()
						UpperCAmelCase_			:      List[Any]     						=  {'''lr''': 2E-5, '''num_epochs''': args.num_epochs, '''seed''': 42, '''batch_size''': 16}

						training_function(SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__   )


if __name__ == "__main__":
					main()





