



from math import sqrt





def 		a__			(		_UpperCamelCase			: int							):
   assert isinstance(_UpperCamelCase ,_UpperCamelCase							) and (
       number >= 0
   ), "'number' must been an int and positive"

   __lowerCamelCase													=    True

   # 0 and 1 are none primes.
   if number <= 1:
      __lowerCamelCase													=    False

   for divisor in range(2 ,int(round(sqrt(_UpperCamelCase							)							)							) + 1							):
      # if 'number' divisible by 'divisor' then sets 'status'
      # of false and break up the loop.
      if number % divisor == 0:
         __lowerCamelCase													=    False
         break

    # precondition
   assert isinstance(_UpperCamelCase ,_UpperCamelCase							), "'status' must been from type bool"

   return status





def 		a__			(		_UpperCamelCase			: Any							):
   assert isinstance(_UpperCamelCase ,_UpperCamelCase							) and (n > 2), "'N' must been an int and > 2"

   # beginList: contains all natural numbers from 2 up to N
   __lowerCamelCase													=    list(range(2 ,n + 1							)							)

   __lowerCamelCase													=    []  # this list will be returns.

   # actual sieve of erathostenes
   for i in range(len(_UpperCamelCase							)							):
      for j in range(i + 1 ,len(_UpperCamelCase							)							):
         if (begin_list[i] != 0) and (begin_list[j] % begin_list[i] == 0):
            __lowerCamelCase													=    0

    # filters actual prime numbers.
   __lowerCamelCase													=    [x for x in begin_list if x != 0]

   # precondition
   assert isinstance(_UpperCamelCase ,_UpperCamelCase							), "'ans' must been from type list"

   return ans





def 		a__			(		_UpperCamelCase			: List[Any]							):
   assert isinstance(_UpperCamelCase ,_UpperCamelCase							) and (n > 2), "'N' must been an int and > 2"

   __lowerCamelCase													=    []

   # iterates over all numbers between 2 up to N+1
   # if a number is prime then appends to list 'ans'
   for number in range(2 ,n + 1							):
      if is_prime(_UpperCamelCase							):
         ans.append(_UpperCamelCase							)

    # precondition
   assert isinstance(_UpperCamelCase ,_UpperCamelCase							), "'ans' must been from type list"

   return ans





def 		a__			(		_UpperCamelCase			: Dict							):
   assert isinstance(_UpperCamelCase ,_UpperCamelCase							) and number >= 0, "'number' must been an int and >= 0"

   __lowerCamelCase													=    []  # this list will be returns of the function.

   # potential prime number factors.

   __lowerCamelCase													=    2

   __lowerCamelCase													=    number

   if number == 0 or number == 1:
      ans.append(_UpperCamelCase							)

   # if 'number' not prime then builds the prime factorization of 'number'
   elif not is_prime(_UpperCamelCase							):
      while quotient != 1:
         if is_prime(_UpperCamelCase							) and (quotient % factor == 0):
            ans.append(_UpperCamelCase							)
            quotient /= factor
         else:
            factor += 1

   else:
      ans.append(_UpperCamelCase							)

   # precondition
   assert isinstance(_UpperCamelCase ,_UpperCamelCase							), "'ans' must been from type list"

   return ans





def 		a__			(		_UpperCamelCase			: Tuple							):
   assert isinstance(_UpperCamelCase ,_UpperCamelCase							) and (
       number >= 0
   ), "'number' bust been an int and >= 0"

   __lowerCamelCase													=    0

   # prime factorization of 'number'
   __lowerCamelCase													=    prime_factorization(_UpperCamelCase							)

   __lowerCamelCase													=    max(_UpperCamelCase							)

   # precondition
   assert isinstance(_UpperCamelCase ,_UpperCamelCase							), "'ans' must been from type int"

   return ans





def 		a__			(		_UpperCamelCase			: int							):
   assert isinstance(_UpperCamelCase ,_UpperCamelCase							) and (
       number >= 0
   ), "'number' bust been an int and >= 0"

   __lowerCamelCase													=    0

   # prime factorization of 'number'
   __lowerCamelCase													=    prime_factorization(_UpperCamelCase							)

   __lowerCamelCase													=    min(_UpperCamelCase							)

   # precondition
   assert isinstance(_UpperCamelCase ,_UpperCamelCase							), "'ans' must been from type int"

   return ans





def 		a__			(		_UpperCamelCase			: int							):
   assert isinstance(_UpperCamelCase ,_UpperCamelCase							), "'number' must been an int"
   assert isinstance(number % 2 == 0 ,_UpperCamelCase							), "compare bust been from type bool"

   return number % 2 == 0





def 		a__			(		_UpperCamelCase			: str							):
   assert isinstance(_UpperCamelCase ,_UpperCamelCase							), "'number' must been an int"
   assert isinstance(number % 2 != 0 ,_UpperCamelCase							), "compare bust been from type bool"

   return number % 2 != 0





def 		a__			(		_UpperCamelCase			: List[Any]							):
   assert (
       isinstance(_UpperCamelCase ,_UpperCamelCase							) and (number > 2) and is_even(_UpperCamelCase							)
   ), "'number' must been an int, even and > 2"

   __lowerCamelCase													=    []  # this list will returned

   # creates a list of prime numbers between 2 up to 'number'
   __lowerCamelCase													=    get_prime_numbers(_UpperCamelCase							)
   __lowerCamelCase													=    len(_UpperCamelCase							)

   # run variable for while-loops.
   __lowerCamelCase													=    0
   __lowerCamelCase													=    None

   # exit variable. for break up the loops
   __lowerCamelCase													=    True

   while i < len_pn and loop:
      __lowerCamelCase													=    i + 1

      while j < len_pn and loop:
         if prime_numbers[i] + prime_numbers[j] == number:
            __lowerCamelCase													=    False
            ans.append(prime_numbers[i]							)
            ans.append(prime_numbers[j]							)

         j += 1

      i += 1

   # precondition
   assert (
       isinstance(_UpperCamelCase ,_UpperCamelCase							)
       and (len(_UpperCamelCase							) == 2)
       and (ans[0] + ans[1] == number)
       and is_prime(ans[0]							)
       and is_prime(ans[1]							)
   ), "'ans' must contains two primes. And sum of elements must been eq 'number'"

   return ans





def 		a__			(		_UpperCamelCase			: Tuple ,_UpperCamelCase			: Optional[Any]							):
   assert (
       isinstance(_UpperCamelCase ,_UpperCamelCase							)
       and isinstance(_UpperCamelCase ,_UpperCamelCase							)
       and (numbera >= 0)
       and (numbera >= 0)
   ), "'number1' and 'number2' must been positive integer."

   __lowerCamelCase													=    0

   while numbera != 0:
      __lowerCamelCase													=    numbera % numbera
      __lowerCamelCase													=    numbera
      __lowerCamelCase													=    rest

   # precondition
   assert isinstance(_UpperCamelCase ,_UpperCamelCase							) and (
       numbera >= 0
   ), "'number' must been from type int and positive"

   return numbera





def 		a__			(		_UpperCamelCase			: Dict ,_UpperCamelCase			: Union[str, Any]							):
   assert (
       isinstance(_UpperCamelCase ,_UpperCamelCase							)
       and isinstance(_UpperCamelCase ,_UpperCamelCase							)
       and (numbera >= 1)
       and (numbera >= 1)
   ), "'number1' and 'number2' must been positive integer."

   __lowerCamelCase													=    1  # actual answer that will be return.

   # for kgV (x,1)
   if numbera > 1 and numbera > 1:
      # builds the prime factorization of 'number1' and 'number2'
      __lowerCamelCase													=    prime_factorization(_UpperCamelCase							)
      __lowerCamelCase													=    prime_factorization(_UpperCamelCase							)

   elif numbera == 1 or numbera == 1:
      __lowerCamelCase													=    []
      __lowerCamelCase													=    []
      __lowerCamelCase													=    max(_UpperCamelCase ,_UpperCamelCase							)

   __lowerCamelCase													=    0
   __lowerCamelCase													=    0

   __lowerCamelCase													=    []  # captured numbers int both 'primeFac1' and 'primeFac2'

   # iterates through primeFac1
   for n in prime_fac_a:
      if n not in done:
         if n in prime_fac_a:
            __lowerCamelCase													=    prime_fac_a.count(_UpperCamelCase							)
            __lowerCamelCase													=    prime_fac_a.count(_UpperCamelCase							)

            for _ in range(max(_UpperCamelCase ,_UpperCamelCase							)							):
               ans *= n

         else:
            __lowerCamelCase													=    prime_fac_a.count(_UpperCamelCase							)

            for _ in range(_UpperCamelCase							):
               ans *= n

         done.append(_UpperCamelCase							)

    # iterates through primeFac2
   for n in prime_fac_a:
      if n not in done:
         __lowerCamelCase													=    prime_fac_a.count(_UpperCamelCase							)

         for _ in range(_UpperCamelCase							):
            ans *= n

         done.append(_UpperCamelCase							)

    # precondition
   assert isinstance(_UpperCamelCase ,_UpperCamelCase							) and (
       ans >= 0
   ), "'ans' must been from type int and positive"

   return ans





def 		a__			(		_UpperCamelCase			: Optional[int]							):
   assert isinstance(_UpperCamelCase ,_UpperCamelCase							) and (n >= 0), "'number' must been a positive int"

   __lowerCamelCase													=    0
   __lowerCamelCase													=    2  # this variable holds the answer

   while index < n:
      index += 1

      ans += 1  # counts to the next number

      # if ans not prime then
      # runs to the next prime number.
      while not is_prime(_UpperCamelCase							):
         ans += 1

    # precondition
   assert isinstance(_UpperCamelCase ,_UpperCamelCase							) and is_prime(
       _UpperCamelCase							), "'ans' must been a prime number and from type int"

   return ans





def 		a__			(		_UpperCamelCase			: str ,_UpperCamelCase			: Union[str, Any]							):
   assert (
       is_prime(_UpperCamelCase							) and is_prime(_UpperCamelCase							) and (p_number_a < p_number_a)
   ), "The arguments must been prime numbers and 'pNumber1' < 'pNumber2'"

   __lowerCamelCase													=    p_number_a + 1  # jump to the next number

   __lowerCamelCase													=    []  # this list will be returns.

   # if number is not prime then
   # fetch the next prime number.
   while not is_prime(_UpperCamelCase							):
      number += 1

   while number < p_number_a:
      ans.append(_UpperCamelCase							)

      number += 1

      # fetch the next prime number.
      while not is_prime(_UpperCamelCase							):
         number += 1

    # precondition
   assert (
       isinstance(_UpperCamelCase ,_UpperCamelCase							)
       and ans[0] != p_number_a
       and ans[len(_UpperCamelCase							) - 1] != p_number_a
   ), "'ans' must been a list without the arguments"

   # 'ans' contains not 'pNumber1' and 'pNumber2' !
   return ans





def 		a__			(		_UpperCamelCase			: str							):
   assert isinstance(_UpperCamelCase ,_UpperCamelCase							) and (n >= 1), "'n' must been int and >= 1"

   __lowerCamelCase													=    []  # will be returned.

   for divisor in range(1 ,n + 1							):
      if n % divisor == 0:
         ans.append(_UpperCamelCase							)

    # precondition
   assert ans[0] == 1 and ans[len(_UpperCamelCase							) - 1] == n, "Error in function getDivisiors(...)"

   return ans





def 		a__			(		_UpperCamelCase			: str							):
   assert isinstance(_UpperCamelCase ,_UpperCamelCase							) and (
       number > 1
   ), "'number' must been an int and >= 1"

   __lowerCamelCase													=    get_divisors(_UpperCamelCase							)

   # precondition
   assert (
       isinstance(_UpperCamelCase ,_UpperCamelCase							)
       and (divisors[0] == 1)
       and (divisors[len(_UpperCamelCase							) - 1] == number)
   ), "Error in help-function getDivisiors(...)"

   # summed all divisors up to 'number' (exclusive), hence [:-1]
   return sum(divisors[:-1]							) == number





def 		a__			(		_UpperCamelCase			: List[str] ,_UpperCamelCase			: Optional[int]							):
   assert (
       isinstance(_UpperCamelCase ,_UpperCamelCase							)
       and isinstance(_UpperCamelCase ,_UpperCamelCase							)
       and (denominator != 0)
   ), "The arguments must been from type int and 'denominator' != 0"

   # build the greatest common divisor of numerator and denominator.
   __lowerCamelCase													=    gcd(abs(_UpperCamelCase							) ,abs(_UpperCamelCase							)							)

   # precondition
   assert (
       isinstance(_UpperCamelCase ,_UpperCamelCase							)
       and (numerator % gcd_of_fraction == 0)
       and (denominator % gcd_of_fraction == 0)
   ), "Error in function gcd(...,...)"

   return (numerator // gcd_of_fraction, denominator // gcd_of_fraction)





def 		a__			(		_UpperCamelCase			: Dict							):
   assert isinstance(_UpperCamelCase ,_UpperCamelCase							) and (n >= 0), "'n' must been a int and >= 0"

   __lowerCamelCase													=    1  # this will be return.

   for factor in range(1 ,n + 1							):
      ans *= factor

   return ans





def 		a__			(		_UpperCamelCase			: Optional[Any]							):
   assert isinstance(_UpperCamelCase ,_UpperCamelCase							) and (n >= 0), "'n' must been an int and >= 0"

   __lowerCamelCase													=    0
   __lowerCamelCase													=    1
   __lowerCamelCase													=    1  # this will be return

   for _ in range(n - 1							):
      __lowerCamelCase													=    ans
      ans += fiba
      __lowerCamelCase													=    tmp

   return ans











import argparse
import csv
import logging
import os
import random

import numpy as np
import torch
from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, TensorDataset
from tqdm import tqdm, trange

from transformers import (
    CONFIG_NAME,
    WEIGHTS_NAME,
    AdamW,
    OpenAIGPTDoubleHeadsModel,
    OpenAIGPTTokenizer,
    get_linear_schedule_with_warmup,
)


logging.basicConfig(
    format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s""", datefmt="""%m/%d/%Y %H:%M:%S""", level=logging.INFO
)
a_			=					logging.getLogger(__name__)





def 		a__			(		_UpperCamelCase			: str ,_UpperCamelCase			: List[Any]							):
   __lowerCamelCase													=    np.argmax(_UpperCamelCase ,axis=1							)
   return np.sum(outputs == labels							)





def 		a__			(		_UpperCamelCase			: Optional[int]							):
   with open(_UpperCamelCase ,encoding='''utf_8'''							) as f:
      __lowerCamelCase													=    csv.reader(_UpperCamelCase							)
      __lowerCamelCase													=    []
      next(_UpperCamelCase							)  # skip the first line
      for line in tqdm(_UpperCamelCase							):
         output.append((''' '''.join(line[1:5]							), line[5], line[6], int(line[-1]							) - 1)							)
   return output





def 		a__			(		_UpperCamelCase			: Any ,_UpperCamelCase			: Dict ,_UpperCamelCase			: str ,_UpperCamelCase			: Union[str, Any] ,_UpperCamelCase			: Any ,_UpperCamelCase			: Dict							):
   __lowerCamelCase													=    []
   for dataset in encoded_datasets:
      __lowerCamelCase													=    len(_UpperCamelCase							)
      __lowerCamelCase													=    np.zeros((n_batch, 2, input_len) ,dtype=np.intaa							)
      __lowerCamelCase													=    np.zeros((n_batch, 2) ,dtype=np.intaa							)
      __lowerCamelCase													=    np.full((n_batch, 2, input_len) ,fill_value=-1_00 ,dtype=np.intaa							)
      __lowerCamelCase													=    np.zeros((n_batch,) ,dtype=np.intaa							)
      for (
          i,
          (story, conta, conta, mc_label),
      ) in enumerate(_UpperCamelCase							):
         __lowerCamelCase													=    [start_token] + story[:cap_length] + [delimiter_token] + conta[:cap_length] + [clf_token]
         __lowerCamelCase													=    [start_token] + story[:cap_length] + [delimiter_token] + conta[:cap_length] + [clf_token]
         __lowerCamelCase													=    with_conta
         __lowerCamelCase													=    with_conta
         __lowerCamelCase													=    len(_UpperCamelCase							) - 1
         __lowerCamelCase													=    len(_UpperCamelCase							) - 1
         __lowerCamelCase													=    with_conta
         __lowerCamelCase													=    with_conta
         __lowerCamelCase													=    mc_label
      __lowerCamelCase													=    (input_ids, mc_token_ids, lm_labels, mc_labels)
      tensor_datasets.append(tuple(torch.tensor(_UpperCamelCase							) for t in all_inputs							)							)
   return tensor_datasets





def 		a__			(		):
   __lowerCamelCase													=    argparse.ArgumentParser()
   parser.add_argument('''--model_name''' ,type=_UpperCamelCase ,default='''openai-gpt''' ,help='''pretrained model name'''							)
   parser.add_argument('''--do_train''' ,action='''store_true''' ,help='''Whether to run training.'''							)
   parser.add_argument('''--do_eval''' ,action='''store_true''' ,help='''Whether to run eval on the dev set.'''							)
   parser.add_argument(
       '''--output_dir''' ,default=_UpperCamelCase ,type=_UpperCamelCase ,required=_UpperCamelCase ,help='''The output directory where the model predictions and checkpoints will be written.''' ,)
   parser.add_argument('''--train_dataset''' ,type=_UpperCamelCase ,default=''''''							)
   parser.add_argument('''--eval_dataset''' ,type=_UpperCamelCase ,default=''''''							)
   parser.add_argument('''--seed''' ,type=_UpperCamelCase ,default=42							)
   parser.add_argument('''--num_train_epochs''' ,type=_UpperCamelCase ,default=3							)
   parser.add_argument('''--train_batch_size''' ,type=_UpperCamelCase ,default=8							)
   parser.add_argument('''--eval_batch_size''' ,type=_UpperCamelCase ,default=16							)
   parser.add_argument('''--adam_epsilon''' ,default=1e-8 ,type=_UpperCamelCase ,help='''Epsilon for Adam optimizer.'''							)
   parser.add_argument('''--max_grad_norm''' ,type=_UpperCamelCase ,default=1							)
   parser.add_argument(
       '''--max_steps''' ,default=-1 ,type=_UpperCamelCase ,help=(
           '''If > 0: set total number of training                         steps to perform. Override num_train_epochs.'''
       ) ,)
   parser.add_argument(
       '''--gradient_accumulation_steps''' ,type=_UpperCamelCase ,default=1 ,help='''Number of updates steps to accumulate before                        performing a backward/update pass.''' ,)
   parser.add_argument('''--learning_rate''' ,type=_UpperCamelCase ,default=6.25e-5							)
   parser.add_argument('''--warmup_steps''' ,default=0 ,type=_UpperCamelCase ,help='''Linear warmup over warmup_steps.'''							)
   parser.add_argument('''--lr_schedule''' ,type=_UpperCamelCase ,default='''warmup_linear'''							)
   parser.add_argument('''--weight_decay''' ,type=_UpperCamelCase ,default=0.01							)
   parser.add_argument('''--lm_coef''' ,type=_UpperCamelCase ,default=0.9							)
   parser.add_argument('''--n_valid''' ,type=_UpperCamelCase ,default=3_74							)

   parser.add_argument('''--server_ip''' ,type=_UpperCamelCase ,default='''''' ,help='''Can be used for distant debugging.'''							)
   parser.add_argument('''--server_port''' ,type=_UpperCamelCase ,default='''''' ,help='''Can be used for distant debugging.'''							)
   __lowerCamelCase													=    parser.parse_args()
   print(_UpperCamelCase							)

   if args.server_ip and args.server_port:
      # Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
      import ptvsd

      print('''Waiting for debugger attach'''							)
      ptvsd.enable_attach(address=(args.server_ip, args.server_port) ,redirect_output=_UpperCamelCase							)
      ptvsd.wait_for_attach()

   random.seed(args.seed							)
   np.random.seed(args.seed							)
   torch.manual_seed(args.seed							)
   torch.cuda.manual_seed_all(args.seed							)

   __lowerCamelCase													=    torch.device('''cuda''' if torch.cuda.is_available() else '''cpu'''							)
   __lowerCamelCase													=    torch.cuda.device_count()
   logger.info('''device: {}, n_gpu {}'''.format(_UpperCamelCase ,_UpperCamelCase							)							)

   if not args.do_train and not args.do_eval:
      raise ValueError('''At least one of `do_train` or `do_eval` must be True.'''							)

   if not os.path.exists(args.output_dir							):
      os.makedirs(args.output_dir							)

   # Load tokenizer and model
   # This loading functions also add new tokens and embeddings called `special tokens`
   # These new embeddings will be fine-tuned on the RocStories dataset
   __lowerCamelCase													=    ['''_start_''', '''_delimiter_''', '''_classify_''']
   __lowerCamelCase													=    OpenAIGPTTokenizer.from_pretrained(args.model_name							)
   tokenizer.add_tokens(_UpperCamelCase							)
   __lowerCamelCase													=    tokenizer.convert_tokens_to_ids(_UpperCamelCase							)
   __lowerCamelCase													=    OpenAIGPTDoubleHeadsModel.from_pretrained(args.model_name							)
   model.resize_token_embeddings(len(_UpperCamelCase							)							)
   model.to(_UpperCamelCase							)

   # Load and encode the datasets
   def tokenize_and_encode(_UpperCamelCase			: Dict							):
      if isinstance(_UpperCamelCase ,_UpperCamelCase							):
         return tokenizer.convert_tokens_to_ids(tokenizer.tokenize(_UpperCamelCase							)							)
      elif isinstance(_UpperCamelCase ,_UpperCamelCase							):
         return obj
      return [tokenize_and_encode(_UpperCamelCase							) for o in obj]

   logger.info('''Encoding dataset...'''							)
   __lowerCamelCase													=    load_rocstories_dataset(args.train_dataset							)
   __lowerCamelCase													=    load_rocstories_dataset(args.eval_dataset							)
   __lowerCamelCase													=    (train_dataset, eval_dataset)
   __lowerCamelCase													=    tokenize_and_encode(_UpperCamelCase							)

   # Compute the max input length for the Transformer
   __lowerCamelCase													=    model.config.n_positions // 2 - 2
   __lowerCamelCase													=    max(
       len(story[:max_length]							) + max(len(conta[:max_length]							) ,len(conta[:max_length]							)							) + 3
       for dataset in encoded_datasets
       for story, conta, conta, _ in dataset							)
   __lowerCamelCase													=    min(_UpperCamelCase ,model.config.n_positions							)  # Max size of input for the pre-trained model

   # Prepare inputs tensors and dataloaders
   __lowerCamelCase													=    pre_process_datasets(_UpperCamelCase ,_UpperCamelCase ,_UpperCamelCase ,*_UpperCamelCase							)
   __lowerCamelCase							,__lowerCamelCase													=    tensor_datasets[0], tensor_datasets[1]

   __lowerCamelCase													=    TensorDataset(*_UpperCamelCase							)
   __lowerCamelCase													=    RandomSampler(_UpperCamelCase							)
   __lowerCamelCase													=    DataLoader(_UpperCamelCase ,sampler=_UpperCamelCase ,batch_size=args.train_batch_size							)

   __lowerCamelCase													=    TensorDataset(*_UpperCamelCase							)
   __lowerCamelCase													=    SequentialSampler(_UpperCamelCase							)
   __lowerCamelCase													=    DataLoader(_UpperCamelCase ,sampler=_UpperCamelCase ,batch_size=args.eval_batch_size							)

   # Prepare optimizer
   if args.do_train:
      if args.max_steps > 0:
         __lowerCamelCase													=    args.max_steps
         __lowerCamelCase													=    args.max_steps // (len(_UpperCamelCase							) // args.gradient_accumulation_steps) + 1
      else:
         __lowerCamelCase													=    len(_UpperCamelCase							) // args.gradient_accumulation_steps * args.num_train_epochs

      __lowerCamelCase													=    list(model.named_parameters()							)
      __lowerCamelCase													=    ['''bias''', '''LayerNorm.bias''', '''LayerNorm.weight''']
      __lowerCamelCase													=    [
          {
              '''params''': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay							)],
              '''weight_decay''': args.weight_decay,
          },
          {'''params''': [p for n, p in param_optimizer if any(nd in n for nd in no_decay							)], '''weight_decay''': 0.0},
      ]
      __lowerCamelCase													=    AdamW(_UpperCamelCase ,lr=args.learning_rate ,eps=args.adam_epsilon							)
      __lowerCamelCase													=    get_linear_schedule_with_warmup(
          _UpperCamelCase ,num_warmup_steps=args.warmup_steps ,num_training_steps=_UpperCamelCase							)

   if args.do_train:
      __lowerCamelCase							,__lowerCamelCase							,__lowerCamelCase													=    0, 0, None
      model.train()
      for _ in trange(int(args.num_train_epochs							) ,desc='''Epoch'''							):
         __lowerCamelCase													=    0
         __lowerCamelCase													=    0
         __lowerCamelCase													=    tqdm(_UpperCamelCase ,desc='''Training'''							)
         for step, batch in enumerate(_UpperCamelCase							):
            __lowerCamelCase													=    tuple(t.to(_UpperCamelCase							) for t in batch							)
            __lowerCamelCase							,__lowerCamelCase							,__lowerCamelCase							,__lowerCamelCase													=    batch
            __lowerCamelCase													=    model(_UpperCamelCase ,mc_token_ids=_UpperCamelCase ,lm_labels=_UpperCamelCase ,mc_labels=_UpperCamelCase							)
            __lowerCamelCase													=    args.lm_coef * losses[0] + losses[1]
            loss.backward()
            optimizer.step()
            scheduler.step()
            optimizer.zero_grad()
            tr_loss += loss.item()
            __lowerCamelCase													=    (
                loss.item() if exp_average_loss is None else 0.7 * exp_average_loss + 0.3 * loss.item()
            )
            nb_tr_steps += 1
            __lowerCamelCase													=    '''Training loss: {:.2e} lr: {:.2e}'''.format(_UpperCamelCase ,scheduler.get_lr()[0]							)

    # Save a trained model
   if args.do_train:
      # Save a trained model, configuration and tokenizer
      __lowerCamelCase													=    model.module if hasattr(_UpperCamelCase ,'''module'''							) else model  # Only save the model itself

      # If we save using the predefined names, we can load using `from_pretrained`
      __lowerCamelCase													=    os.path.join(args.output_dir ,_UpperCamelCase							)
      __lowerCamelCase													=    os.path.join(args.output_dir ,_UpperCamelCase							)

      torch.save(model_to_save.state_dict() ,_UpperCamelCase							)
      model_to_save.config.to_json_file(_UpperCamelCase							)
      tokenizer.save_vocabulary(args.output_dir							)

      # Load a trained model and vocabulary that you have fine-tuned
      __lowerCamelCase													=    OpenAIGPTDoubleHeadsModel.from_pretrained(args.output_dir							)
      __lowerCamelCase													=    OpenAIGPTTokenizer.from_pretrained(args.output_dir							)
      model.to(_UpperCamelCase							)

   if args.do_eval:
      model.eval()
      __lowerCamelCase							,__lowerCamelCase													=    0, 0
      __lowerCamelCase							,__lowerCamelCase													=    0, 0
      for batch in tqdm(_UpperCamelCase ,desc='''Evaluating'''							):
         __lowerCamelCase													=    tuple(t.to(_UpperCamelCase							) for t in batch							)
         __lowerCamelCase							,__lowerCamelCase							,__lowerCamelCase							,__lowerCamelCase													=    batch
         with torch.no_grad():
            __lowerCamelCase							,__lowerCamelCase							,__lowerCamelCase							,__lowerCamelCase													=    model(
                _UpperCamelCase ,mc_token_ids=_UpperCamelCase ,lm_labels=_UpperCamelCase ,mc_labels=_UpperCamelCase							)

         __lowerCamelCase													=    mc_logits.detach().cpu().numpy()
         __lowerCamelCase													=    mc_labels.to('''cpu'''							).numpy()
         __lowerCamelCase													=    accuracy(_UpperCamelCase ,_UpperCamelCase							)

         eval_loss += mc_loss.mean().item()
         eval_accuracy += tmp_eval_accuracy

         nb_eval_examples += input_ids.size(0							)
         nb_eval_steps += 1

      __lowerCamelCase													=    eval_loss / nb_eval_steps
      __lowerCamelCase													=    eval_accuracy / nb_eval_examples
      __lowerCamelCase													=    tr_loss / nb_tr_steps if args.do_train else None
      __lowerCamelCase													=    {'''eval_loss''': eval_loss, '''eval_accuracy''': eval_accuracy, '''train_loss''': train_loss}

      __lowerCamelCase													=    os.path.join(args.output_dir ,'''eval_results.txt'''							)
      with open(_UpperCamelCase ,'''w'''							) as writer:
         logger.info('''***** Eval results *****'''							)
         for key in sorted(result.keys()							):
            logger.info('''  %s = %s''' ,_UpperCamelCase ,str(result[key]							)							)
            writer.write('''%s = %s\n''' % (key, str(result[key]							))							)


if __name__ == "__main__":
  main()






