




'''simple docstring'''
import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


UpperCamelCase__ =			logging.getLogger(__name__)




def 		__SCREAMING_SNAKE_CASE					(  ):


   """simple docstring"""




   lowercase_	:    int           =     argparse.ArgumentParser(
       description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."							)
   parser.add_argument(
       "--dataset_name"						,		type=_UpperCamelCase						,		default="wikitext"						,		help="Name of the training. Explore datasets at: hf.co/datasets."						,		)
   parser.add_argument(
       "--dataset_config"						,		type=_UpperCamelCase						,		default="wikitext-103-raw-v1"						,		help="Configuration name of the dataset."							)
   parser.add_argument(
       "--tokenizer_name_or_path"						,		type=_UpperCamelCase						,		default="sayakpaul/unigram-tokenizer-wikitext"						,		help="Tokenizer identifier. Can be a local filepath or a Hub identifier."						,		)
   parser.add_argument(
       "--shard_size"						,		type=_UpperCamelCase						,		default=1000						,		help="Number of entries to go in a single shard."						,		)
   parser.add_argument("--split"						,		type=_UpperCamelCase						,		default="train"						,		choices=["train", "test", "validation"]							)
   parser.add_argument(
       "--limit"						,		default=_UpperCamelCase						,		type=_UpperCamelCase						,		help="Limit the number of shards (used for debugging)."						,		)
   parser.add_argument(
       "--max_length"						,		type=_UpperCamelCase						,		default=512						,		help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
       " sequence length that is a multiple of 8."						,		)
   parser.add_argument(
       "--output_dir"						,		default="tf-tpu"						,		type=_UpperCamelCase						,		help="Output directory where the TFRecord shards will be saved. If the"
       " path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord"
       " shards will be directly saved to a Google Cloud Storage bucket."						,		)

   lowercase_	:    Optional[Any]           =     parser.parse_args()
   return args




def 		__SCREAMING_SNAKE_CASE					(  _UpperCamelCase							):


   """simple docstring"""




   def fn(_UpperCamelCase							):
      return tokenizer(examples["text"]							)

   return fn




def 		__SCREAMING_SNAKE_CASE					(  _UpperCamelCase							):


   """simple docstring"""




   lowercase_	:    Tuple           =     []
   for i in range(len(tokenized_data["input_ids"]							)							):
      lowercase_	:    int           =     {
          "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]							)							),
          "attention_mask": tf.train.Feature(
              intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]							)							),
      }
      lowercase_	:    Optional[int]           =     tf.train.Features(feature=_UpperCamelCase							)
      lowercase_	:    Tuple           =     tf.train.Example(features=_UpperCamelCase							)
      lowercase_	:    List[str]           =     example.SerializeToString()
      records.append(_UpperCamelCase							)
   return records




def 		__SCREAMING_SNAKE_CASE					(  _UpperCamelCase							):


   """simple docstring"""




   lowercase_	:    int           =     datasets.load_dataset(args.dataset_name						,		args.dataset_config						,		split=args.split							)

   if args.limit is not None:
      lowercase_	:    Dict           =     min(len(_UpperCamelCase							)						,		args.limit							)
      lowercase_	:    str           =     dataset.select(range(_UpperCamelCase							)							)
      print(F"""Limiting the dataset to {args.limit} entries."""							)

   lowercase_	:    str           =     AutoTokenizer.from_pretrained(args.tokenizer_name_or_path							)

   # Handle output directory creation.
   # For serializing into a Google Cloud Storage Bucket, one needs to first
   # create a bucket.
   if "gs" not in args.output_dir:
      if not os.path.exists(args.output_dir							):
         os.makedirs(args.output_dir							)
      lowercase_	:    Optional[Any]           =     os.path.join(args.output_dir						,		args.split							)
      if not os.path.exists(_UpperCamelCase							):
         os.makedirs(_UpperCamelCase							)
   else:
      lowercase_	:    int           =     os.path.join(args.output_dir						,		args.split							)

   # Tokenize the whole dataset at once.
   lowercase_	:    Dict           =     tokenize_function(_UpperCamelCase							)
   lowercase_	:    Optional[Any]           =     dataset.map(_UpperCamelCase						,		batched=_UpperCamelCase						,		num_proc=4						,		remove_columns=["text"]							)

   # We need to concatenate all our texts together, and then split the result
   # into chunks of a fixed size, which we will call block_size. To do this, we
   # will use the map method again, with the option batched=True. When we use batched=True,
   # the function we pass to map() will be passed multiple inputs at once, allowing us
   # to group them into more or fewer examples than we had in the input.
   # This allows us to create our new fixed-length samples. The advantage of this
   # method is that we don't lose a whole lot of content from the dataset compared to the
   # case where we simply tokenize with a pre-defined max_length.

   def group_texts(_UpperCamelCase							):
      # Concatenate all texts.
      lowercase_	:    Dict           =     {k: sum(examples[k]						,		[]							) for k in examples.keys()}
      lowercase_	:    Optional[Any]           =     len(concatenated_examples[list(examples.keys()							)[0]]							)
      # We drop the small remainder, though you could add padding instead if the model supports it
      # In this, as in all things, we advise you to follow your heart 🫀
      lowercase_	:    Dict           =     (total_length // args.max_length) * args.max_length
      # Split by chunks of max_len.
      lowercase_	:    Union[str, Any]           =     {
          k: [t[i : i + args.max_length] for i in range(0						,		_UpperCamelCase						,		args.max_length							)]
          for k, t in concatenated_examples.items()
      }
      return result

   lowercase_	:    Tuple           =     dataset_tokenized.map(_UpperCamelCase						,		batched=_UpperCamelCase						,		batch_size=1000						,		num_proc=4							)

   lowercase_	:    List[Any]           =     0
   lowercase_	:    Tuple           =     0
   for shard in range(0						,		len(_UpperCamelCase							)						,		args.shard_size							):
      lowercase_	:    Union[str, Any]           =     grouped_dataset[shard : shard + args.shard_size]
      lowercase_	:    Union[str, Any]           =     len(dataset_snapshot["input_ids"]							)
      lowercase_	:    str           =     os.path.join(_UpperCamelCase						,		F"""dataset-{shard_count}-{records_containing}.tfrecord"""							)
      lowercase_	:    str           =     get_serialized_examples(_UpperCamelCase							)

      with tf.io.TFRecordWriter(_UpperCamelCase							) as out_file:
         for i in range(len(_UpperCamelCase							)							):
            lowercase_	:    Optional[int]           =     serialized_examples[i]
            out_file.write(_UpperCamelCase							)
         print("Wrote file {} containing {} records".format(_UpperCamelCase						,		_UpperCamelCase							)							)

      shard_count += 1
      total_records += records_containing

   with open(F"""split-{args.split}-records-count.txt"""						,		"w"							) as f:
      print(F"""Total {args.split} records: {total_records}"""						,		file=_UpperCamelCase							)


if __name__ == "__main__":
 UpperCamelCase__ =			parse_args()
 main(args)












'''simple docstring'''
import math
from enum import Enum
from typing import Optional, Union

from torch.optim import Optimizer
from torch.optim.lr_scheduler import LambdaLR

from .utils import logging


UpperCamelCase__ =			logging.get_logger(__name__)


class        _UpperCAmelCase						(		snake_case			):
    __lowerCamelCase:   Tuple		       =    'linear'
    __lowerCamelCase:   Any		       =    'cosine'
    __lowerCamelCase:   Optional[Any]		       =    'cosine_with_restarts'
    __lowerCamelCase:   Tuple		       =    'polynomial'
    __lowerCamelCase:   int		       =    'constant'
    __lowerCamelCase:   Optional[Any]		       =    'constant_with_warmup'
    __lowerCamelCase:   List[str]		       =    'piecewise_constant'




def 		__SCREAMING_SNAKE_CASE					(  _UpperCamelCase						,		_UpperCamelCase = -1							):


   """simple docstring"""




   return LambdaLR(_UpperCamelCase						,		lambda _UpperCamelCase							: 1						,		last_epoch=_UpperCamelCase							)




def 		__SCREAMING_SNAKE_CASE					(  _UpperCamelCase						,		_UpperCamelCase						,		_UpperCamelCase = -1							):


   """simple docstring"""




   def lr_lambda(_UpperCamelCase							):
      if current_step < num_warmup_steps:
         return float(_UpperCamelCase							) / float(max(1.0						,		_UpperCamelCase							)							)
      return 1.0

   return LambdaLR(_UpperCamelCase						,		_UpperCamelCase						,		last_epoch=_UpperCamelCase							)




def 		__SCREAMING_SNAKE_CASE					(  _UpperCamelCase						,		_UpperCamelCase						,		_UpperCamelCase = -1							):


   """simple docstring"""




   lowercase_	:    List[Any]           =     {}
   lowercase_	:    Dict           =     step_rules.split(","							)
   for rule_str in rule_list[:-1]:
      lowercase_    ,		lowercase_	:    Any           =     rule_str.split(":"							)
      lowercase_	:    List[Any]           =     int(_UpperCamelCase							)
      lowercase_	:    int           =     float(_UpperCamelCase							)
      lowercase_	:    Optional[int]           =     value
   lowercase_	:    Union[str, Any]           =     float(rule_list[-1]							)

   def create_rules_function(_UpperCamelCase						,		_UpperCamelCase							):
      def rule_func(_UpperCamelCase							) -> float:
         lowercase_	:    Optional[Any]           =     sorted(rules_dict.keys()							)
         for i, sorted_step in enumerate(_UpperCamelCase							):
            if steps < sorted_step:
               return rules_dict[sorted_steps[i]]
         return last_lr_multiple

      return rule_func

   lowercase_	:    Optional[int]           =     create_rules_function(_UpperCamelCase						,		_UpperCamelCase							)

   return LambdaLR(_UpperCamelCase						,		_UpperCamelCase						,		last_epoch=_UpperCamelCase							)




def 		__SCREAMING_SNAKE_CASE					(  _UpperCamelCase						,		_UpperCamelCase						,		_UpperCamelCase						,		_UpperCamelCase=-1							):


   """simple docstring"""




   def lr_lambda(_UpperCamelCase							):
      if current_step < num_warmup_steps:
         return float(_UpperCamelCase							) / float(max(1						,		_UpperCamelCase							)							)
      return max(
          0.0						,		float(num_training_steps - current_step							) / float(max(1						,		num_training_steps - num_warmup_steps							)							)							)

   return LambdaLR(_UpperCamelCase						,		_UpperCamelCase						,		_UpperCamelCase							)




def 		__SCREAMING_SNAKE_CASE					(  _UpperCamelCase						,		_UpperCamelCase						,		_UpperCamelCase						,		_UpperCamelCase = 0.5						,		_UpperCamelCase = -1							):


   """simple docstring"""




   def lr_lambda(_UpperCamelCase							):
      if current_step < num_warmup_steps:
         return float(_UpperCamelCase							) / float(max(1						,		_UpperCamelCase							)							)
      lowercase_	:    List[str]           =     float(current_step - num_warmup_steps							) / float(max(1						,		num_training_steps - num_warmup_steps							)							)
      return max(0.0						,		0.5 * (1.0 + math.cos(math.pi * float(_UpperCamelCase							) * 2.0 * progress							))							)

   return LambdaLR(_UpperCamelCase						,		_UpperCamelCase						,		_UpperCamelCase							)




def 		__SCREAMING_SNAKE_CASE					(  _UpperCamelCase						,		_UpperCamelCase						,		_UpperCamelCase						,		_UpperCamelCase = 1						,		_UpperCamelCase = -1							):


   """simple docstring"""




   def lr_lambda(_UpperCamelCase							):
      if current_step < num_warmup_steps:
         return float(_UpperCamelCase							) / float(max(1						,		_UpperCamelCase							)							)
      lowercase_	:    Dict           =     float(current_step - num_warmup_steps							) / float(max(1						,		num_training_steps - num_warmup_steps							)							)
      if progress >= 1.0:
         return 0.0
      return max(0.0						,		0.5 * (1.0 + math.cos(math.pi * ((float(_UpperCamelCase							) * progress) % 1.0)							))							)

   return LambdaLR(_UpperCamelCase						,		_UpperCamelCase						,		_UpperCamelCase							)




def 		__SCREAMING_SNAKE_CASE					(  _UpperCamelCase						,		_UpperCamelCase						,		_UpperCamelCase						,		_UpperCamelCase=1e-7						,		_UpperCamelCase=1.0						,		_UpperCamelCase=-1							):


   """simple docstring"""




   lowercase_	:    Dict           =     optimizer.defaults["lr"]
   if not (lr_init > lr_end):
      raise ValueError(F"""lr_end ({lr_end}) must be be smaller than initial lr ({lr_init})"""							)

   def lr_lambda(_UpperCamelCase							):
      if current_step < num_warmup_steps:
         return float(_UpperCamelCase							) / float(max(1						,		_UpperCamelCase							)							)
      elif current_step > num_training_steps:
         return lr_end / lr_init  # as LambdaLR multiplies by lr_init
      else:
         lowercase_	:    int           =     lr_init - lr_end
         lowercase_	:    Optional[int]           =     num_training_steps - num_warmup_steps
         lowercase_	:    Optional[Any]           =     1 - (current_step - num_warmup_steps) / decay_steps
         lowercase_	:    List[Any]           =     lr_range * pct_remaining**power + lr_end
         return decay / lr_init  # as LambdaLR multiplies by lr_init

   return LambdaLR(_UpperCamelCase						,		_UpperCamelCase						,		_UpperCamelCase							)


UpperCamelCase__ =			{
    SchedulerType.LINEAR: get_linear_schedule_with_warmup,
    SchedulerType.COSINE: get_cosine_schedule_with_warmup,
    SchedulerType.COSINE_WITH_RESTARTS: get_cosine_with_hard_restarts_schedule_with_warmup,
    SchedulerType.POLYNOMIAL: get_polynomial_decay_schedule_with_warmup,
    SchedulerType.CONSTANT: get_constant_schedule,
    SchedulerType.CONSTANT_WITH_WARMUP: get_constant_schedule_with_warmup,
    SchedulerType.PIECEWISE_CONSTANT: get_piecewise_constant_schedule,
}





def 		__SCREAMING_SNAKE_CASE					(  _UpperCamelCase						,		_UpperCamelCase						,		_UpperCamelCase = None						,		_UpperCamelCase = None						,		_UpperCamelCase = None						,		_UpperCamelCase = 1						,		_UpperCamelCase = 1.0						,		_UpperCamelCase = -1						,		):


   """simple docstring"""




   lowercase_	:    Any           =     SchedulerType(_UpperCamelCase							)
   lowercase_	:    List[Any]           =     TYPE_TO_SCHEDULER_FUNCTION[name]
   if name == SchedulerType.CONSTANT:
      return schedule_func(_UpperCamelCase						,		last_epoch=_UpperCamelCase							)

   if name == SchedulerType.PIECEWISE_CONSTANT:
      return schedule_func(_UpperCamelCase						,		step_rules=_UpperCamelCase						,		last_epoch=_UpperCamelCase							)

   # All other schedulers require `num_warmup_steps`
   if num_warmup_steps is None:
      raise ValueError(F"""{name} requires `num_warmup_steps`, please provide that argument."""							)

   if name == SchedulerType.CONSTANT_WITH_WARMUP:
      return schedule_func(_UpperCamelCase						,		num_warmup_steps=_UpperCamelCase						,		last_epoch=_UpperCamelCase							)

   # All other schedulers require `num_training_steps`
   if num_training_steps is None:
      raise ValueError(F"""{name} requires `num_training_steps`, please provide that argument."""							)

   if name == SchedulerType.COSINE_WITH_RESTARTS:
      return schedule_func(
          _UpperCamelCase						,		num_warmup_steps=_UpperCamelCase						,		num_training_steps=_UpperCamelCase						,		num_cycles=_UpperCamelCase						,		last_epoch=_UpperCamelCase						,		)

   if name == SchedulerType.POLYNOMIAL:
      return schedule_func(
          _UpperCamelCase						,		num_warmup_steps=_UpperCamelCase						,		num_training_steps=_UpperCamelCase						,		power=_UpperCamelCase						,		last_epoch=_UpperCamelCase						,		)

   return schedule_func(
       _UpperCamelCase						,		num_warmup_steps=_UpperCamelCase						,		num_training_steps=_UpperCamelCase						,		last_epoch=_UpperCamelCase							)




