





import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


SCREAMING_SNAKE_CASE__       :			Optional[Any]			      =						logging.getLogger(__name__)






def       _A  (      ):
   a__	:						Union[str, Any]      		=  argparse.ArgumentParser(
       description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."   )
   parser.add_argument(
       "--dataset_name"							, type=lowerCamelCase							, default="wikitext"							, help="Name of the training. Explore datasets at: hf.co/datasets."							, )
   parser.add_argument(
       "--dataset_config"							, type=lowerCamelCase							, default="wikitext-103-raw-v1"							, help="Configuration name of the dataset."   )
   parser.add_argument(
       "--tokenizer_name_or_path"							, type=lowerCamelCase							, default="sayakpaul/unigram-tokenizer-wikitext"							, help="Tokenizer identifier. Can be a local filepath or a Hub identifier."							, )
   parser.add_argument(
       "--shard_size"							, type=lowerCamelCase							, default=1000							, help="Number of entries to go in a single shard."							, )
   parser.add_argument("--split"							, type=lowerCamelCase							, default="train"							, choices=["train", "test", "validation"]   )
   parser.add_argument(
       "--limit"							, default=lowerCamelCase							, type=lowerCamelCase							, help="Limit the number of shards (used for debugging)."							, )
   parser.add_argument(
       "--max_length"							, type=lowerCamelCase							, default=512							, help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
       " sequence length that is a multiple of 8."							, )
   parser.add_argument(
       "--output_dir"							, default="tf-tpu"							, type=lowerCamelCase							, help="Output directory where the TFRecord shards will be saved. If the"
       " path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord"
       " shards will be directly saved to a Google Cloud Storage bucket."							, )

   a__	:						Union[str, Any]      		=  parser.parse_args()
   return args






def       _A  (      lowerCamelCase   ):

   def fn(lowerCamelCase   ):
      return tokenizer(examples["text"]   )

   return fn






def       _A  (      lowerCamelCase   ):
   a__	:						Tuple      		=  []
   for i in range(len(tokenized_data["input_ids"]   )   ):
      a__	:						Any      		=  {
          "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]   )   ),
          "attention_mask": tf.train.Feature(
              intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]   )   ),
      }
      a__	:						Optional[int]      		=  tf.train.Features(feature=lowerCamelCase   )
      a__	:						int      		=  tf.train.Example(features=lowerCamelCase   )
      a__	:						Dict      		=  example.SerializeToString()
      records.append(lowerCamelCase   )
   return records






def       _A  (      lowerCamelCase   ):
   a__	:						Dict      		=  datasets.load_dataset(args.dataset_name							, args.dataset_config							, split=args.split   )

   if args.limit is not None:
      a__	:						Optional[int]      		=  min(len(lowerCamelCase   )							, args.limit   )
      a__	:						Tuple      		=  dataset.select(range(lowerCamelCase   )   )
      print(F"""Limiting the dataset to {args.limit} entries."""   )

   a__	:						str      		=  AutoTokenizer.from_pretrained(args.tokenizer_name_or_path   )

   # Handle output directory creation.
   # For serializing into a Google Cloud Storage Bucket, one needs to first
   # create a bucket.
   if "gs" not in args.output_dir:
      if not os.path.exists(args.output_dir   ):
         os.makedirs(args.output_dir   )
      a__	:						List[str]      		=  os.path.join(args.output_dir							, args.split   )
      if not os.path.exists(lowerCamelCase   ):
         os.makedirs(lowerCamelCase   )
   else:
      a__	:						int      		=  os.path.join(args.output_dir							, args.split   )

   # Tokenize the whole dataset at once.
   a__	:						str      		=  tokenize_function(lowerCamelCase   )
   a__	:						List[Any]      		=  dataset.map(lowerCamelCase							, batched=lowerCamelCase							, num_proc=4							, remove_columns=["text"]   )

   # We need to concatenate all our texts together, and then split the result
   # into chunks of a fixed size, which we will call block_size. To do this, we
   # will use the map method again, with the option batched=True. When we use batched=True,
   # the function we pass to map() will be passed multiple inputs at once, allowing us
   # to group them into more or fewer examples than we had in the input.
   # This allows us to create our new fixed-length samples. The advantage of this
   # method is that we don't lose a whole lot of content from the dataset compared to the
   # case where we simply tokenize with a pre-defined max_length.

   def group_texts(lowerCamelCase   ):
      # Concatenate all texts.
      a__	:						List[Any]      		=  {k: sum(examples[k]							, []   ) for k in examples.keys()}
      a__	:						Union[str, Any]      		=  len(concatenated_examples[list(examples.keys()   )[0]]   )
      # We drop the small remainder, though you could add padding instead if the model supports it
      # In this, as in all things, we advise you to follow your heart 🫀
      a__	:						int      		=  (total_length // args.max_length) * args.max_length
      # Split by chunks of max_len.
      a__	:						Tuple      		=  {
          k: [t[i : i + args.max_length] for i in range(0							, lowerCamelCase							, args.max_length   )]
          for k, t in concatenated_examples.items()
      }
      return result

   a__	:						Tuple      		=  dataset_tokenized.map(lowerCamelCase							, batched=lowerCamelCase							, batch_size=1000							, num_proc=4   )

   a__	:						List[str]      		=  0
   a__	:						List[Any]      		=  0
   for shard in range(0							, len(lowerCamelCase   )							, args.shard_size   ):
      a__	:						Union[str, Any]      		=  grouped_dataset[shard : shard + args.shard_size]
      a__	:						List[str]      		=  len(dataset_snapshot["input_ids"]   )
      a__	:						List[str]      		=  os.path.join(lowerCamelCase							, F"""dataset-{shard_count}-{records_containing}.tfrecord"""   )
      a__	:						List[str]      		=  get_serialized_examples(lowerCamelCase   )

      with tf.io.TFRecordWriter(lowerCamelCase   ) as out_file:
         for i in range(len(lowerCamelCase   )   ):
            a__	:						Any      		=  serialized_examples[i]
            out_file.write(lowerCamelCase   )
         print("Wrote file {} containing {} records".format(lowerCamelCase							, lowerCamelCase   )   )

      shard_count += 1
      total_records += records_containing

   with open(F"""split-{args.split}-records-count.txt"""							, "w"   ) as f:
      print(F"""Total {args.split} records: {total_records}"""							, file=lowerCamelCase   )


if __name__ == "__main__":
     SCREAMING_SNAKE_CASE__       :			Any			      =						parse_args()
     main(args)








# Copyright 2022 The HuggingFace Team and The OpenBMB Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING

# rely on isort to merge the imports
from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


SCREAMING_SNAKE_CASE__       :			int			      =						{
    """configuration_cpmant""": ["""CPMANT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """CpmAntConfig"""],
    """tokenization_cpmant""": ["""CpmAntTokenizer"""],
}

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  SCREAMING_SNAKE_CASE__       :			List[str]			      =						[
      """CPMANT_PRETRAINED_MODEL_ARCHIVE_LIST""",
      """CpmAntForCausalLM""",
      """CpmAntModel""",
      """CpmAntPreTrainedModel""",
  ]


if TYPE_CHECKING:
  from .configuration_cpmant import CPMANT_PRETRAINED_CONFIG_ARCHIVE_MAP, CpmAntConfig
  from .tokenization_cpmant import CpmAntTokenizer

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_cpmant import (
        CPMANT_PRETRAINED_MODEL_ARCHIVE_LIST,
        CpmAntForCausalLM,
        CpmAntModel,
        CpmAntPreTrainedModel,
    )


else:
  import sys

  SCREAMING_SNAKE_CASE__       :			Union[str, Any]			      =						_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)

