


# coding=utf-8
# Copyright 2020 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# this script dumps information about the environment

import os
import sys

import transformers


__UpperCAmelCase : Optional[int] 			=				"3"

print("Python version:", sys.version)
print("transformers version:", transformers.__version__)

try:
  import torch

  print("Torch version:", torch.__version__)
  print("Cuda available:", torch.cuda.is_available())
  print("Cuda version:", torch.version.cuda)
  print("CuDNN version:", torch.backends.cudnn.version())
  print("Number of GPUs available:", torch.cuda.device_count())
  print("NCCL version:", torch.cuda.nccl.version())
except ImportError:
  print("Torch version:", None)

try:
  import deepspeed

  print("DeepSpeed version:", deepspeed.__version__)
except ImportError:
  print("DeepSpeed version:", None)

try:
  import tensorflow as tf

  print("TensorFlow version:", tf.__version__)
  print("TF GPUs available:", bool(tf.config.list_physical_devices("GPU")))
  print("Number of TF GPUs available:", len(tf.config.list_physical_devices("GPU")))
except ImportError:
  print("TensorFlow version:", None)





import json
import os
import tempfile

import transformers

import datasets
from utils import generate_example_dataset, get_duration


__UpperCAmelCase : Optional[int] 			=				500000

__UpperCAmelCase			,    __UpperCAmelCase : Any 			=				os.path.split(__file__)
__UpperCAmelCase : int 			=				os.path.join(RESULTS_BASEPATH, "results", RESULTS_FILENAME.replace(".py", ".json"))


@get_duration
def      a	(							SCREAMING_SNAKE_CASE_      :		datasets.Dataset							,			**SCREAMING_SNAKE_CASE_      :		Union[str, Any]       ):






       """simple docstring"""





       UpperCamelCase    :	Tuple   	=		dataset.map(**SCREAMING_SNAKE_CASE_       )


@get_duration
def      a	(							SCREAMING_SNAKE_CASE_      :		datasets.Dataset							,			**SCREAMING_SNAKE_CASE_      :		Any       ):






       """simple docstring"""





       UpperCamelCase    :	int   	=		dataset.filter(**SCREAMING_SNAKE_CASE_       )


def      a	(							):






       """simple docstring"""





       UpperCamelCase    :	Optional[int]   	=		{'''num examples''': SPEED_TEST_N_EXAMPLES}
       with tempfile.TemporaryDirectory() as tmp_dir:
              UpperCamelCase    :	Dict   	=		datasets.Features({'''text''': datasets.Value('''string'''       ), '''numbers''': datasets.Value('''float32'''       )}       )
              UpperCamelCase    :	List[str]   	=		generate_example_dataset(
                  os.path.join(SCREAMING_SNAKE_CASE_							,			'''dataset.arrow'''       )							,			SCREAMING_SNAKE_CASE_							,			num_examples=SCREAMING_SNAKE_CASE_       )

              UpperCamelCase    :	Union[str, Any]   	=		transformers.AutoTokenizer.from_pretrained('''bert-base-cased'''							,			use_fast=SCREAMING_SNAKE_CASE_       )

              def tokenize(SCREAMING_SNAKE_CASE_      :		Dict       ):
                     return tokenizer(examples['''text''']       )

              UpperCamelCase    :	List[Any]   	=		map(SCREAMING_SNAKE_CASE_       )

              UpperCamelCase    :	List[str]   	=		map(SCREAMING_SNAKE_CASE_							,			batched=SCREAMING_SNAKE_CASE_       )

              UpperCamelCase    :	List[Any]   	=		map(SCREAMING_SNAKE_CASE_							,			function=lambda SCREAMING_SNAKE_CASE_       : None							,			batched=SCREAMING_SNAKE_CASE_       )

              with dataset.formatted_as(type='''numpy'''       ):
                     UpperCamelCase    :	Tuple   	=		map(SCREAMING_SNAKE_CASE_							,			function=lambda SCREAMING_SNAKE_CASE_       : None							,			batched=SCREAMING_SNAKE_CASE_       )

              with dataset.formatted_as(type='''pandas'''       ):
                     UpperCamelCase    :	int   	=		map(SCREAMING_SNAKE_CASE_							,			function=lambda SCREAMING_SNAKE_CASE_       : None							,			batched=SCREAMING_SNAKE_CASE_       )

              with dataset.formatted_as(type='''torch'''							,			columns='''numbers'''       ):
                     UpperCamelCase    :	Dict   	=		map(SCREAMING_SNAKE_CASE_							,			function=lambda SCREAMING_SNAKE_CASE_       : None							,			batched=SCREAMING_SNAKE_CASE_       )

              with dataset.formatted_as(type='''tensorflow'''							,			columns='''numbers'''       ):
                     UpperCamelCase    :	Tuple   	=		map(SCREAMING_SNAKE_CASE_							,			function=lambda SCREAMING_SNAKE_CASE_       : None							,			batched=SCREAMING_SNAKE_CASE_       )

              UpperCamelCase    :	List[str]   	=		map(SCREAMING_SNAKE_CASE_							,			function=SCREAMING_SNAKE_CASE_							,			batched=SCREAMING_SNAKE_CASE_       )

              UpperCamelCase    :	Dict   	=		filter(SCREAMING_SNAKE_CASE_       )

              # Activate later when tokenizer support batched inputs
              # with dataset.formatted_as(type='numpy'):
              #     times[func.__name__ + " fast-tokenizer batched numpy"] = func(dataset, function=tokenize, batched=True)

       with open(SCREAMING_SNAKE_CASE_							,			'''wb'''       ) as f:
              f.write(json.dumps(SCREAMING_SNAKE_CASE_       ).encode('''utf-8'''       )       )


if __name__ == "__main__":  # useful to run the profiler
  benchmark_map_filter()

