





'''simple docstring'''







import argparse
import logging
import pickle
import random
import time

import numpy as np

from transformers import BertTokenizer, GPTaTokenizer, RobertaTokenizer


logging.basicConfig(
    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO
)
snake_case_     :       str						=  logging.getLogger(__name__)





def        lowerCamelCase_      (  ) ->	Tuple:
						UpperCAmelCase_			:      int     						=  argparse.ArgumentParser(
						    description='''Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids).'''   )
						parser.add_argument('''--file_path''',			type=SCREAMING_SNAKE_CASE__,			default='''data/dump.txt''',			help='''The path to the data.'''   )
						parser.add_argument('''--tokenizer_type''',			type=SCREAMING_SNAKE_CASE__,			default='''bert''',			choices=['''bert''', '''roberta''', '''gpt2''']   )
						parser.add_argument('''--tokenizer_name''',			type=SCREAMING_SNAKE_CASE__,			default='''bert-base-uncased''',			help='''The tokenizer to use.'''   )
						parser.add_argument('''--dump_file''',			type=SCREAMING_SNAKE_CASE__,			default='''data/dump''',			help='''The dump file prefix.'''   )
						UpperCAmelCase_			:      int     						=  parser.parse_args()

						logger.info(F"""Loading Tokenizer ({args.tokenizer_name})"""   )
						if args.tokenizer_type == "bert":
												UpperCAmelCase_			:      Any     						=  BertTokenizer.from_pretrained(args.tokenizer_name   )
												UpperCAmelCase_			:      Tuple     						=  tokenizer.special_tokens_map['''cls_token''']  # `[CLS]`
												UpperCAmelCase_			:      Optional[int]     						=  tokenizer.special_tokens_map['''sep_token''']  # `[SEP]`
						elif args.tokenizer_type == "roberta":
												UpperCAmelCase_			:      Optional[Any]     						=  RobertaTokenizer.from_pretrained(args.tokenizer_name   )
												UpperCAmelCase_			:      str     						=  tokenizer.special_tokens_map['''cls_token''']  # `<s>`
												UpperCAmelCase_			:      Any     						=  tokenizer.special_tokens_map['''sep_token''']  # `</s>`
						elif args.tokenizer_type == "gpt2":
												UpperCAmelCase_			:      int     						=  GPTaTokenizer.from_pretrained(args.tokenizer_name   )
												UpperCAmelCase_			:      Optional[Any]     						=  tokenizer.special_tokens_map['''bos_token''']  # `<|endoftext|>`
												UpperCAmelCase_			:      List[Any]     						=  tokenizer.special_tokens_map['''eos_token''']  # `<|endoftext|>`

						logger.info(F"""Loading text from {args.file_path}"""   )
						with open(args.file_path,			'''r''',			encoding='''utf8'''   ) as fp:
												UpperCAmelCase_			:      List[str]     						=  fp.readlines()

						logger.info('''Start encoding'''   )
						logger.info(F"""{len(SCREAMING_SNAKE_CASE__   )} examples to process."""   )

						UpperCAmelCase_			:      str     						=  []
						UpperCAmelCase_			:      Tuple     						=  0
						UpperCAmelCase_			:      List[str]     						=  10000
						UpperCAmelCase_			:      Any     						=  time.time()
						for text in data:
												UpperCAmelCase_			:      Optional[int]     						=  F"""{bos} {text.strip()} {sep}"""
												UpperCAmelCase_			:      str     						=  tokenizer.encode(SCREAMING_SNAKE_CASE__,			add_special_tokens=SCREAMING_SNAKE_CASE__   )
												rslt.append(SCREAMING_SNAKE_CASE__   )

												iter += 1
												if iter % interval == 0:
																		UpperCAmelCase_			:      int     						=  time.time()
																		logger.info(F"""{iter} examples processed. - {(end-start):.2f}s/{interval}expl"""   )
																		UpperCAmelCase_			:      Tuple     						=  time.time()
						logger.info('''Finished binarization'''   )
						logger.info(F"""{len(SCREAMING_SNAKE_CASE__   )} examples processed."""   )

						UpperCAmelCase_			:      int     						=  F"""{args.dump_file}.{args.tokenizer_name}.pickle"""
						UpperCAmelCase_			:      Optional[Any]     						=  tokenizer.vocab_size
						if vocab_size < (1 << 16):
												UpperCAmelCase_			:      Tuple     						=  [np.uintaa(SCREAMING_SNAKE_CASE__   ) for d in rslt]
						else:
												UpperCAmelCase_			:      List[str]     						=  [np.intaa(SCREAMING_SNAKE_CASE__   ) for d in rslt]
						random.shuffle(rslt_   )
						logger.info(F"""Dump to {dp_file}"""   )
						with open(SCREAMING_SNAKE_CASE__,			'''wb'''   ) as handle:
												pickle.dump(rslt_,			SCREAMING_SNAKE_CASE__,			protocol=pickle.HIGHEST_PROTOCOL   )


if __name__ == "__main__":
						main()












'''simple docstring'''







import unittest

import torch
from torch import nn

from diffusers.models.activations import get_activation


class 					__a      (unittest.TestCase     ):

							def     UpperCAmelCase__	(							self							:     Dict   )					->       Dict:







													"""simple docstring"""
													UpperCAmelCase_			:      Dict     						=  get_activation('''swish'''   )

													self.assertIsInstance(__magic_name__       ,     nn.SiLU   )

													self.assertEqual(act(torch.tensor(-1_00       ,     dtype=torch.floataa   )   ).item()       ,     0   )
													self.assertNotEqual(act(torch.tensor(-1       ,     dtype=torch.floataa   )   ).item()       ,     0   )
													self.assertEqual(act(torch.tensor(0       ,     dtype=torch.floataa   )   ).item()       ,     0   )
													self.assertEqual(act(torch.tensor(20       ,     dtype=torch.floataa   )   ).item()       ,     20   )

							def     UpperCAmelCase__	(							self							:     Tuple   )					->       Tuple:







													"""simple docstring"""
													UpperCAmelCase_			:      Union[str, Any]     						=  get_activation('''silu'''   )

													self.assertIsInstance(__magic_name__       ,     nn.SiLU   )

													self.assertEqual(act(torch.tensor(-1_00       ,     dtype=torch.floataa   )   ).item()       ,     0   )
													self.assertNotEqual(act(torch.tensor(-1       ,     dtype=torch.floataa   )   ).item()       ,     0   )
													self.assertEqual(act(torch.tensor(0       ,     dtype=torch.floataa   )   ).item()       ,     0   )
													self.assertEqual(act(torch.tensor(20       ,     dtype=torch.floataa   )   ).item()       ,     20   )

							def     UpperCAmelCase__	(							self							:     Tuple   )					->       Optional[int]:







													"""simple docstring"""
													UpperCAmelCase_			:      Optional[int]     						=  get_activation('''mish'''   )

													self.assertIsInstance(__magic_name__       ,     nn.Mish   )

													self.assertEqual(act(torch.tensor(-2_00       ,     dtype=torch.floataa   )   ).item()       ,     0   )
													self.assertNotEqual(act(torch.tensor(-1       ,     dtype=torch.floataa   )   ).item()       ,     0   )
													self.assertEqual(act(torch.tensor(0       ,     dtype=torch.floataa   )   ).item()       ,     0   )
													self.assertEqual(act(torch.tensor(20       ,     dtype=torch.floataa   )   ).item()       ,     20   )



							def     UpperCAmelCase__	(							self							:     str   )					->       Optional[Any]:







													"""simple docstring"""
													UpperCAmelCase_			:      List[Any]     						=  get_activation('''gelu'''   )

													self.assertIsInstance(__magic_name__       ,     nn.GELU   )

													self.assertEqual(act(torch.tensor(-1_00       ,     dtype=torch.floataa   )   ).item()       ,     0   )
													self.assertNotEqual(act(torch.tensor(-1       ,     dtype=torch.floataa   )   ).item()       ,     0   )
													self.assertEqual(act(torch.tensor(0       ,     dtype=torch.floataa   )   ).item()       ,     0   )
													self.assertEqual(act(torch.tensor(20       ,     dtype=torch.floataa   )   ).item()       ,     20   )





