






'''simple docstring'''




import logging
import os
from dataclasses import dataclass, field
from typing import Dict, Optional

import datasets
import numpy as np
import tensorflow as tf

from transformers import (
    AutoConfig,
    AutoTokenizer,
    EvalPrediction,
    HfArgumentParser,
    PreTrainedTokenizer,
    TFAutoModelForSequenceClassification,
    TFTrainer,
    TFTrainingArguments,
)
from transformers.utils import logging as hf_logging


hf_logging.set_verbosity_info()
hf_logging.enable_default_handler()
hf_logging.enable_explicit_format()

def 	__UpperCAmelCase       ( __magic_name__     ,__magic_name__     ,__magic_name__     ,__magic_name__     ,__magic_name__     ,__magic_name__ = None     ,)->    Union[str, Any]:



			"""simple docstring"""



			snake_case_				:					Tuple   =       {}

			if train_file is not None:
						snake_case_				:					Any   =       [train_file]
			if eval_file is not None:
						snake_case_				:					int   =       [eval_file]
			if test_file is not None:
						snake_case_				:					Union[str, Any]   =       [test_file]

			snake_case_				:					Dict   =       datasets.load_dataset("csv"     ,data_files=_SCREAMING_SNAKE_CASE						)
			snake_case_				:					List[Any]   =       list(ds[list(files.keys()						)[0]].features.keys()						)
			snake_case_				:					Tuple   =       features_name.pop(_SCREAMING_SNAKE_CASE						)
			snake_case_				:					Tuple   =       list(set(ds[list(files.keys()						)[0]][label_name]						)						)
			snake_case_				:					Any   =       {label: i for i, label in enumerate(_SCREAMING_SNAKE_CASE						)}
			snake_case_				:					Tuple   =       tokenizer.model_input_names
			snake_case_				:					List[str]   =       {}

			if len(_SCREAMING_SNAKE_CASE						) == 1:
						for k in files.keys():
									snake_case_				:					List[str]   =       ds[k].map(
									    lambda __magic_name__						: tokenizer.batch_encode_plus(
									        example[features_name[0]]     ,truncation=_SCREAMING_SNAKE_CASE     ,max_length=_SCREAMING_SNAKE_CASE     ,padding="max_length"						)     ,batched=_SCREAMING_SNAKE_CASE     ,)
			elif len(_SCREAMING_SNAKE_CASE						) == 2:
						for k in files.keys():
									snake_case_				:					Union[str, Any]   =       ds[k].map(
									    lambda __magic_name__						: tokenizer.batch_encode_plus(
									        (example[features_name[0]], example[features_name[1]])     ,truncation=_SCREAMING_SNAKE_CASE     ,max_length=_SCREAMING_SNAKE_CASE     ,padding="max_length"     ,)     ,batched=_SCREAMING_SNAKE_CASE     ,)

			def gen_train():
						for ex in transformed_ds[datasets.Split.TRAIN]:
									snake_case_				:					Dict   =       {k: v for k, v in ex.items() if k in input_names}
									snake_case_				:					List[Any]   =       labelaid[ex[label_name]]
									yield (d, label)

			def gen_val():
						for ex in transformed_ds[datasets.Split.VALIDATION]:
									snake_case_				:					List[str]   =       {k: v for k, v in ex.items() if k in input_names}
									snake_case_				:					Union[str, Any]   =       labelaid[ex[label_name]]
									yield (d, label)

			def gen_test():
						for ex in transformed_ds[datasets.Split.TEST]:
									snake_case_				:					List[Any]   =       {k: v for k, v in ex.items() if k in input_names}
									snake_case_				:					Dict   =       labelaid[ex[label_name]]
									yield (d, label)

			snake_case_				:					Optional[int]   =       (
			    tf.data.Dataset.from_generator(
			        _SCREAMING_SNAKE_CASE     ,({k: tf.intaa for k in input_names}, tf.intaa)     ,({k: tf.TensorShape([None]						) for k in input_names}, tf.TensorShape([]						))     ,)
			    if datasets.Split.TRAIN in transformed_ds
			    else None
			)

			if train_ds is not None:
						snake_case_				:					Optional[int]   =       train_ds.apply(tf.data.experimental.assert_cardinality(len(ds[datasets.Split.TRAIN]						)						)						)

			snake_case_				:					List[str]   =       (
			    tf.data.Dataset.from_generator(
			        _SCREAMING_SNAKE_CASE     ,({k: tf.intaa for k in input_names}, tf.intaa)     ,({k: tf.TensorShape([None]						) for k in input_names}, tf.TensorShape([]						))     ,)
			    if datasets.Split.VALIDATION in transformed_ds
			    else None
			)

			if val_ds is not None:
						snake_case_				:					List[str]   =       val_ds.apply(tf.data.experimental.assert_cardinality(len(ds[datasets.Split.VALIDATION]						)						)						)

			snake_case_				:					Tuple   =       (
			    tf.data.Dataset.from_generator(
			        _SCREAMING_SNAKE_CASE     ,({k: tf.intaa for k in input_names}, tf.intaa)     ,({k: tf.TensorShape([None]						) for k in input_names}, tf.TensorShape([]						))     ,)
			    if datasets.Split.TEST in transformed_ds
			    else None
			)

			if test_ds is not None:
						snake_case_				:					List[Any]   =       test_ds.apply(tf.data.experimental.assert_cardinality(len(ds[datasets.Split.TEST]						)						)						)

			return train_ds, val_ds, test_ds, labelaid


__lowerCamelCase   :					Optional[Any]   							=					logging.getLogger(__name__)







@dataclass
class    A_	:
							"""simple docstring"""




							a__   		=	field(metadata={'''help''': '''Which column contains the label'''}							)
							a__   		=	field(default=snake_case__						,					metadata={'''help''': '''The path of the training file'''}							)
							a__   		=	field(default=snake_case__						,					metadata={'''help''': '''The path of the development file'''}							)
							a__   		=	field(default=snake_case__						,					metadata={'''help''': '''The path of the test file'''}							)
							a__   		=	field(
							    default=128						,					metadata={
							        '''help''': (
							            '''The maximum total input sequence length after tokenization. Sequences longer '''
							            '''than this will be truncated, sequences shorter will be padded.'''
							        )
							    }						,					)
							a__   		=	field(
							    default=snake_case__						,					metadata={'''help''': '''Overwrite the cached training and evaluation sets'''}							)







@dataclass
class    A_	:
							"""simple docstring"""




							a__   		=	field(
							    metadata={'''help''': '''Path to pretrained model or model identifier from huggingface.co/models'''}							)
							a__   		=	field(
							    default=snake_case__						,					metadata={'''help''': '''Pretrained config name or path if not the same as model_name'''}							)
							a__   		=	field(
							    default=snake_case__						,					metadata={'''help''': '''Pretrained tokenizer name or path if not the same as model_name'''}							)
							a__   		=	field(default=snake_case__						,					metadata={'''help''': '''Set this flag to use fast tokenization.'''}							)
							# If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
							# or just modify its tokenizer_config.json.
							a__   		=	field(
							    default=snake_case__						,					metadata={'''help''': '''Where do you want to store the pretrained models downloaded from huggingface.co'''}						,					)

def 	__UpperCAmelCase       ( )->    Union[str, Any]:



			"""simple docstring"""



			snake_case_				:					Optional[Any]   =       HfArgumentParser((ModelArguments, DataTrainingArguments, TFTrainingArguments)						)
			snake_case_,	snake_case_,	snake_case_				:					Union[str, Any]   =       parser.parse_args_into_dataclasses()

			if (
			    os.path.exists(training_args.output_dir						)
			    and os.listdir(training_args.output_dir						)
			    and training_args.do_train
			    and not training_args.overwrite_output_dir
			):
						raise ValueError(
						    F'''Output directory ({training_args.output_dir}) already exists and is not empty. Use'''
						    " --overwrite_output_dir to overcome."						)

			# Setup logging
			logging.basicConfig(
			    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s"     ,datefmt="%m/%d/%Y %H:%M:%S"     ,level=logging.INFO     ,)
			logger.info(
			    F'''n_replicas: {training_args.n_replicas}, distributed training: {bool(training_args.n_replicas > 1						)}, '''
			    F'''16-bits training: {training_args.fpaa}'''						)
			logger.info(F'''Training/evaluation parameters {training_args}'''						)

			# Load pretrained model and tokenizer
			#
			# Distributed training:
			# The .from_pretrained methods guarantee that only one local process can concurrently
			# download model & vocab.

			snake_case_				:					Dict   =       AutoTokenizer.from_pretrained(
			    model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path     ,cache_dir=model_args.cache_dir     ,)

			snake_case_,	snake_case_,	snake_case_,	snake_case_				:					int   =       get_tfds(
			    train_file=data_args.train_file     ,eval_file=data_args.dev_file     ,test_file=data_args.test_file     ,tokenizer=_SCREAMING_SNAKE_CASE     ,label_column_id=data_args.label_column_id     ,max_seq_length=data_args.max_seq_length     ,)

			snake_case_				:					List[Any]   =       AutoConfig.from_pretrained(
			    model_args.config_name if model_args.config_name else model_args.model_name_or_path     ,num_labels=len(_SCREAMING_SNAKE_CASE						)     ,labelaid=_SCREAMING_SNAKE_CASE     ,idalabel={id: label for label, id in labelaid.items()}     ,finetuning_task="text-classification"     ,cache_dir=model_args.cache_dir     ,)

			with training_args.strategy.scope():
						snake_case_				:					Dict   =       TFAutoModelForSequenceClassification.from_pretrained(
						    model_args.model_name_or_path     ,from_pt=bool(".bin" in model_args.model_name_or_path						)     ,config=_SCREAMING_SNAKE_CASE     ,cache_dir=model_args.cache_dir     ,)

			def compute_metrics(__magic_name__						) -> Dict:
						snake_case_				:					List[Any]   =       np.argmax(p.predictions     ,axis=1						)

						return {"acc": (preds == p.label_ids).mean()}

			# Initialize our Trainer
			snake_case_				:					Any   =       TFTrainer(
			    model=_SCREAMING_SNAKE_CASE     ,args=_SCREAMING_SNAKE_CASE     ,train_dataset=_SCREAMING_SNAKE_CASE     ,eval_dataset=_SCREAMING_SNAKE_CASE     ,compute_metrics=_SCREAMING_SNAKE_CASE     ,)

			# Training
			if training_args.do_train:
						trainer.train()
						trainer.save_model()
						tokenizer.save_pretrained(training_args.output_dir						)

			# Evaluation
			snake_case_				:					Tuple   =       {}
			if training_args.do_eval:
						logger.info("*** Evaluate ***"						)
						snake_case_				:					Union[str, Any]   =       trainer.evaluate()
						snake_case_				:					Optional[int]   =       os.path.join(training_args.output_dir     ,"eval_results.txt"						)

						with open(_SCREAMING_SNAKE_CASE     ,"w"						) as writer:
									logger.info("***** Eval results *****"						)

									for key, value in result.items():
												logger.info(F'''  {key} = {value}'''						)
												writer.write(F'''{key} = {value}\n'''						)

									results.update(_SCREAMING_SNAKE_CASE						)

			return results


if __name__ == "__main__":
		main()












'''simple docstring'''




import gzip
import hashlib
import json
import multiprocessing
import os
import re
import shutil
import time
from pathlib import Path

import numpy as np
from arguments import PreprocessingArguments
from datasets import load_dataset
from minhash_deduplication import deduplicate_dataset

from transformers import AutoTokenizer, HfArgumentParser


__lowerCamelCase   :					List[str]   							=					re.compile(R'''\s+''')

def 	__UpperCAmelCase       ( __magic_name__						)->    Union[str, Any]:



			"""simple docstring"""



			return {"hash": hashlib.mda(re.sub(__magic_name__     ,""     ,example["content"]						).encode("utf-8"						)						).hexdigest()}

def 	__UpperCAmelCase       ( __magic_name__						)->    str:



			"""simple docstring"""



			snake_case_				:					Optional[Any]   =       [len(__magic_name__						) for line in example["content"].splitlines()]
			return {"line_mean": np.mean(__magic_name__						), "line_max": max(__magic_name__						)}

def 	__UpperCAmelCase       ( __magic_name__						)->    int:



			"""simple docstring"""



			snake_case_				:					Optional[int]   =       np.mean([c.isalnum() for c in example["content"]]						)
			return {"alpha_frac": alpha_frac}

def 	__UpperCAmelCase       ( __magic_name__     ,__magic_name__						)->    Tuple:



			"""simple docstring"""



			if example["hash"] in uniques:
						uniques.remove(example["hash"]						)
						return True
			else:
						return False

def 	__UpperCAmelCase       ( __magic_name__     ,__magic_name__=5						)->    Tuple:



			"""simple docstring"""



			snake_case_				:					List[str]   =       ["auto-generated", "autogenerated", "automatically generated"]
			snake_case_				:					Optional[Any]   =       example["content"].splitlines()
			for _, line in zip(range(__magic_name__						)     ,__magic_name__						):
						for keyword in keywords:
									if keyword in line.lower():
												return {"autogenerated": True}
			else:
						return {"autogenerated": False}

def 	__UpperCAmelCase       ( __magic_name__     ,__magic_name__=5     ,__magic_name__=0.05						)->    Optional[Any]:



			"""simple docstring"""



			snake_case_				:					str   =       ["unit tests", "test file", "configuration file"]
			snake_case_				:					int   =       example["content"].splitlines()
			snake_case_				:					Optional[Any]   =       0
			snake_case_				:					Any   =       0
			# first test
			for _, line in zip(range(__magic_name__						)     ,__magic_name__						):
						for keyword in keywords:
									if keyword in line.lower():
												return {"config_or_test": True}
    # second test
			snake_case_				:					Tuple   =       example["content"].count("\n"						)
			snake_case_				:					int   =       int(coeff * nlines						)
			for line in lines:
						count_config += line.lower().count("config"						)
						count_test += line.lower().count("test"						)
						if count_config > threshold or count_test > threshold:
									return {"config_or_test": True}
			return {"config_or_test": False}

def 	__UpperCAmelCase       ( __magic_name__						)->    str:



			"""simple docstring"""



			snake_case_				:					List[Any]   =       ["def ", "class ", "for ", "while "]
			snake_case_				:					Optional[Any]   =       example["content"].splitlines()
			for line in lines:
						for keyword in keywords:
									if keyword in line.lower():
												return {"has_no_keywords": False}
			return {"has_no_keywords": True}

def 	__UpperCAmelCase       ( __magic_name__     ,__magic_name__=4						)->    Optional[int]:



			"""simple docstring"""



			snake_case_				:					Tuple   =       example["content"].splitlines()
			snake_case_				:					Tuple   =       0
			for line in lines:
						counter += line.lower().count("="						)
						if counter > minimum:
									return {"has_few_assignments": False}
			return {"has_few_assignments": True}

def 	__UpperCAmelCase       ( __magic_name__						)->    List[Any]:



			"""simple docstring"""



			snake_case_				:					Tuple   =       tokenizer(example["content"]     ,truncation=__magic_name__						)["input_ids"]
			snake_case_				:					int   =       len(example["content"]						) / len(__magic_name__						)
			return {"ratio": ratio}

def 	__UpperCAmelCase       ( __magic_name__						)->    Optional[Any]:



			"""simple docstring"""



			snake_case_				:					Union[str, Any]   =       {}
			results.update(get_hash(__magic_name__						)						)
			results.update(line_stats(__magic_name__						)						)
			results.update(alpha_stats(__magic_name__						)						)
			results.update(char_token_ratio(__magic_name__						)						)
			results.update(is_autogenerated(__magic_name__						)						)
			results.update(is_config_or_test(__magic_name__						)						)
			results.update(has_no_keywords(__magic_name__						)						)
			results.update(has_few_assignments(__magic_name__						)						)
			return results

def 	__UpperCAmelCase       ( __magic_name__     ,__magic_name__     ,__magic_name__						)->    Tuple:



			"""simple docstring"""



			if not check_uniques(__magic_name__     ,__magic_name__						):
						return False
			elif example["autogenerated"]:
						return False
			elif example["line_max"] > args.line_max:
						return False
			elif example["line_mean"] > args.line_mean:
						return False
			elif example["alpha_frac"] < args.alpha_frac:
						return False
			elif example["ratio"] < args.min_token_ratio:
						return False
			elif example["config_or_test"] and np.random.rand() <= args.filter_proba:
						return False
			elif example["has_no_keywords"] and np.random.rand() <= args.filter_proba:
						return False
			elif example["has_few_assignments"]:
						return False
			else:
						return True

def 	__UpperCAmelCase       ( __magic_name__						)->    Dict:



			"""simple docstring"""



			with open(__magic_name__     ,"rb"						) as f_in:
						with gzip.open(str(__magic_name__						) + ".gz"     ,"wb"     ,compresslevel=6						) as f_out:
									shutil.copyfileobj(__magic_name__     ,__magic_name__						)
			os.unlink(__magic_name__						)


# Settings
__lowerCamelCase   :					List[Any]   							=					HfArgumentParser(PreprocessingArguments)
__lowerCamelCase   :					str   							=					parser.parse_args()
if args.num_workers is None:
		__lowerCamelCase   :					List[Any]   							=					multiprocessing.cpu_count()
__lowerCamelCase   :					Optional[int]   							=					AutoTokenizer.from_pretrained(args.tokenizer_dir)

# Load dataset
__lowerCamelCase   :					Any   							=					time.time()
__lowerCamelCase   :					str   							=					load_dataset(args.dataset_name, split='''train''')
print(f'''Time to load dataset: {time.time()-t_start:.2f}''')

# Run preprocessing
__lowerCamelCase   :					List[str]   							=					time.time()
__lowerCamelCase   :					Any   							=					ds.map(preprocess, num_proc=args.num_workers)
print(f'''Time to preprocess dataset: {time.time()-t_start:.2f}''')

# Deduplicate hashes
__lowerCamelCase   :					Any   							=					set(ds.unique('''hash'''))
__lowerCamelCase   :					Optional[int]   							=					len(uniques) / len(ds)
print(f'''Fraction of duplicates: {1-frac:.2%}''')

# Deduplicate data and apply heuristics
__lowerCamelCase   :					List[str]   							=					time.time()
__lowerCamelCase   :					Tuple   							=					ds.filter(filter, fn_kwargs={'''uniques''': uniques, '''args''': args})
print(f'''Time to filter dataset: {time.time()-t_start:.2f}''')
print(f'''Size of filtered dataset: {len(ds_filter)}''')

# Deduplicate with minhash and jaccard similarity
if args.near_deduplication:
		__lowerCamelCase   :					List[str]   							=					time.time()
		__lowerCamelCase							,      __lowerCamelCase   :					Tuple   							=					deduplicate_dataset(ds_filter, args.jaccard_threshold)
		print(f'''Time to deduplicate dataset: {time.time()-t_start:.2f}''')
		print(f'''Size of deduplicate dataset: {len(ds_filter)}''')

# Save data in batches of samples_per_file
__lowerCamelCase   :					List[Any]   							=					Path(args.output_dir)
output_dir.mkdir(exist_ok=True)

# save duplicate_clusters in the output_dir as artifacts
# not sure it is the right place the save it
if args.near_deduplication:
		with open(output_dir / '''duplicate_clusters.json''', '''w''') as f:
				json.dump(duplicate_clusters, f)

__lowerCamelCase   :					List[str]   							=					output_dir / '''data'''
data_dir.mkdir(exist_ok=True)

__lowerCamelCase   :					int   							=					time.time()
for file_number, index in enumerate(range(0, len(ds_filter), args.samples_per_file)):
		__lowerCamelCase   :					Union[str, Any]   							=					str(data_dir / f'''file-{file_number+1:012}.json''')
		__lowerCamelCase   :					List[Any]   							=					min(len(ds_filter), index + args.samples_per_file)
		ds_filter.select(list(range(index, end_index))).to_json(file_path)
		compress_file(file_path)






print(f'''Time to save dataset: {time.time()-t_start:.2f}''')




