


'''simple docstring'''




from scipy.stats import pearsonr

import datasets


_lowerCAmelCase  :		List[str]		     =	'\nPearson correlation coefficient and p-value for testing non-correlation.\nThe Pearson correlation coefficient measures the linear relationship between two datasets. The calculation of the p-value relies on the assumption that each dataset is normally distributed. Like other correlation coefficients, this one varies between -1 and +1 with 0 implying no correlation. Correlations of -1 or +1 imply an exact linear relationship. Positive correlations imply that as x increases, so does y. Negative correlations imply that as x increases, y decreases.\nThe p-value roughly indicates the probability of an uncorrelated system producing datasets that have a Pearson correlation at least as extreme as the one computed from these datasets.\n'


_lowerCAmelCase  :		str		     =	'\nArgs:\n    predictions (`list` of `int`): Predicted class labels, as returned by a model.\n    references (`list` of `int`): Ground truth labels.\n    return_pvalue (`boolean`): If `True`, returns the p-value, along with the correlation coefficient. If `False`, returns only the correlation coefficient. Defaults to `False`.\n\nReturns:\n    pearsonr (`float`): Pearson correlation coefficient. Minimum possible value is -1. Maximum possible value is 1. Values of 1 and -1 indicate exact linear positive and negative relationships, respectively. A value of 0 implies no correlation.\n    p-value (`float`): P-value, which roughly indicates the probability of an The p-value roughly indicates the probability of an uncorrelated system producing datasets that have a Pearson correlation at least as extreme as the one computed from these datasets. Minimum possible value is 0. Maximum possible value is 1. Higher values indicate higher probabilities.\n\nExamples:\n\n    Example 1-A simple example using only predictions and references.\n        >>> pearsonr_metric = datasets.load_metric("pearsonr")\n        >>> results = pearsonr_metric.compute(predictions=[10, 9, 2.5, 6, 4], references=[1, 2, 3, 4, 5])\n        >>> print(round(results[\'pearsonr\'], 2))\n        -0.74\n\n    Example 2-The same as Example 1, but that also returns the `p-value`.\n        >>> pearsonr_metric = datasets.load_metric("pearsonr")\n        >>> results = pearsonr_metric.compute(predictions=[10, 9, 2.5, 6, 4], references=[1, 2, 3, 4, 5], return_pvalue=True)\n        >>> print(sorted(list(results.keys())))\n        [\'p-value\', \'pearsonr\']\n        >>> print(round(results[\'pearsonr\'], 2))\n        -0.74\n        >>> print(round(results[\'p-value\'], 2))\n        0.15\n'


_lowerCAmelCase  :		List[Any]		     =	'\n@article{2020SciPy-NMeth,\nauthor  = {Virtanen, Pauli and Gommers, Ralf and Oliphant, Travis E. and\n      Haberland, Matt and Reddy, Tyler and Cournapeau, David and\n      Burovski, Evgeni and Peterson, Pearu and Weckesser, Warren and\n      Bright, Jonathan and {van der Walt}, St{\'e}fan J. and\n      Brett, Matthew and Wilson, Joshua and Millman, K. Jarrod and\n      Mayorov, Nikolay and Nelson, Andrew R. J. and Jones, Eric and\n      Kern, Robert and Larson, Eric and Carey, C J and\n      Polat, Ilhan and Feng, Yu and Moore, Eric W. and\n      {VanderPlas}, Jake and Laxalde, Denis and Perktold, Josef and\n      Cimrman, Robert and Henriksen, Ian and Quintero, E. A. and\n      Harris, Charles R. and Archibald, Anne M. and\n      Ribeiro, Antonio H. and Pedregosa, Fabian and\n      {van Mulbregt}, Paul and {SciPy 1.0 Contributors}},\ntitle   = {{{SciPy} 1.0: Fundamental Algorithms for Scientific\n      Computing in Python}},\njournal = {Nature Methods},\nyear    = {2020},\nvolume  = {17},\npages   = {261--272},\nadsurl  = {https://rdcu.be/b08Wh},\ndoi = {10.1038/s41592-019-0686-2},\n}\n'



@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION      ,				_KWARGS_DESCRIPTION							)
class  lowerCAmelCase				(				datasets.Metric							):



	def 				lowercase     (					self    ):
		return datasets.MetricInfo(
		    description=_DESCRIPTION ,					citation=_CITATION ,					inputs_description=_KWARGS_DESCRIPTION ,					features=datasets.Features(
		        {
		            'predictions': datasets.Value('float'    ),
		            'references': datasets.Value('float'    ),
		        }    ) ,					reference_urls=['https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.pearsonr.html'] ,					)



	def 				lowercase     (					self ,					snake_case__ ,					snake_case__ ,					snake_case__=False    ):
		if return_pvalue:
			lowerCAmelCase					:    str      							= pearsonr(snake_case__ ,					snake_case__    )
			return {"pearsonr": results[0], "p-value": results[1]}
		else:
			return {"pearsonr": float(pearsonr(snake_case__ ,					snake_case__    )[0]    )}




'''simple docstring'''




import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


_lowerCAmelCase  :		List[Any]		     =	logging.getLogger(__name__)

def 			__UpperCamelCase    (    )    ->      Any:




	"""simple docstring"""





	lowerCAmelCase					:    str      							= argparse.ArgumentParser(
	    description='Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'				)
	parser.add_argument(
	    '--dataset_name'					,   type=_A					,   default='wikitext'					,   help='Name of the training. Explore datasets at: hf.co/datasets.'					,   )
	parser.add_argument(
	    '--dataset_config'					,   type=_A					,   default='wikitext-103-raw-v1'					,   help='Configuration name of the dataset.'				)
	parser.add_argument(
	    '--tokenizer_name_or_path'					,   type=_A					,   default='sayakpaul/unigram-tokenizer-wikitext'					,   help='Tokenizer identifier. Can be a local filepath or a Hub identifier.'					,   )
	parser.add_argument(
	    '--shard_size'					,   type=_A					,   default=10_00					,   help='Number of entries to go in a single shard.'					,   )
	parser.add_argument('--split'					,   type=_A					,   default='train'					,   choices=['train', 'test', 'validation']				)
	parser.add_argument(
	    '--limit'					,   default=_A					,   type=_A					,   help='Limit the number of shards (used for debugging).'					,   )
	parser.add_argument(
	    '--max_length'					,   type=_A					,   default=5_12					,   help='Maximum sequence length. For training on TPUs, it helps to have a maximum'
	    ' sequence length that is a multiple of 8.'					,   )
	parser.add_argument(
	    '--output_dir'					,   default='tf-tpu'					,   type=_A					,   help='Output directory where the TFRecord shards will be saved. If the'
	    ' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'
	    ' shards will be directly saved to a Google Cloud Storage bucket.'					,   )

	lowerCAmelCase					:    Any      							= parser.parse_args()
	return args

def 			__UpperCamelCase    (    _A    :				Optional[int]				)    ->      int:




	"""simple docstring"""





	def fn(_A    :				Tuple				):
		return tokenizer(examples['text']				)

	return fn

def 			__UpperCamelCase    (    _A    :				int				)    ->      int:




	"""simple docstring"""





	lowerCAmelCase					:    Tuple      							= []
	for i in range(len(tokenized_data['input_ids']				)				):
		lowerCAmelCase					:    Optional[Any]      							= {
		    'input_ids': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['input_ids'][i]				)				),
		    'attention_mask': tf.train.Feature(
		        intaa_list=tf.train.IntaaList(value=tokenized_data['attention_mask'][i]				)				),
		}
		lowerCAmelCase					:    Any      							= tf.train.Features(feature=_A				)
		lowerCAmelCase					:    List[str]      							= tf.train.Example(features=_A				)
		lowerCAmelCase					:    Tuple      							= example.SerializeToString()
		records.append(_A				)
	return records

def 			__UpperCamelCase    (    _A    :				int				)    ->      Union[str, Any]:




	"""simple docstring"""





	lowerCAmelCase					:    Union[str, Any]      							= datasets.load_dataset(args.dataset_name					,   args.dataset_config					,   split=args.split				)

	if args.limit is not None:
		lowerCAmelCase					:    Optional[Any]      							= min(len(_A				)					,   args.limit				)
		lowerCAmelCase					:    Dict      							= dataset.select(range(_A				)				)
		print(F"Limiting the dataset to {args.limit} entries."				)

	lowerCAmelCase					:    str      							= AutoTokenizer.from_pretrained(args.tokenizer_name_or_path				)

	# Handle output directory creation.
	# For serializing into a Google Cloud Storage Bucket, one needs to first
	# create a bucket.
	if "gs" not in args.output_dir:
		if not os.path.exists(args.output_dir				):
			os.makedirs(args.output_dir				)
		lowerCAmelCase					:    Any      							= os.path.join(args.output_dir					,   args.split				)
		if not os.path.exists(_A				):
			os.makedirs(_A				)
	else:
		lowerCAmelCase					:    List[Any]      							= os.path.join(args.output_dir					,   args.split				)

	# Tokenize the whole dataset at once.
	lowerCAmelCase					:    Any      							= tokenize_function(_A				)
	lowerCAmelCase					:    Optional[int]      							= dataset.map(_A					,   batched=_A					,   num_proc=4					,   remove_columns=['text']				)

	# We need to concatenate all our texts together, and then split the result
	# into chunks of a fixed size, which we will call block_size. To do this, we
	# will use the map method again, with the option batched=True. When we use batched=True,
	# the function we pass to map() will be passed multiple inputs at once, allowing us
	# to group them into more or fewer examples than we had in the input.
	# This allows us to create our new fixed-length samples. The advantage of this
	# method is that we don't lose a whole lot of content from the dataset compared to the
	# case where we simply tokenize with a pre-defined max_length.

	def group_texts(_A    :				str				):
		# Concatenate all texts.
		lowerCAmelCase					:    Optional[int]      							= {k: sum(examples[k]					,   []				) for k in examples.keys()}
		lowerCAmelCase					:    str      							= len(concatenated_examples[list(examples.keys()				)[0]]				)
		# We drop the small remainder, though you could add padding instead if the model supports it
		# In this, as in all things, we advise you to follow your heart 🫀
		lowerCAmelCase					:    List[Any]      							= (total_length // args.max_length) * args.max_length
		# Split by chunks of max_len.
		lowerCAmelCase					:    str      							= {
		    k: [t[i : i + args.max_length] for i in range(0					,   _A					,   args.max_length				)]
		    for k, t in concatenated_examples.items()
		}
		return result

	lowerCAmelCase					:    List[Any]      							= dataset_tokenized.map(_A					,   batched=_A					,   batch_size=10_00					,   num_proc=4				)

	lowerCAmelCase					:    Union[str, Any]      							= 0
	lowerCAmelCase					:    Tuple      							= 0
	for shard in range(0					,   len(_A				)					,   args.shard_size				):
		lowerCAmelCase					:    Optional[Any]      							= grouped_dataset[shard : shard + args.shard_size]
		lowerCAmelCase					:    List[str]      							= len(dataset_snapshot['input_ids']				)
		lowerCAmelCase					:    Union[str, Any]      							= os.path.join(_A					,   F"dataset-{shard_count}-{records_containing}.tfrecord"				)
		lowerCAmelCase					:    List[Any]      							= get_serialized_examples(_A				)

		with tf.io.TFRecordWriter(_A				) as out_file:
			for i in range(len(_A				)				):
				lowerCAmelCase					:    Union[str, Any]      							= serialized_examples[i]
				out_file.write(_A				)
			print('Wrote file {} containing {} records'.format(_A					,   _A				)				)

		shard_count += 1
		total_records += records_containing

	with open(F"split-{args.split}-records-count.txt"					,   'w'				) as f:
		print(F"Total {args.split} records: {total_records}"					,   file=_A				)


if __name__ == "__main__":
				_lowerCAmelCase  :		List[Any]		     =	parse_args()
				main(args)
