


import os

import jsonlines
import numpy as np
from tqdm import tqdm


lowerCAmelCase							=						2_048
lowerCAmelCase							=						4_096
lowerCAmelCase							=						42
lowerCAmelCase							=						os.environ.pop("""PROCESS_TRAIN""", """false""")
lowerCAmelCase							=						{"""null""": 0, """short""": 1, """long""": 2, """yes""": 3, """no""": 4}




def 						__SCREAMING_SNAKE_CASE       (					lowercase_			)		->    str:




						'''simple docstring'''







						def choose_first(lowercase_			,					lowercase_=False			):
												assert isinstance(lowercase_			,					lowercase_			)
												if len(lowercase_			) == 1:
																		__UpperCAmelCase :       Any  						=  answer[0]
																		return {k: [answer[k]] for k in answer} if is_long_answer else answer
												for a in answer:
																		if is_long_answer:
																								__UpperCAmelCase :       str  						=  {k: [a[k]] for k in a}
																		if len(a['''start_token''']			) > 0:
																								break
												return a

						__UpperCAmelCase :       Optional[Any]  						=  {'''id''': example['''id''']}
						__UpperCAmelCase :       int  						=  example['''annotations''']
						__UpperCAmelCase :       Optional[Any]  						=  annotation['''yes_no_answer''']
						if 0 in yes_no_answer or 1 in yes_no_answer:
												__UpperCAmelCase :       Optional[int]  						=  ['''yes'''] if 1 in yes_no_answer else ['''no''']
												__UpperCAmelCase :       int  						=  []
												__UpperCAmelCase :       Optional[int]  						=  []
												__UpperCAmelCase :       Optional[int]  						=  ['''<cls>''']
						else:
												__UpperCAmelCase :       Optional[int]  						=  ['''short''']
												__UpperCAmelCase :       int  						=  choose_first(annotation['''short_answers''']			)
												if len(out['''start_token''']			) == 0:
																		# answer will be long if short is not available
																		__UpperCAmelCase :       Union[str, Any]  						=  ['''long''']
																		__UpperCAmelCase :       List[str]  						=  choose_first(annotation['''long_answer''']			,					is_long_answer=lowercase_			)
																		__UpperCAmelCase :       Optional[int]  						=  []
												answer.update(lowercase_			)

						# disregard some samples
						if len(answer['''start_token''']			) > 1 or answer["start_token"] == answer["end_token"]:
												__UpperCAmelCase :       Tuple  						=  True
						else:
												__UpperCAmelCase :       Any  						=  False

						__UpperCAmelCase :       Tuple  						=  ['''start_token''', '''end_token''', '''start_byte''', '''end_byte''', '''text''']
						if not all(isinstance(answer[k]			,					lowercase_			) for k in cols			):
												raise ValueError('''Issue in ID'''			,					example['''id''']			)

						return answer




def 						__SCREAMING_SNAKE_CASE       (					lowercase_			,					lowercase_=False			)		->    Tuple:




						'''simple docstring'''







						__UpperCAmelCase :       List[Any]  						=  _get_single_answer(lowercase_			)
						# bytes are of no use
						del answer["start_byte"]
						del answer["end_byte"]

						# handle yes_no answers explicitly
						if answer["category"][0] in ["yes", "no"]:  # category is list with one element
												__UpperCAmelCase :       List[str]  						=  example['''document''']['''tokens''']
												__UpperCAmelCase :       Tuple  						=  []
												for i in range(len(doc['''token''']			)			):
																		if not doc["is_html"][i]:
																								context.append(doc['''token'''][i]			)
												return {
												    "context": " ".join(lowercase_			),
												    "answer": {
												        "start_token": -100,  # ignore index in cross-entropy
												        "end_token": -100,  # ignore index in cross-entropy
												        "category": answer["category"],
												        "span": answer["category"],  # extra
												    },
												}

						# later, help in removing all no answers
						if answer["start_token"] == [-1]:
												return {
												    "context": "None",
												    "answer": {
												        "start_token": -1,
												        "end_token": -1,
												        "category": "null",
												        "span": "None",  # extra
												    },
												}

						# handling normal samples

						__UpperCAmelCase :       Union[str, Any]  						=  ['''start_token''', '''end_token''']
						answer.update({k: answer[k][0] if len(answer[k]			) > 0 else answer[k] for k in cols}			)  # e.g. [10] == 10

						__UpperCAmelCase :       Dict  						=  example['''document''']['''tokens''']
						__UpperCAmelCase :       str  						=  answer['''start_token''']
						__UpperCAmelCase :       Dict  						=  answer['''end_token''']

						__UpperCAmelCase :       Union[str, Any]  						=  []
						for i in range(len(doc['''token''']			)			):
												if not doc["is_html"][i]:
																		context.append(doc['''token'''][i]			)
												else:
																		if answer["start_token"] > i:
																								start_token -= 1
																		if answer["end_token"] > i:
																								end_token -= 1
						__UpperCAmelCase :       List[str]  						=  ''' '''.join(context[start_token:end_token]			)

						# checking above code
						if assertion:
												__UpperCAmelCase :       Dict  						=  doc['''is_html'''][answer['''start_token'''] : answer['''end_token''']]
												__UpperCAmelCase :       Optional[int]  						=  doc['''token'''][answer['''start_token'''] : answer['''end_token''']]
												__UpperCAmelCase :       Tuple  						=  ''' '''.join([old[i] for i in range(len(lowercase_			)			) if not is_html[i]]			)
												if new != old:
																		print('''ID:'''			,					example['''id''']			)
																		print('''New:'''			,					lowercase_			,					end='''\n'''			)
																		print('''Old:'''			,					lowercase_			,					end='''\n\n'''			)

						return {
						    "context": " ".join(lowercase_			),
						    "answer": {
						        "start_token": start_token,
						        "end_token": end_token - 1,  # this makes it inclusive
						        "category": answer["category"],  # either long or short
						        "span": new,  # extra
						    },
						}




def 						__SCREAMING_SNAKE_CASE       (					lowercase_			,					lowercase_			,					lowercase_=2048			,					lowercase_=4096			,					lowercase_=True			)		->    Tuple:




						'''simple docstring'''







						__UpperCAmelCase :       int  						=  get_context_and_ans(lowercase_			,					assertion=lowercase_			)
						__UpperCAmelCase :       int  						=  out['''answer''']

						# later, removing these samples
						if answer["start_token"] == -1:
												return {
												    "example_id": example["id"],
												    "input_ids": [[-1]],
												    "labels": {
												        "start_token": [-1],
												        "end_token": [-1],
												        "category": ["null"],
												    },
												}

						__UpperCAmelCase :       Dict  						=  tokenizer(example['''question''']['''text''']			,					out['''context''']			).input_ids
						__UpperCAmelCase :       Any  						=  input_ids.index(tokenizer.sep_token_id			) + 1

						# return yes/no
						if answer["category"][0] in ["yes", "no"]:  # category is list with one element
												__UpperCAmelCase :       Any  						=  []
												__UpperCAmelCase :       str  						=  []
												__UpperCAmelCase :       str  						=  input_ids[:q_len]
												__UpperCAmelCase :       Any  						=  range(lowercase_			,					len(lowercase_			)			,					max_length - doc_stride			)
												for i in doc_start_indices:
																		__UpperCAmelCase :       Optional[int]  						=  i + max_length - q_len
																		__UpperCAmelCase :       int  						=  input_ids[i:end_index]
																		inputs.append(q_indices + slice			)
																		category.append(answer['''category'''][0]			)
																		if slice[-1] == tokenizer.sep_token_id:
																								break

												return {
												    "example_id": example["id"],
												    "input_ids": inputs,
												    "labels": {
												        "start_token": [-100] * len(lowercase_			),
												        "end_token": [-100] * len(lowercase_			),
												        "category": category,
												    },
												}

						__UpperCAmelCase :       List[Any]  						=  out['''context'''].split()
						__UpperCAmelCase :       Optional[Any]  						=  splitted_context[answer['''end_token''']]
						__UpperCAmelCase :       Dict  						=  len(
						    tokenizer(
						        ''' '''.join(splitted_context[: answer['''start_token''']]			)			,					add_special_tokens=lowercase_			,					).input_ids			)
						__UpperCAmelCase :       Optional[Any]  						=  len(
						    tokenizer(''' '''.join(splitted_context[: answer['''end_token''']]			)			,					add_special_tokens=lowercase_			).input_ids			)

						answer["start_token"] += q_len
						answer["end_token"] += q_len

						# fixing end token
						__UpperCAmelCase :       Dict  						=  len(tokenizer(lowercase_			,					add_special_tokens=lowercase_			).input_ids			)
						if num_sub_tokens > 1:
												answer["end_token"] += num_sub_tokens - 1

						__UpperCAmelCase :       List[str]  						=  input_ids[answer['''start_token'''] : answer['''end_token'''] + 1]  # right & left are inclusive
						__UpperCAmelCase :       str  						=  answer['''start_token''']
						__UpperCAmelCase :       Optional[int]  						=  answer['''end_token''']

						if assertion:
												__UpperCAmelCase :       Optional[int]  						=  tokenizer.decode(lowercase_			)
												if answer["span"] != new:
																		print('''ISSUE IN TOKENIZATION'''			)
																		print('''OLD:'''			,					answer['''span''']			)
																		print('''NEW:'''			,					lowercase_			,					end='''\n\n'''			)

						if len(lowercase_			) <= max_length:
												return {
												    "example_id": example["id"],
												    "input_ids": [input_ids],
												    "labels": {
												        "start_token": [answer["start_token"]],
												        "end_token": [answer["end_token"]],
												        "category": answer["category"],
												    },
												}

						__UpperCAmelCase :       Optional[Any]  						=  input_ids[:q_len]
						__UpperCAmelCase :       Tuple  						=  range(lowercase_			,					len(lowercase_			)			,					max_length - doc_stride			)

						__UpperCAmelCase :       str  						=  []
						__UpperCAmelCase :       List[Any]  						=  []
						__UpperCAmelCase :       Optional[Any]  						=  []
						__UpperCAmelCase :       Any  						=  []  # null, yes, no, long, short
						for i in doc_start_indices:
												__UpperCAmelCase :       Union[str, Any]  						=  i + max_length - q_len
												__UpperCAmelCase :       Optional[int]  						=  input_ids[i:end_index]
												inputs.append(q_indices + slice			)
												assert len(inputs[-1]			) <= max_length, "Issue in truncating length"

												if start_token >= i and end_token <= end_index - 1:
																		__UpperCAmelCase :       int  						=  start_token - i + q_len
																		__UpperCAmelCase :       Any  						=  end_token - i + q_len
																		answers_category.append(answer['''category'''][0]			)  # ["short"] -> "short"
												else:
																		__UpperCAmelCase :       Any  						=  -100
																		__UpperCAmelCase :       List[Any]  						=  -100
																		answers_category.append('''null'''			)
												__UpperCAmelCase :       Dict  						=  inputs[-1][start_token : end_token + 1]

												answers_start_token.append(lowercase_			)
												answers_end_token.append(lowercase_			)
												if assertion:
																		if new != old and new != [tokenizer.cls_token_id]:
																								print('''ISSUE in strided for ID:'''			,					example['''id''']			)
																								print('''New:'''			,					tokenizer.decode(lowercase_			)			)
																								print('''Old:'''			,					tokenizer.decode(lowercase_			)			,					end='''\n\n'''			)
												if slice[-1] == tokenizer.sep_token_id:
																		break

						return {
						    "example_id": example["id"],
						    "input_ids": inputs,
						    "labels": {
						        "start_token": answers_start_token,
						        "end_token": answers_end_token,
						        "category": answers_category,
						    },
						}




def 						__SCREAMING_SNAKE_CASE       (					lowercase_			,					lowercase_			,					lowercase_=2048			,					lowercase_=4096			,					lowercase_=False			)		->    Any:




						'''simple docstring'''







						__UpperCAmelCase :       List[str]  						=  get_strided_contexts_and_ans(
						    lowercase_			,					lowercase_			,					doc_stride=lowercase_			,					max_length=lowercase_			,					assertion=lowercase_			,					)

						return example




def 						__SCREAMING_SNAKE_CASE       (					lowercase_			,					lowercase_			)		->    Union[str, Any]:




						'''simple docstring'''







						with jsonlines.open(lowercase_			,					'''a'''			) as writer:
												for example in tqdm(lowercase_			,					total=len(lowercase_			)			,					desc='''Saving samples ... '''			):
																		__UpperCAmelCase :       Union[str, Any]  						=  example['''labels''']
																		for ids, start, end, cat in zip(
																		    example['''input_ids''']			,					labels['''start_token''']			,					labels['''end_token''']			,					labels['''category''']			,					):
																								if start == -1 and end == -1:
																														continue  # leave waste samples with no answer
																								if cat == "null" and np.random.rand() < 0.6:
																														continue  # removing 50 % samples
																								writer.write(
																								    {
																								        '''input_ids''': ids,
																								        '''start_token''': start,
																								        '''end_token''': end,
																								        '''category''': CATEGORY_MAPPING[cat],
																								    }			)


if __name__ == "__main__":
				from datasets import load_dataset

				from transformers import BigBirdTokenizer

				lowerCAmelCase							=						load_dataset("""natural_questions""")
				lowerCAmelCase							=						BigBirdTokenizer.from_pretrained("""google/bigbird-roberta-base""")

				lowerCAmelCase							=						data["""train""" if PROCESS_TRAIN == """true""" else """validation"""]

				lowerCAmelCase							=						{
				    """tokenizer""": tokenizer,
				    """doc_stride""": DOC_STRIDE,
				    """max_length""": MAX_LENGTH,
				    """assertion""": False,
				}
				lowerCAmelCase							=						data.map(prepare_inputs, fn_kwargs=fn_kwargs)
				lowerCAmelCase							=						data.remove_columns(["""annotations""", """document""", """id""", """question"""])
				print(data)

				np.random.seed(SEED)
				lowerCAmelCase							=						"""nq-training.jsonl""" if PROCESS_TRAIN == """true""" else """nq-validation.jsonl"""
				save_to_disk(data, file_name=cache_file_name)





from random import shuffle

import tensorflow as tf
from numpy import array




def 						__SCREAMING_SNAKE_CASE       (					lowercase_			,					lowercase_			)		->    Optional[int]:




						'''simple docstring'''







						__UpperCAmelCase :       Optional[Any]  						=  int(lowercase_			)
						assert noofclusters < len(lowercase_			)

						# Find out the dimensionality
						__UpperCAmelCase :       str  						=  len(vectors[0]			)

						# Will help select random centroids from among the available vectors
						__UpperCAmelCase :       Union[str, Any]  						=  list(range(len(lowercase_			)			)			)
						shuffle(lowercase_			)

						# GRAPH OF COMPUTATION
						# We initialize a new graph and set it as the default during each run
						# of this algorithm. This ensures that as this function is called
						# multiple times, the default graph doesn't keep getting crowded with
						# unused ops and Variables from previous function calls.

						__UpperCAmelCase :       Union[str, Any]  						=  tf.Graph()

						with graph.as_default():
												# SESSION OF COMPUTATION

												__UpperCAmelCase :       str  						=  tf.Session()

												##CONSTRUCTING THE ELEMENTS OF COMPUTATION

												##First lets ensure we have a Variable vector for each centroid,
												##initialized to one of the vectors from the available data points
												__UpperCAmelCase :       List[str]  						=  [
												    tf.Variable(vectors[vector_indices[i]]			) for i in range(lowercase_			)
												]
												##These nodes will assign the centroid Variables the appropriate
												##values
												__UpperCAmelCase :       str  						=  tf.placeholder('''float64'''			,					[dim]			)
												__UpperCAmelCase :       Tuple  						=  []
												for centroid in centroids:
																		cent_assigns.append(tf.assign(lowercase_			,					lowercase_			)			)

												##Variables for cluster assignments of individual vectors(initialized
												##to 0 at first)
												__UpperCAmelCase :       Union[str, Any]  						=  [tf.Variable(0			) for i in range(len(lowercase_			)			)]
												##These nodes will assign an assignment Variable the appropriate
												##value
												__UpperCAmelCase :       Dict  						=  tf.placeholder('''int32'''			)
												__UpperCAmelCase :       Optional[Any]  						=  []
												for assignment in assignments:
																		cluster_assigns.append(tf.assign(lowercase_			,					lowercase_			)			)

												##Now lets construct the node that will compute the mean
												# The placeholder for the input
												__UpperCAmelCase :       Union[str, Any]  						=  tf.placeholder('''float'''			,					[None, dim]			)
												# The Node/op takes the input and computes a mean along the 0th
												# dimension, i.e. the list of input vectors
												__UpperCAmelCase :       Any  						=  tf.reduce_mean(lowercase_			,					0			)

												##Node for computing Euclidean distances
												# Placeholders for input
												__UpperCAmelCase :       Tuple  						=  tf.placeholder('''float'''			,					[dim]			)
												__UpperCAmelCase :       Any  						=  tf.placeholder('''float'''			,					[dim]			)
												__UpperCAmelCase :       Any  						=  tf.sqrt(tf.reduce_sum(tf.pow(tf.sub(lowercase_			,					lowercase_			)			,					2			)			)			)

												##This node will figure out which cluster to assign a vector to,
												##based on Euclidean distances of the vector from the centroids.
												# Placeholder for input
												__UpperCAmelCase :       Union[str, Any]  						=  tf.placeholder('''float'''			,					[noofclusters]			)
												__UpperCAmelCase :       Optional[Any]  						=  tf.argmin(lowercase_			,					0			)

												##INITIALIZING STATE VARIABLES

												##This will help initialization of all Variables defined with respect
												##to the graph. The Variable-initializer should be defined after
												##all the Variables have been constructed, so that each of them
												##will be included in the initialization.
												__UpperCAmelCase :       Optional[Any]  						=  tf.initialize_all_variables()

												# Initialize all variables
												sess.run(lowercase_			)

												##CLUSTERING ITERATIONS

												# Now perform the Expectation-Maximization steps of K-Means clustering
												# iterations. To keep things simple, we will only do a set number of
												# iterations, instead of using a Stopping Criterion.
												__UpperCAmelCase :       Union[str, Any]  						=  100
												for _ in range(lowercase_			):
																		##EXPECTATION STEP
																		##Based on the centroid locations till last iteration, compute
																		##the _expected_ centroid assignments.
																		# Iterate over each vector
																		for vector_n in range(len(lowercase_			)			):
																								__UpperCAmelCase :       List[str]  						=  vectors[vector_n]
																								# Compute Euclidean distance between this vector and each
																								# centroid. Remember that this list cannot be named
																								#'centroid_distances', since that is the input to the
																								# cluster assignment node.
																								__UpperCAmelCase :       List[Any]  						=  [
																								    sess.run(lowercase_			,					feed_dict={va: vect, va: sess.run(lowercase_			)}			)
																								    for centroid in centroids
																								]
																								# Now use the cluster assignment node, with the distances
																								# as the input
																								__UpperCAmelCase :       Optional[Any]  						=  sess.run(
																								    lowercase_			,					feed_dict={centroid_distances: distances}			)
																								# Now assign the value to the appropriate state variable
																								sess.run(
																								    cluster_assigns[vector_n]			,					feed_dict={assignment_value: assignment}			)

																		##MAXIMIZATION STEP
																		# Based on the expected state computed from the Expectation Step,
																		# compute the locations of the centroids so as to maximize the
																		# overall objective of minimizing within-cluster Sum-of-Squares
																		for cluster_n in range(lowercase_			):
																								# Collect all the vectors assigned to this cluster
																								__UpperCAmelCase :       Optional[Any]  						=  [
																								    vectors[i]
																								    for i in range(len(lowercase_			)			)
																								    if sess.run(assignments[i]			) == cluster_n
																								]
																								# Compute new centroid location
																								__UpperCAmelCase :       str  						=  sess.run(
																								    lowercase_			,					feed_dict={mean_input: array(lowercase_			)}			)
																								# Assign value to appropriate variable
																								sess.run(
																								    cent_assigns[cluster_n]			,					feed_dict={centroid_value: new_location}			)

        # Return centroids and assignments
												__UpperCAmelCase :       List[str]  						=  sess.run(lowercase_			)
												__UpperCAmelCase :       Tuple  						=  sess.run(lowercase_			)
												return centroids, assignments

