



import os

import jsonlines
import numpy as np
from tqdm import tqdm


snake_case__					:     Dict    		=   20_48
snake_case__					:     str    		=   40_96
snake_case__					:     List[str]    		=   42
snake_case__					:     Dict    		=   os.environ.pop("""PROCESS_TRAIN""", """false""")
snake_case__					:     Optional[Any]    		=   {"""null""": 0, """short""": 1, """long""": 2, """yes""": 3, """no""": 4}



def    snake_case_				(    _SCREAMING_SNAKE_CASE   ):
	def choose_first(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE=False   ):
		assert isinstance(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE   )
		if len(_SCREAMING_SNAKE_CASE   ) == 1:
			__lowercase		       =			answer[0]
			return {k: [answer[k]] for k in answer} if is_long_answer else answer
		for a in answer:
			if is_long_answer:
				__lowercase		       =			{k: [a[k]] for k in a}
			if len(a["start_token"]   ) > 0:
				break
		return a

	__lowercase		       =			{"id": example["id"]}
	__lowercase		       =			example["annotations"]
	__lowercase		       =			annotation["yes_no_answer"]
	if 0 in yes_no_answer or 1 in yes_no_answer:
		__lowercase		       =			["yes"] if 1 in yes_no_answer else ["no"]
		__lowercase		       =			__lowercase		       =			[]
		__lowercase		       =			__lowercase		       =			[]
		__lowercase		       =			["<cls>"]
	else:
		__lowercase		       =			["short"]
		__lowercase		       =			choose_first(annotation["short_answers"]   )
		if len(out["start_token"]   ) == 0:
			# answer will be long if short is not available
			__lowercase		       =			["long"]
			__lowercase		       =			choose_first(annotation["long_answer"]  ,	is_long_answer=_SCREAMING_SNAKE_CASE   )
			__lowercase		       =			[]
		answer.update(_SCREAMING_SNAKE_CASE   )

	# disregard some samples
	if len(answer["start_token"]   ) > 1 or answer["start_token"] == answer["end_token"]:
		__lowercase		       =			True
	else:
		__lowercase		       =			False

	__lowercase		       =			["start_token", "end_token", "start_byte", "end_byte", "text"]
	if not all(isinstance(answer[k]  ,	_SCREAMING_SNAKE_CASE   ) for k in cols   ):
		raise ValueError("Issue in ID"  ,	example["id"]   )

	return answer



def    snake_case_				(    _SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE=False   ):
	__lowercase		       =			_get_single_answer(_SCREAMING_SNAKE_CASE   )
	# bytes are of no use
	del answer["start_byte"]
	del answer["end_byte"]

	# handle yes_no answers explicitly
	if answer["category"][0] in ["yes", "no"]:  # category is list with one element
		__lowercase		       =			example["document"]["tokens"]
		__lowercase		       =			[]
		for i in range(len(doc["token"]   )   ):
			if not doc["is_html"][i]:
				context.append(doc["token"][i]   )
		return {
		    "context": " ".join(_SCREAMING_SNAKE_CASE   ),
		    "answer": {
		        "start_token": -1_0_0,  # ignore index in cross-entropy
		        "end_token": -1_0_0,  # ignore index in cross-entropy
		        "category": answer["category"],
		        "span": answer["category"],  # extra
		    },
		}

	# later, help in removing all no answers
	if answer["start_token"] == [-1]:
		return {
		    "context": "None",
		    "answer": {
		        "start_token": -1,
		        "end_token": -1,
		        "category": "null",
		        "span": "None",  # extra
		    },
		}

	# handling normal samples

	__lowercase		       =			["start_token", "end_token"]
	answer.update({k: answer[k][0] if len(answer[k]   ) > 0 else answer[k] for k in cols}   )  # e.g. [10] == 10

	__lowercase		       =			example["document"]["tokens"]
	__lowercase		       =			answer["start_token"]
	__lowercase		       =			answer["end_token"]

	__lowercase		       =			[]
	for i in range(len(doc["token"]   )   ):
		if not doc["is_html"][i]:
			context.append(doc["token"][i]   )
		else:
			if answer["start_token"] > i:
				start_token -= 1
			if answer["end_token"] > i:
				end_token -= 1
	__lowercase		       =			" ".join(context[start_token:end_token]   )

	# checking above code
	if assertion:
		__lowercase		       =			doc["is_html"][answer["start_token"] : answer["end_token"]]
		__lowercase		       =			doc["token"][answer["start_token"] : answer["end_token"]]
		__lowercase		       =			" ".join([old[i] for i in range(len(_SCREAMING_SNAKE_CASE   )   ) if not is_html[i]]   )
		if new != old:
			print("ID:"  ,	example["id"]   )
			print("New:"  ,	_SCREAMING_SNAKE_CASE  ,	end="\n"   )
			print("Old:"  ,	_SCREAMING_SNAKE_CASE  ,	end="\n\n"   )

	return {
	    "context": " ".join(_SCREAMING_SNAKE_CASE   ),
	    "answer": {
	        "start_token": start_token,
	        "end_token": end_token - 1,  # this makes it inclusive
	        "category": answer["category"],  # either long or short
	        "span": new,  # extra
	    },
	}



def    snake_case_				(    _SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE=2_0_4_8  ,	_SCREAMING_SNAKE_CASE=4_0_9_6  ,	_SCREAMING_SNAKE_CASE=True   ):
	# overlap will be of doc_stride - q_len

	__lowercase		       =			get_context_and_ans(_SCREAMING_SNAKE_CASE  ,	assertion=_SCREAMING_SNAKE_CASE   )
	__lowercase		       =			out["answer"]

	# later, removing these samples
	if answer["start_token"] == -1:
		return {
		    "example_id": example["id"],
		    "input_ids": [[-1]],
		    "labels": {
		        "start_token": [-1],
		        "end_token": [-1],
		        "category": ["null"],
		    },
		}

	__lowercase		       =			tokenizer(example["question"]["text"]  ,	out["context"]   ).input_ids
	__lowercase		       =			input_ids.index(tokenizer.sep_token_id   ) + 1

	# return yes/no
	if answer["category"][0] in ["yes", "no"]:  # category is list with one element
		__lowercase		       =			[]
		__lowercase		       =			[]
		__lowercase		       =			input_ids[:q_len]
		__lowercase		       =			range(_SCREAMING_SNAKE_CASE  ,	len(_SCREAMING_SNAKE_CASE   )  ,	max_length - doc_stride   )
		for i in doc_start_indices:
			__lowercase		       =			i + max_length - q_len
			__lowercase		       =			input_ids[i:end_index]
			inputs.append(q_indices + slice   )
			category.append(answer["category"][0]   )
			if slice[-1] == tokenizer.sep_token_id:
				break

		return {
		    "example_id": example["id"],
		    "input_ids": inputs,
		    "labels": {
		        "start_token": [-1_0_0] * len(_SCREAMING_SNAKE_CASE   ),
		        "end_token": [-1_0_0] * len(_SCREAMING_SNAKE_CASE   ),
		        "category": category,
		    },
		}

	__lowercase		       =			out["context"].split()
	__lowercase		       =			splitted_context[answer["end_token"]]
	__lowercase		       =			len(
	    tokenizer(
	        " ".join(splitted_context[: answer["start_token"]]   )  ,	add_special_tokens=_SCREAMING_SNAKE_CASE  ,	).input_ids   )
	__lowercase		       =			len(
	    tokenizer(" ".join(splitted_context[: answer["end_token"]]   )  ,	add_special_tokens=_SCREAMING_SNAKE_CASE   ).input_ids   )

	answer["start_token"] += q_len
	answer["end_token"] += q_len

	# fixing end token
	__lowercase		       =			len(tokenizer(_SCREAMING_SNAKE_CASE  ,	add_special_tokens=_SCREAMING_SNAKE_CASE   ).input_ids   )
	if num_sub_tokens > 1:
		answer["end_token"] += num_sub_tokens - 1

	__lowercase		       =			input_ids[answer["start_token"] : answer["end_token"] + 1]  # right & left are inclusive
	__lowercase		       =			answer["start_token"]
	__lowercase		       =			answer["end_token"]

	if assertion:
		__lowercase		       =			tokenizer.decode(_SCREAMING_SNAKE_CASE   )
		if answer["span"] != new:
			print("ISSUE IN TOKENIZATION"   )
			print("OLD:"  ,	answer["span"]   )
			print("NEW:"  ,	_SCREAMING_SNAKE_CASE  ,	end="\n\n"   )

	if len(_SCREAMING_SNAKE_CASE   ) <= max_length:
		return {
		    "example_id": example["id"],
		    "input_ids": [input_ids],
		    "labels": {
		        "start_token": [answer["start_token"]],
		        "end_token": [answer["end_token"]],
		        "category": answer["category"],
		    },
		}

	__lowercase		       =			input_ids[:q_len]
	__lowercase		       =			range(_SCREAMING_SNAKE_CASE  ,	len(_SCREAMING_SNAKE_CASE   )  ,	max_length - doc_stride   )

	__lowercase		       =			[]
	__lowercase		       =			[]
	__lowercase		       =			[]
	__lowercase		       =			[]  # null, yes, no, long, short
	for i in doc_start_indices:
		__lowercase		       =			i + max_length - q_len
		__lowercase		       =			input_ids[i:end_index]
		inputs.append(q_indices + slice   )
		assert len(inputs[-1]   ) <= max_length, "Issue in truncating length"

		if start_token >= i and end_token <= end_index - 1:
			__lowercase		       =			start_token - i + q_len
			__lowercase		       =			end_token - i + q_len
			answers_category.append(answer["category"][0]   )  # ["short"] -> "short"
		else:
			__lowercase		       =			-1_0_0
			__lowercase		       =			-1_0_0
			answers_category.append("null"   )
		__lowercase		       =			inputs[-1][start_token : end_token + 1]

		answers_start_token.append(_SCREAMING_SNAKE_CASE   )
		answers_end_token.append(_SCREAMING_SNAKE_CASE   )
		if assertion:
			if new != old and new != [tokenizer.cls_token_id]:
				print("ISSUE in strided for ID:"  ,	example["id"]   )
				print("New:"  ,	tokenizer.decode(_SCREAMING_SNAKE_CASE   )   )
				print("Old:"  ,	tokenizer.decode(_SCREAMING_SNAKE_CASE   )  ,	end="\n\n"   )
		if slice[-1] == tokenizer.sep_token_id:
			break

	return {
	    "example_id": example["id"],
	    "input_ids": inputs,
	    "labels": {
	        "start_token": answers_start_token,
	        "end_token": answers_end_token,
	        "category": answers_category,
	    },
	}



def    snake_case_				(    _SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE=2_0_4_8  ,	_SCREAMING_SNAKE_CASE=4_0_9_6  ,	_SCREAMING_SNAKE_CASE=False   ):
	__lowercase		       =			get_strided_contexts_and_ans(
	    _SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	doc_stride=_SCREAMING_SNAKE_CASE  ,	max_length=_SCREAMING_SNAKE_CASE  ,	assertion=_SCREAMING_SNAKE_CASE  ,	)

	return example



def    snake_case_				(    _SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE   ):
	with jsonlines.open(_SCREAMING_SNAKE_CASE  ,	"a"   ) as writer:
		for example in tqdm(_SCREAMING_SNAKE_CASE  ,	total=len(_SCREAMING_SNAKE_CASE   )  ,	desc="Saving samples ... "   ):
			__lowercase		       =			example["labels"]
			for ids, start, end, cat in zip(
			    example["input_ids"]  ,	labels["start_token"]  ,	labels["end_token"]  ,	labels["category"]  ,	):
				if start == -1 and end == -1:
					continue  # leave waste samples with no answer
				if cat == "null" and np.random.rand() < 0.6:
					continue  # removing 50 % samples
				writer.write(
				    {
				        "input_ids": ids,
				        "start_token": start,
				        "end_token": end,
				        "category": CATEGORY_MAPPING[cat],
				    }   )


if __name__ == "__main__":
	from datasets import load_dataset

	from transformers import BigBirdTokenizer

	snake_case__					:     Union[str, Any]    		=   load_dataset("""natural_questions""")
	snake_case__					:     Union[str, Any]    		=   BigBirdTokenizer.from_pretrained("""google/bigbird-roberta-base""")

	snake_case__					:     List[Any]    		=   data["""train""" if PROCESS_TRAIN == """true""" else """validation"""]

	snake_case__					:     List[str]    		=   {
	    """tokenizer""": tokenizer,
	    """doc_stride""": DOC_STRIDE,
	    """max_length""": MAX_LENGTH,
	    """assertion""": False,
	}
	snake_case__					:     str    		=   data.map(prepare_inputs, fn_kwargs=fn_kwargs)
	snake_case__					:     Dict    		=   data.remove_columns(["""annotations""", """document""", """id""", """question"""])
	print(data)

	np.random.seed(SEED)
	snake_case__					:     str    		=   """nq-training.jsonl""" if PROCESS_TRAIN == """true""" else """nq-validation.jsonl"""
	save_to_disk(data, file_name=cache_file_name)








import numpy as np

snake_case__					:     Tuple    		=   [
    ["""a""", """b""", """c""", """d""", """e"""],
    ["""f""", """g""", """h""", """i""", """k"""],
    ["""l""", """m""", """n""", """o""", """p"""],
    ["""q""", """r""", """s""", """t""", """u"""],
    ["""v""", """w""", """x""", """y""", """z"""],
]

class   _A   :


						'''simple docstring'''





						def __init__(	self	:							Dict	):






							'''simple docstring'''



							__lowercase		       =			np.array(lowerCamelCase	)



						def     _snake_case	(	self	:							Union[str, Any]							,      lowerCamelCase	:							str	):






							'''simple docstring'''



							__lowercase       ,    __lowercase		       =			np.where(letter == self.SQUARE	)
							__lowercase		       =			np.concatenate([indexa + 1, indexa + 1]	)
							return indexes



						def     _snake_case	(	self	:							List[Any]							,      lowerCamelCase	:							int							,      lowerCamelCase	:							int	):






							'''simple docstring'''



							__lowercase		       =			self.SQUARE[indexa - 1, indexa - 1]
							return letter



						def     _snake_case	(	self	:							int							,      lowerCamelCase	:							str	):






							'''simple docstring'''



							__lowercase		       =			message.lower()
							__lowercase		       =			message.replace(" "							,      ""	)
							__lowercase		       =			message.replace("j"							,      "i"	)

							__lowercase		       =			np.empty((2, len(lowerCamelCase	))	)
							for letter_index in range(len(lowerCamelCase	)	):
								__lowercase		       =			self.letter_to_numbers(message[letter_index]	)

								__lowercase		       =			numbers[0]
								__lowercase		       =			numbers[1]

							__lowercase		       =			first_step.reshape(2 * len(lowerCamelCase	)	)
							__lowercase		       =			""
							for numbers_index in range(len(lowerCamelCase	)	):
								__lowercase		       =			int(second_step[numbers_index * 2]	)
								__lowercase		       =			int(second_step[(numbers_index * 2) + 1]	)
								__lowercase		       =			self.numbers_to_letter(lowerCamelCase							,      lowerCamelCase	)
								__lowercase		       =			encoded_message + letter

							return encoded_message



						def     _snake_case	(	self	:							Optional[Any]							,      lowerCamelCase	:							str	):






							'''simple docstring'''



							__lowercase		       =			message.lower()
							message.replace(" "							,      ""	)
							__lowercase		       =			np.empty(2 * len(lowerCamelCase	)	)
							for letter_index in range(len(lowerCamelCase	)	):
								__lowercase		       =			self.letter_to_numbers(message[letter_index]	)
								__lowercase		       =			numbers[0]
								__lowercase		       =			numbers[1]

							__lowercase		       =			first_step.reshape((2, len(lowerCamelCase	))	)
							__lowercase		       =			""
							for numbers_index in range(len(lowerCamelCase	)	):
								__lowercase		       =			int(second_step[0, numbers_index]	)
								__lowercase		       =			int(second_step[1, numbers_index]	)
								__lowercase		       =			self.numbers_to_letter(lowerCamelCase							,      lowerCamelCase	)
								__lowercase		       =			decoded_message + letter

							return decoded_message



