






import os

import jsonlines
import numpy as np
from tqdm import tqdm


A__			:    Dict         =				20_48
A__			:    Dict         =				40_96
A__			:    List[Any]         =				42
A__			:    List[str]         =				os.environ.pop('PROCESS_TRAIN', 'false')
A__			:    Optional[int]         =				{"null": 0, "short": 1, "long": 2, "yes": 3, "no": 4}






def 						a    (		lowerCamelCase_       ):






				'''simple docstring'''


				def choose_first(lowerCamelCase_	,     lowerCamelCase_=False       ):
								assert isinstance(_UpperCamelCase	,     _UpperCamelCase       )
								if len(_UpperCamelCase       ) == 1:
												lowercase__     				=    answer[0]
												return {k: [answer[k]] for k in answer} if is_long_answer else answer
								for a in answer:
												if is_long_answer:
																lowercase__     				=    {k: [a[k]] for k in a}
												if len(a['''start_token''']       ) > 0:
																break
								return a

				lowercase__     				=    {'''id''': example['''id''']}
				lowercase__     				=    example['''annotations''']
				lowercase__     				=    annotation['''yes_no_answer''']
				if 0 in yes_no_answer or 1 in yes_no_answer:
								lowercase__     				=    ['''yes'''] if 1 in yes_no_answer else ['''no''']
								lowercase__     				=    lowercase__     				=    []
								lowercase__     				=    lowercase__     				=    []
								lowercase__     				=    ['''<cls>''']
				else:
								lowercase__     				=    ['''short''']
								lowercase__     				=    choose_first(annotation['''short_answers''']       )
								if len(out['''start_token''']       ) == 0:
												# answer will be long if short is not available
												lowercase__     				=    ['''long''']
												lowercase__     				=    choose_first(annotation['''long_answer''']	,     is_long_answer=_UpperCamelCase       )
												lowercase__     				=    []
								answer.update(_UpperCamelCase       )

				# disregard some samples
				if len(answer['''start_token''']       ) > 1 or answer["start_token"] == answer["end_token"]:
								lowercase__     				=    True
				else:
								lowercase__     				=    False

				lowercase__     				=    ['''start_token''', '''end_token''', '''start_byte''', '''end_byte''', '''text''']
				if not all(isinstance(answer[k]	,     _UpperCamelCase       ) for k in cols       ):
								raise ValueError('''Issue in ID'''	,     example['''id''']       )

				return answer






def 						a    (		lowerCamelCase_	,     lowerCamelCase_=False       ):






				'''simple docstring'''
				lowercase__     				=    _get_single_answer(_UpperCamelCase       )
				# bytes are of no use
				del answer["start_byte"]
				del answer["end_byte"]

				# handle yes_no answers explicitly
				if answer["category"][0] in ["yes", "no"]:  # category is list with one element
								lowercase__     				=    example['''document''']['''tokens''']
								lowercase__     				=    []
								for i in range(len(doc['''token''']       )       ):
												if not doc["is_html"][i]:
																context.append(doc['''token'''][i]       )
								return {
								    "context": " ".join(_UpperCamelCase       ),
								    "answer": {
								        "start_token": -100,  # ignore index in cross-entropy
								        "end_token": -100,  # ignore index in cross-entropy
								        "category": answer["category"],
								        "span": answer["category"],  # extra
								    },
								}

				# later, help in removing all no answers
				if answer["start_token"] == [-1]:
								return {
								    "context": "None",
								    "answer": {
								        "start_token": -1,
								        "end_token": -1,
								        "category": "null",
								        "span": "None",  # extra
								    },
								}

				# handling normal samples

				lowercase__     				=    ['''start_token''', '''end_token''']
				answer.update({k: answer[k][0] if len(answer[k]       ) > 0 else answer[k] for k in cols}       )  # e.g. [10] == 10

				lowercase__     				=    example['''document''']['''tokens''']
				lowercase__     				=    answer['''start_token''']
				lowercase__     				=    answer['''end_token''']

				lowercase__     				=    []
				for i in range(len(doc['''token''']       )       ):
								if not doc["is_html"][i]:
												context.append(doc['''token'''][i]       )
								else:
												if answer["start_token"] > i:
																start_token -= 1
												if answer["end_token"] > i:
																end_token -= 1
				lowercase__     				=    ''' '''.join(context[start_token:end_token]       )

				# checking above code
				if assertion:
								lowercase__     				=    doc['''is_html'''][answer['''start_token'''] : answer['''end_token''']]
								lowercase__     				=    doc['''token'''][answer['''start_token'''] : answer['''end_token''']]
								lowercase__     				=    ''' '''.join([old[i] for i in range(len(_UpperCamelCase       )       ) if not is_html[i]]       )
								if new != old:
												print('''ID:'''	,     example['''id''']       )
												print('''New:'''	,     _UpperCamelCase	,     end='''\n'''       )
												print('''Old:'''	,     _UpperCamelCase	,     end='''\n\n'''       )

				return {
				    "context": " ".join(_UpperCamelCase       ),
				    "answer": {
				        "start_token": start_token,
				        "end_token": end_token - 1,  # this makes it inclusive
				        "category": answer["category"],  # either long or short
				        "span": new,  # extra
				    },
				}






def 						a    (		lowerCamelCase_	,     lowerCamelCase_	,     lowerCamelCase_=2048	,     lowerCamelCase_=4096	,     lowerCamelCase_=True       ):






				'''simple docstring'''
				lowercase__     				=    get_context_and_ans(_UpperCamelCase	,     assertion=_UpperCamelCase       )
				lowercase__     				=    out['''answer''']

				# later, removing these samples
				if answer["start_token"] == -1:
								return {
								    "example_id": example["id"],
								    "input_ids": [[-1]],
								    "labels": {
								        "start_token": [-1],
								        "end_token": [-1],
								        "category": ["null"],
								    },
								}

				lowercase__     				=    tokenizer(example['''question''']['''text''']	,     out['''context''']       ).input_ids
				lowercase__     				=    input_ids.index(tokenizer.sep_token_id       ) + 1

				# return yes/no
				if answer["category"][0] in ["yes", "no"]:  # category is list with one element
								lowercase__     				=    []
								lowercase__     				=    []
								lowercase__     				=    input_ids[:q_len]
								lowercase__     				=    range(_UpperCamelCase	,     len(_UpperCamelCase       )	,     max_length - doc_stride       )
								for i in doc_start_indices:
												lowercase__     				=    i + max_length - q_len
												lowercase__     				=    input_ids[i:end_index]
												inputs.append(q_indices + slice       )
												category.append(answer['''category'''][0]       )
												if slice[-1] == tokenizer.sep_token_id:
																break

								return {
								    "example_id": example["id"],
								    "input_ids": inputs,
								    "labels": {
								        "start_token": [-100] * len(_UpperCamelCase       ),
								        "end_token": [-100] * len(_UpperCamelCase       ),
								        "category": category,
								    },
								}

				lowercase__     				=    out['''context'''].split()
				lowercase__     				=    splitted_context[answer['''end_token''']]
				lowercase__     				=    len(
				    tokenizer(
				        ''' '''.join(splitted_context[: answer['''start_token''']]       )	,     add_special_tokens=_UpperCamelCase	,     ).input_ids       )
				lowercase__     				=    len(
				    tokenizer(''' '''.join(splitted_context[: answer['''end_token''']]       )	,     add_special_tokens=_UpperCamelCase       ).input_ids       )

				answer["start_token"] += q_len
				answer["end_token"] += q_len

				# fixing end token
				lowercase__     				=    len(tokenizer(_UpperCamelCase	,     add_special_tokens=_UpperCamelCase       ).input_ids       )
				if num_sub_tokens > 1:
								answer["end_token"] += num_sub_tokens - 1

				lowercase__     				=    input_ids[answer['''start_token'''] : answer['''end_token'''] + 1]  # right & left are inclusive
				lowercase__     				=    answer['''start_token''']
				lowercase__     				=    answer['''end_token''']

				if assertion:
								lowercase__     				=    tokenizer.decode(_UpperCamelCase       )
								if answer["span"] != new:
												print('''ISSUE IN TOKENIZATION'''       )
												print('''OLD:'''	,     answer['''span''']       )
												print('''NEW:'''	,     _UpperCamelCase	,     end='''\n\n'''       )

				if len(_UpperCamelCase       ) <= max_length:
								return {
								    "example_id": example["id"],
								    "input_ids": [input_ids],
								    "labels": {
								        "start_token": [answer["start_token"]],
								        "end_token": [answer["end_token"]],
								        "category": answer["category"],
								    },
								}

				lowercase__     				=    input_ids[:q_len]
				lowercase__     				=    range(_UpperCamelCase	,     len(_UpperCamelCase       )	,     max_length - doc_stride       )

				lowercase__     				=    []
				lowercase__     				=    []
				lowercase__     				=    []
				lowercase__     				=    []  # null, yes, no, long, short
				for i in doc_start_indices:
								lowercase__     				=    i + max_length - q_len
								lowercase__     				=    input_ids[i:end_index]
								inputs.append(q_indices + slice       )
								assert len(inputs[-1]       ) <= max_length, "Issue in truncating length"

								if start_token >= i and end_token <= end_index - 1:
												lowercase__     				=    start_token - i + q_len
												lowercase__     				=    end_token - i + q_len
												answers_category.append(answer['''category'''][0]       )  # ["short"] -> "short"
								else:
												lowercase__     				=    -100
												lowercase__     				=    -100
												answers_category.append('''null'''       )
								lowercase__     				=    inputs[-1][start_token : end_token + 1]

								answers_start_token.append(_UpperCamelCase       )
								answers_end_token.append(_UpperCamelCase       )
								if assertion:
												if new != old and new != [tokenizer.cls_token_id]:
																print('''ISSUE in strided for ID:'''	,     example['''id''']       )
																print('''New:'''	,     tokenizer.decode(_UpperCamelCase       )       )
																print('''Old:'''	,     tokenizer.decode(_UpperCamelCase       )	,     end='''\n\n'''       )
								if slice[-1] == tokenizer.sep_token_id:
												break

				return {
				    "example_id": example["id"],
				    "input_ids": inputs,
				    "labels": {
				        "start_token": answers_start_token,
				        "end_token": answers_end_token,
				        "category": answers_category,
				    },
				}






def 						a    (		lowerCamelCase_	,     lowerCamelCase_	,     lowerCamelCase_=2048	,     lowerCamelCase_=4096	,     lowerCamelCase_=False       ):






				'''simple docstring'''
				lowercase__     				=    get_strided_contexts_and_ans(
				    _UpperCamelCase	,     _UpperCamelCase	,     doc_stride=_UpperCamelCase	,     max_length=_UpperCamelCase	,     assertion=_UpperCamelCase	,     )

				return example






def 						a    (		lowerCamelCase_	,     lowerCamelCase_       ):






				'''simple docstring'''


				with jsonlines.open(_UpperCamelCase	,     '''a'''       ) as writer:
								for example in tqdm(_UpperCamelCase	,     total=len(_UpperCamelCase       )	,     desc='''Saving samples ... '''       ):
												lowercase__     				=    example['''labels''']
												for ids, start, end, cat in zip(
												    example['''input_ids''']	,     labels['''start_token''']	,     labels['''end_token''']	,     labels['''category''']	,     ):
																if start == -1 and end == -1:
																				continue  # leave waste samples with no answer
																if cat == "null" and np.random.rand() < 0.6:
																				continue  # removing 50 % samples
																writer.write(
																    {
																        '''input_ids''': ids,
																        '''start_token''': start,
																        '''end_token''': end,
																        '''category''': CATEGORY_MAPPING[cat],
																    }       )


if __name__ == "__main__":
							from datasets import load_dataset

							from transformers import BigBirdTokenizer

							A__			:    List[Any]         =				load_dataset('natural_questions')
							A__			:    Any         =				BigBirdTokenizer.from_pretrained('google/bigbird-roberta-base')

							A__			:    Dict         =				data["train" if PROCESS_TRAIN == "true" else "validation"]

							A__			:    List[Any]         =				{
							    "tokenizer": tokenizer,
							    "doc_stride": DOC_STRIDE,
							    "max_length": MAX_LENGTH,
							    "assertion": False,
							}
							A__			:    List[str]         =				data.map(prepare_inputs, fn_kwargs=fn_kwargs)
							A__			:    Optional[Any]         =				data.remove_columns(['annotations', 'document', 'id', 'question'])
							print(data)

							np.random.seed(SEED)
							A__			:    Optional[int]         =				"nq-training.jsonl" if PROCESS_TRAIN == "true" else "nq-validation.jsonl"
							save_to_disk(data, file_name=cache_file_name)












from collections import defaultdict
from math import gcd






def 						a    (		lowerCamelCase_ = 150_0000       ):






				'''simple docstring'''
				lowercase__     				=    defaultdict(lowerCamelCase_       )
				lowercase__     				=    2
				while 2 * euclid_m * (euclid_m + 1) <= limit:
								for euclid_n in range((euclid_m % 2) + 1	,     lowerCamelCase_	,     2       ):
												if gcd(lowerCamelCase_	,     lowerCamelCase_       ) > 1:
																continue
												lowercase__     				=    2 * euclid_m * (euclid_m + euclid_n)
												for perimeter in range(lowerCamelCase_	,     limit + 1	,     lowerCamelCase_       ):
																frequencies[perimeter] += 1
								euclid_m += 1
				return sum(1 for frequency in frequencies.values() if frequency == 1       )


if __name__ == "__main__":
							print(F"{solution() = }")




