





'''simple docstring'''




import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
    _A:					Union[str, Any]							=				"""pt"""
elif is_tf_available():
    _A:					str							=				"""tf"""
else:
    _A:					Dict							=				"""jax"""






class UpperCAmelCase							(		UpperCAmelCase_					,			unittest.TestCase							):
    _A :			str						   =       ByTaTokenizer
    _A :			Optional[int]						   =       False



    def 					__lowerCamelCase						(  self					):
     super().setUp()
     __UpperCAmelCase					=   ByTaTokenizer()
     tokenizer.save_pretrained(self.tmpdirname					)



    @cached_property
    def 					__lowerCamelCase						(  self					):
     return ByTaTokenizer.from_pretrained('google/byt5-small'					)



    def 					__lowerCamelCase						(  self     ,			**__A					):
     return self.tokenizer_class.from_pretrained(self.tmpdirname     ,			**__A					)



    def 					__lowerCamelCase						(  self     ,			__A     ,			__A=False     ,			__A=20     ,			__A=5					):
     # XXX The default common tokenizer tests assume that every ID is decodable on its own.
     # This assumption is invalid for ByT5 because single bytes might not be
     # valid utf-8 (byte 128 for instance).
     # Here we're overriding the smallest possible method to provide
     # a clean sequence without making the same assumption.

     __UpperCAmelCase					=   []
     for i in range(len(__A					)					):
      try:
       __UpperCAmelCase					=   tokenizer.decode([i]     ,			clean_up_tokenization_spaces=__A					)
      except UnicodeDecodeError:
       pass
      toks.append((i, tok)					)

     __UpperCAmelCase					=   list(filter(lambda __A					: re.match(r'^[ a-zA-Z]+$'     ,			t[1]					)     ,			__A					)					)
     __UpperCAmelCase					=   list(filter(lambda __A					: [t[0]] == tokenizer.encode(t[1]     ,			add_special_tokens=__A					)     ,			__A					)					)
     if max_length is not None and len(__A					) > max_length:
      __UpperCAmelCase					=   toks[:max_length]
     if min_length is not None and len(__A					) < min_length and len(__A					) > 0:
      while len(__A					) < min_length:
       __UpperCAmelCase					=   toks + toks
        # toks_str = [t[1] for t in toks]
     __UpperCAmelCase					=   [t[0] for t in toks]

     # Ensure consistency
     __UpperCAmelCase					=   tokenizer.decode(__A     ,			clean_up_tokenization_spaces=__A					)
     if " " not in output_txt and len(__A					) > 1:
      __UpperCAmelCase					=   (
          tokenizer.decode([toks_ids[0]]     ,			clean_up_tokenization_spaces=__A					)
          + ' '
          + tokenizer.decode(toks_ids[1:]     ,			clean_up_tokenization_spaces=__A					)
      )
     if with_prefix_space:
      __UpperCAmelCase					=   ' ' + output_txt
     __UpperCAmelCase					=   tokenizer.encode(__A     ,			add_special_tokens=__A					)
     return output_txt, output_ids



    def 					__lowerCamelCase						(  self					):
     __UpperCAmelCase					=   self.ta_base_tokenizer
     __UpperCAmelCase					=   tokenizer(['hi</s>', 'I went to the gym</s>', '</s>']					)
     __UpperCAmelCase					=   tokenizer(['hi', 'I went to the gym', '']					)
     self.assertListEqual(batch_with_eos_added['input_ids']     ,			batch_without_eos_added['input_ids']					)



    def 					__lowerCamelCase						(  self					):
     __UpperCAmelCase					=   self.ta_base_tokenizer
     __UpperCAmelCase					=   'Unicode €.'
     __UpperCAmelCase					=   tokenizer(__A					)
     __UpperCAmelCase					=   [88, 113, 108, 102, 114, 103, 104, 35, 229, 133, 175, 49, 1]
     self.assertEqual(encoded['input_ids']     ,			__A					)

     # decoding
     __UpperCAmelCase					=   tokenizer.decode(__A					)
     self.assertEqual(__A     ,			'Unicode €.</s>'					)

     __UpperCAmelCase					=   tokenizer('e è é ê ë'					)
     __UpperCAmelCase					=   [104, 35, 198, 171, 35, 198, 172, 35, 198, 173, 35, 198, 174, 1]
     self.assertEqual(encoded['input_ids']     ,			__A					)
     # decoding
     __UpperCAmelCase					=   tokenizer.decode(__A					)
     self.assertEqual(__A     ,			'e è é ê ë</s>'					)

     # encode/decode, but with `encode` instead of `__call__`
     self.assertEqual(tokenizer.decode(tokenizer.encode('e è é ê ë'					)					)     ,			'e è é ê ë</s>'					)



    def 					__lowerCamelCase						(  self					):
     __UpperCAmelCase					=   self.ta_base_tokenizer
     __UpperCAmelCase					=   ['A long paragraph for summarization.', 'Another paragraph for summarization.']
     # fmt: off
     __UpperCAmelCase					=   [68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 1, 0]
     # fmt: on
     __UpperCAmelCase					=   tokenizer(__A     ,			padding=__A     ,			return_tensors=__A					)
     self.assertIsInstance(__A     ,			__A					)

     if FRAMEWORK != "jax":
      __UpperCAmelCase					=   list(batch.input_ids.numpy()[0]					)
     else:
      __UpperCAmelCase					=   list(batch.input_ids.tolist()[0]					)

     self.assertListEqual(__A     ,			__A					)

     self.assertEqual((2, 37)     ,			batch.input_ids.shape					)
     self.assertEqual((2, 37)     ,			batch.attention_mask.shape					)



    def 					__lowerCamelCase						(  self					):
     __UpperCAmelCase					=   self.ta_base_tokenizer
     __UpperCAmelCase					=   ['A long paragraph for summarization.', 'Another paragraph for summarization.']
     __UpperCAmelCase					=   tokenizer(__A     ,			padding=__A     ,			return_tensors=__A					)
     # check if input_ids are returned and no decoder_input_ids
     self.assertIn('input_ids'     ,			__A					)
     self.assertIn('attention_mask'     ,			__A					)
     self.assertNotIn('decoder_input_ids'     ,			__A					)
     self.assertNotIn('decoder_attention_mask'     ,			__A					)



    def 					__lowerCamelCase						(  self					):
     __UpperCAmelCase					=   self.ta_base_tokenizer
     __UpperCAmelCase					=   [
         'Summary of the text.',
         'Another summary.',
     ]
     __UpperCAmelCase					=   tokenizer(
         text_target=__A     ,			max_length=32     ,			padding='max_length'     ,			truncation=__A     ,			return_tensors=__A					)
     self.assertEqual(32     ,			targets['input_ids'].shape[1]					)



    def 					__lowerCamelCase						(  self					):
     __UpperCAmelCase					=   self.ta_base_tokenizer
     __UpperCAmelCase					=   ['A long paragraph for summarization. </s>']
     __UpperCAmelCase					=   ['Summary of the text. </s>']
     # fmt: off
     __UpperCAmelCase					=   [68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 35, 1]
     __UpperCAmelCase					=   [86, 120, 112, 112, 100, 117, 124, 35, 114, 105, 35, 119, 107, 104, 35, 119, 104, 123, 119, 49, 35, 1]
     # fmt: on

     __UpperCAmelCase					=   tokenizer(__A     ,			text_target=__A					)

     self.assertEqual(__A     ,			batch['input_ids'][0]					)
     self.assertEqual(__A     ,			batch['labels'][0]					)



    def 					__lowerCamelCase						(  self					):
     # safety check on max_len default value so we are sure the test works
     __UpperCAmelCase					=   self.get_tokenizers()
     for tokenizer in tokenizers:
      with self.subTest(f'{tokenizer.__class__.__name__}'					):
       self.assertNotEqual(tokenizer.model_max_length     ,			42					)

        # Now let's start the test
     __UpperCAmelCase					=   self.get_tokenizers()
     for tokenizer in tokenizers:
      with self.subTest(f'{tokenizer.__class__.__name__}'					):
       # Isolate this from the other tests because we save additional tokens/etc
       __UpperCAmelCase					=   tempfile.mkdtemp()

       __UpperCAmelCase					=   ' He is very happy, UNwant\u00E9d,running'
       __UpperCAmelCase					=   tokenizer.encode(__A     ,			add_special_tokens=__A					)
       tokenizer.save_pretrained(__A					)

       __UpperCAmelCase					=   tokenizer.__class__.from_pretrained(__A					)
       __UpperCAmelCase					=   after_tokenizer.encode(__A     ,			add_special_tokens=__A					)
       self.assertListEqual(__A     ,			__A					)

       shutil.rmtree(__A					)

     __UpperCAmelCase					=   self.get_tokenizers(model_max_length=42					)
     for tokenizer in tokenizers:
      with self.subTest(f'{tokenizer.__class__.__name__}'					):
       # Isolate this from the other tests because we save additional tokens/etc
       __UpperCAmelCase					=   tempfile.mkdtemp()

       __UpperCAmelCase					=   ' He is very happy, UNwant\u00E9d,running'
       tokenizer.add_tokens(['bim', 'bambam']					)
       __UpperCAmelCase					=   tokenizer.additional_special_tokens
       additional_special_tokens.append('new_additional_special_token'					)
       tokenizer.add_special_tokens({'additional_special_tokens': additional_special_tokens}					)
       __UpperCAmelCase					=   tokenizer.encode(__A     ,			add_special_tokens=__A					)
       tokenizer.save_pretrained(__A					)

       __UpperCAmelCase					=   tokenizer.__class__.from_pretrained(__A					)
       __UpperCAmelCase					=   after_tokenizer.encode(__A     ,			add_special_tokens=__A					)
       self.assertListEqual(__A     ,			__A					)
       self.assertIn('new_additional_special_token'     ,			after_tokenizer.additional_special_tokens					)
       self.assertEqual(after_tokenizer.model_max_length     ,			42					)

       __UpperCAmelCase					=   tokenizer.__class__.from_pretrained(__A     ,			model_max_length=43					)
       self.assertEqual(tokenizer.model_max_length     ,			43					)

       shutil.rmtree(__A					)



    def 					__lowerCamelCase						(  self					):
     __UpperCAmelCase					=   []
     if self.test_slow_tokenizer:
      tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())					)

     if self.test_rust_tokenizer:
      tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())					)

     for tokenizer_class, tokenizer_utils in tokenizer_list:
      with tempfile.TemporaryDirectory() as tmp_dir:
       tokenizer_utils.save_pretrained(__A					)

       with open(os.path.join(__A     ,			'special_tokens_map.json'					)     ,			encoding='utf-8'					) as json_file:
        __UpperCAmelCase					=   json.load(__A					)

       with open(os.path.join(__A     ,			'tokenizer_config.json'					)     ,			encoding='utf-8'					) as json_file:
        __UpperCAmelCase					=   json.load(__A					)

       __UpperCAmelCase					=   [f'<extra_id_{i}>' for i in range(125					)]

       __UpperCAmelCase					=   added_tokens_extra_ids + [
           'an_additional_special_token'
       ]
       __UpperCAmelCase					=   added_tokens_extra_ids + [
           'an_additional_special_token'
       ]

       with open(os.path.join(__A     ,			'special_tokens_map.json'					)     ,			'w'     ,			encoding='utf-8'					) as outfile:
        json.dump(__A     ,			__A					)
       with open(os.path.join(__A     ,			'tokenizer_config.json'					)     ,			'w'     ,			encoding='utf-8'					) as outfile:
        json.dump(__A     ,			__A					)

       # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
       # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
       # "special_tokens_map.json" files
       __UpperCAmelCase					=   tokenizer_class.from_pretrained(
           __A     ,			)
       self.assertIn(
           'an_additional_special_token'     ,			tokenizer_without_change_in_init.additional_special_tokens					)
       # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
       self.assertEqual(
           ['an_additional_special_token']     ,			tokenizer_without_change_in_init.convert_ids_to_tokens(
               tokenizer_without_change_in_init.convert_tokens_to_ids(['an_additional_special_token']					)					)     ,			)

       # Now we test that we can change the value of additional_special_tokens in the from_pretrained
       __UpperCAmelCase					=   added_tokens_extra_ids + [AddedToken('a_new_additional_special_token'     ,			lstrip=__A					)]
       __UpperCAmelCase					=   tokenizer_class.from_pretrained(
           __A     ,			additional_special_tokens=__A     ,			)

       self.assertIn('a_new_additional_special_token'     ,			tokenizer.additional_special_tokens					)
       self.assertEqual(
           ['a_new_additional_special_token']     ,			tokenizer.convert_ids_to_tokens(
               tokenizer.convert_tokens_to_ids(['a_new_additional_special_token']					)					)     ,			)



    def 					__lowerCamelCase						(  self					):
     __UpperCAmelCase					=   []
     if self.test_slow_tokenizer:
      tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())					)

     if self.test_rust_tokenizer:
      tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())					)

     for tokenizer_class, tokenizer_utils in tokenizer_list:
      with tempfile.TemporaryDirectory() as tmp_dir:
       tokenizer_utils.save_pretrained(__A					)

       __UpperCAmelCase					=   tokenizer_class.from_pretrained(__A					)

       self.assertTrue(tokenizer.decode([255]					) == ''					)



    def 					__lowerCamelCase						(  self					):
     pass



    def 					__lowerCamelCase						(  self					):
     pass



    def 					__lowerCamelCase						(  self					):
     pass



    def 					__lowerCamelCase						(  self					):
     pass



    def 					__lowerCamelCase						(  self					):
     # The default common tokenizer tests uses invalid tokens for ByT5 that can only accept one-character strings
     # and special added tokens as tokens
     __UpperCAmelCase					=   self.get_tokenizers(fast=__A     ,			do_lower_case=__A					)
     for tokenizer in tokenizers:
      with self.subTest(f'{tokenizer.__class__.__name__}'					):
       __UpperCAmelCase					=   ['t', 'h', 'i', 's', ' ', 'i', 's', ' ', 'a', ' ', 't', 'e', 'x', 't', '</s>']
       __UpperCAmelCase					=   tokenizer.convert_tokens_to_string(__A					)

       self.assertIsInstance(__A     ,			__A					)




    def 					__lowerCamelCase						(  self					):
     __UpperCAmelCase					=   self.get_tokenizers()
     for tokenizer in tokenizers:
      with self.subTest(f'{tokenizer.__class__.__name__}'					):
       __UpperCAmelCase					=   [
           'bos_token',
           'eos_token',
           'unk_token',
           'sep_token',
           'pad_token',
           'cls_token',
           'mask_token',
       ]

       __UpperCAmelCase					=   0
       __UpperCAmelCase					=   tokenizer.convert_ids_to_tokens(
           __A     ,			skip_special_tokens=__A					)

       for attr in attributes_list:
        setattr(__A     ,			attr + '_id'     ,			__A					)
        self.assertEqual(getattr(__A     ,			__A					)     ,			__A					)
        self.assertEqual(getattr(__A     ,			attr + '_id'					)     ,			__A					)

        setattr(__A     ,			attr + '_id'     ,			__A					)
        self.assertEqual(getattr(__A     ,			__A					)     ,			__A					)
        self.assertEqual(getattr(__A     ,			attr + '_id'					)     ,			__A					)

       setattr(__A     ,			'additional_special_tokens_ids'     ,			[]					)
       self.assertListEqual(getattr(__A     ,			'additional_special_tokens'					)     ,			[]					)
       self.assertListEqual(getattr(__A     ,			'additional_special_tokens_ids'					)     ,			[]					)

       setattr(__A     ,			'additional_special_tokens_ids'     ,			[token_id_to_test_setters]					)
       self.assertListEqual(getattr(__A     ,			'additional_special_tokens'					)     ,			[token_to_test_setters]					)
       self.assertListEqual(getattr(__A     ,			'additional_special_tokens_ids'					)     ,			[token_id_to_test_setters]					)












'''simple docstring'''




import collections
from typing import List, Optional, Union

from ...tokenization_utils_base import BatchEncoding
from ...utils import TensorType, add_end_docstrings, add_start_docstrings, logging
from ..bert.tokenization_bert import BertTokenizer


_A:					List[str]							=				logging.get_logger(__name__)

_A:					Dict							=				{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

_A:					int							=				{
    """vocab_file""": {
        """facebook/dpr-ctx_encoder-single-nq-base""": (
            """https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/vocab.txt"""
        ),
        """facebook/dpr-ctx_encoder-multiset-base""": (
            """https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """facebook/dpr-ctx_encoder-single-nq-base""": (
            """https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/tokenizer.json"""
        ),
        """facebook/dpr-ctx_encoder-multiset-base""": (
            """https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/tokenizer.json"""
        ),
    },
}
_A:					int							=				{
    """vocab_file""": {
        """facebook/dpr-question_encoder-single-nq-base""": (
            """https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/vocab.txt"""
        ),
        """facebook/dpr-question_encoder-multiset-base""": (
            """https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """facebook/dpr-question_encoder-single-nq-base""": (
            """https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/tokenizer.json"""
        ),
        """facebook/dpr-question_encoder-multiset-base""": (
            """https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/tokenizer.json"""
        ),
    },
}
_A:					Dict							=				{
    """vocab_file""": {
        """facebook/dpr-reader-single-nq-base""": (
            """https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/vocab.txt"""
        ),
        """facebook/dpr-reader-multiset-base""": (
            """https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """facebook/dpr-reader-single-nq-base""": (
            """https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/tokenizer.json"""
        ),
        """facebook/dpr-reader-multiset-base""": (
            """https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/tokenizer.json"""
        ),
    },
}

_A:					Any							=				{
    """facebook/dpr-ctx_encoder-single-nq-base""": 512,
    """facebook/dpr-ctx_encoder-multiset-base""": 512,
}
_A:					Tuple							=				{
    """facebook/dpr-question_encoder-single-nq-base""": 512,
    """facebook/dpr-question_encoder-multiset-base""": 512,
}
_A:					int							=				{
    """facebook/dpr-reader-single-nq-base""": 512,
    """facebook/dpr-reader-multiset-base""": 512,
}


_A:					Union[str, Any]							=				{
    """facebook/dpr-ctx_encoder-single-nq-base""": {"""do_lower_case""": True},
    """facebook/dpr-ctx_encoder-multiset-base""": {"""do_lower_case""": True},
}
_A:					List[Any]							=				{
    """facebook/dpr-question_encoder-single-nq-base""": {"""do_lower_case""": True},
    """facebook/dpr-question_encoder-multiset-base""": {"""do_lower_case""": True},
}
_A:					Dict							=				{
    """facebook/dpr-reader-single-nq-base""": {"""do_lower_case""": True},
    """facebook/dpr-reader-multiset-base""": {"""do_lower_case""": True},
}






class UpperCAmelCase							(		UpperCAmelCase_							):
    _A :			Tuple						   =       VOCAB_FILES_NAMES
    _A :			Dict						   =       CONTEXT_ENCODER_PRETRAINED_VOCAB_FILES_MAP
    _A :			Any						   =       CONTEXT_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    _A :			Optional[Any]						   =       CONTEXT_ENCODER_PRETRAINED_INIT_CONFIGURATION






class UpperCAmelCase							(		UpperCAmelCase_							):
    _A :			List[str]						   =       VOCAB_FILES_NAMES
    _A :			str						   =       QUESTION_ENCODER_PRETRAINED_VOCAB_FILES_MAP
    _A :			Union[str, Any]						   =       QUESTION_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    _A :			int						   =       QUESTION_ENCODER_PRETRAINED_INIT_CONFIGURATION



_A:					Any							=				collections.namedtuple(
    """DPRSpanPrediction""", ["""span_score""", """relevance_score""", """doc_id""", """start_index""", """end_index""", """text"""]
)

_A:					Tuple							=				collections.namedtuple("""DPRReaderOutput""", ["""start_logits""", """end_logits""", """relevance_logits"""])


_A:					Optional[Any]							=				r"""
    Return a dictionary with the token ids of the input strings and other information to give to `.decode_best_spans`.
    It converts the strings of a question and different passages (title and text) in a sequence of IDs (integers),
    using the tokenizer and vocabulary. The resulting `input_ids` is a matrix of size `(n_passages, sequence_length)`
    with the format:

    ```
    [CLS] <question token ids> [SEP] <titles ids> [SEP] <texts ids>
    ```

    Args:
        questions (`str` or `List[str]`):
            The questions to be encoded. You can specify one question for many passages. In this case, the question
            will be duplicated like `[questions] * n_passages`. Otherwise you have to specify as many questions as in
            `titles` or `texts`.
        titles (`str` or `List[str]`):
            The passages titles to be encoded. This can be a string or a list of strings if there are several passages.
        texts (`str` or `List[str]`):
            The passages texts to be encoded. This can be a string or a list of strings if there are several passages.
        padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`):
            Activates and controls padding. Accepts the following values:

            - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single sequence
              if provided).
            - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
              acceptable input length for the model if that argument is not provided.
            - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
              lengths).
        truncation (`bool`, `str` or [`~tokenization_utils_base.TruncationStrategy`], *optional*, defaults to `False`):
            Activates and controls truncation. Accepts the following values:

            - `True` or `'longest_first'`: Truncate to a maximum length specified with the argument `max_length` or to
              the maximum acceptable input length for the model if that argument is not provided. This will truncate
              token by token, removing a token from the longest sequence in the pair if a pair of sequences (or a batch
              of pairs) is provided.
            - `'only_first'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum
              acceptable input length for the model if that argument is not provided. This will only truncate the first
              sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
            - `'only_second'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum
              acceptable input length for the model if that argument is not provided. This will only truncate the
              second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
            - `False` or `'do_not_truncate'` (default): No truncation (i.e., can output batch with sequence lengths
              greater than the model maximum admissible input size).
        max_length (`int`, *optional*):
                Controls the maximum length to use by one of the truncation/padding parameters.

                If left unset or set to `None`, this will use the predefined model maximum length if a maximum length
                is required by one of the truncation/padding parameters. If the model has no specific maximum input
                length (like XLNet) truncation/padding to a maximum length will be deactivated.
        return_tensors (`str` or [`~utils.TensorType`], *optional*):
                If set, will return tensors instead of list of python integers. Acceptable values are:

                - `'tf'`: Return TensorFlow `tf.constant` objects.
                - `'pt'`: Return PyTorch `torch.Tensor` objects.
                - `'np'`: Return Numpy `np.ndarray` objects.
        return_attention_mask (`bool`, *optional*):
            Whether or not to return the attention mask. If not set, will return the attention mask according to the
            specific tokenizer's default, defined by the `return_outputs` attribute.

            [What are attention masks?](../glossary#attention-mask)

    Returns:
        `Dict[str, List[List[int]]]`: A dictionary with the following keys:

        - `input_ids`: List of token ids to be fed to a model.
        - `attention_mask`: List of indices specifying which tokens should be attended to by the model.
    """






@add_start_docstrings(UpperCAmelCase_							)
class UpperCAmelCase							:



    def __call__(  self     ,			__A     ,			__A = None     ,			__A = None     ,			__A = False     ,			__A = False     ,			__A = None     ,			__A = None     ,			__A = None     ,			**__A     ,			):
     if titles is None and texts is None:
      return super().__call__(
          __A     ,			padding=__A     ,			truncation=__A     ,			max_length=__A     ,			return_tensors=__A     ,			return_attention_mask=__A     ,			**__A     ,			)
     elif titles is None or texts is None:
      __UpperCAmelCase					=   titles if texts is None else texts
      return super().__call__(
          __A     ,			__A     ,			padding=__A     ,			truncation=__A     ,			max_length=__A     ,			return_tensors=__A     ,			return_attention_mask=__A     ,			**__A     ,			)
     __UpperCAmelCase					=   titles if not isinstance(__A     ,			__A					) else [titles]
     __UpperCAmelCase					=   texts if not isinstance(__A     ,			__A					) else [texts]
     __UpperCAmelCase					=   len(__A					)
     __UpperCAmelCase					=   questions if not isinstance(__A     ,			__A					) else [questions] * n_passages
     if len(__A					) != len(__A					):
      raise ValueError(
          f'There should be as many titles than texts but got {len(__A					)} titles and {len(__A					)} texts.'					)
     __UpperCAmelCase					=   super().__call__(__A     ,			__A     ,			padding=__A     ,			truncation=__A					)['input_ids']
     __UpperCAmelCase					=   super().__call__(__A     ,			add_special_tokens=__A     ,			padding=__A     ,			truncation=__A					)['input_ids']
     __UpperCAmelCase					=   {
         'input_ids': [
             (encoded_question_and_title + encoded_text)[:max_length]
             if max_length is not None and truncation
             else encoded_question_and_title + encoded_text
             for encoded_question_and_title, encoded_text in zip(__A     ,			__A					)
         ]
     }
     if return_attention_mask is not False:
      __UpperCAmelCase					=   []
      for input_ids in encoded_inputs["input_ids"]:
       attention_mask.append([int(input_id != self.pad_token_id					) for input_id in input_ids]					)
      __UpperCAmelCase					=   attention_mask
     return self.pad(__A     ,			padding=__A     ,			max_length=__A     ,			return_tensors=__A					)



    def 					__lowerCamelCase						(  self     ,			__A     ,			__A     ,			__A = 16     ,			__A = 64     ,			__A = 4     ,			):
     __UpperCAmelCase					=   reader_input['input_ids']
     __UpperCAmelCase					,    __UpperCAmelCase					,    __UpperCAmelCase					=   reader_output[:3]
     __UpperCAmelCase					=   len(__A					)
     __UpperCAmelCase					=   sorted(range(__A					)     ,			reverse=__A     ,			key=relevance_logits.__getitem__					)
     __UpperCAmelCase					=   []
     for doc_id in sorted_docs:
      __UpperCAmelCase					=   list(input_ids[doc_id]					)
      # assuming question & title information is at the beginning of the sequence
      __UpperCAmelCase					=   sequence_ids.index(self.sep_token_id     ,			2					) + 1  # second sep id
      if sequence_ids[-1] == self.pad_token_id:
       __UpperCAmelCase					=   sequence_ids.index(self.pad_token_id					)
      else:
       __UpperCAmelCase					=   len(__A					)

      __UpperCAmelCase					=   self._get_best_spans(
          start_logits=start_logits[doc_id][passage_offset:sequence_len]     ,			end_logits=end_logits[doc_id][passage_offset:sequence_len]     ,			max_answer_length=__A     ,			top_spans=__A     ,			)
      for start_index, end_index in best_spans:
       start_index += passage_offset
       end_index += passage_offset
       nbest_spans_predictions.append(
           DPRSpanPrediction(
               span_score=start_logits[doc_id][start_index] + end_logits[doc_id][end_index]     ,			relevance_score=relevance_logits[doc_id]     ,			doc_id=__A     ,			start_index=__A     ,			end_index=__A     ,			text=self.decode(sequence_ids[start_index : end_index + 1]					)     ,			)					)
      if len(__A					) >= num_spans:
       break
     return nbest_spans_predictions[:num_spans]




    def 					__lowerCamelCase						(  self     ,			__A     ,			__A     ,			__A     ,			__A     ,			):
     __UpperCAmelCase					=   []
     for start_index, start_score in enumerate(__A					):
      for answer_length, end_score in enumerate(end_logits[start_index : start_index + max_answer_length]					):
       scores.append(((start_index, start_index + answer_length), start_score + end_score)					)
     __UpperCAmelCase					=   sorted(__A     ,			key=lambda __A					: x[1]     ,			reverse=__A					)
     __UpperCAmelCase					=   []
     for (start_index, end_index), score in scores:
      if start_index > end_index:
       raise ValueError(f'Wrong span indices: [{start_index}:{end_index}]'					)
      __UpperCAmelCase					=   end_index - start_index + 1
      if length > max_answer_length:
       raise ValueError(f'Span is too long: {length} > {max_answer_length}'					)
      if any(
          start_index <= prev_start_index <= prev_end_index <= end_index
          or prev_start_index <= start_index <= end_index <= prev_end_index
          for (prev_start_index, prev_end_index) in chosen_span_intervals					):
       continue
      chosen_span_intervals.append((start_index, end_index)					)

      if len(__A					) == top_spans:
       break
     return chosen_span_intervals






@add_end_docstrings(UpperCAmelCase_							)
class UpperCAmelCase							(		UpperCAmelCase_					,			UpperCAmelCase_							):
    _A :			Optional[int]						   =       VOCAB_FILES_NAMES
    _A :			Optional[int]						   =       READER_PRETRAINED_VOCAB_FILES_MAP
    _A :			Dict						   =       READER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    _A :			Union[str, Any]						   =       READER_PRETRAINED_INIT_CONFIGURATION
    _A :			List[Any]						   =       ["""input_ids""", """attention_mask"""]





