






'''simple docstring'''





import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, MBartaaTokenizer, MBartaaTokenizerFast, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)

from ...test_tokenization_common import TokenizerTesterMixin


snake_case__      :	List[Any]       	=      get_tests_dir("""fixtures/test_sentencepiece.model""")

if is_torch_available():
 from transformers.models.mbart.modeling_mbart import shift_tokens_right

snake_case__      :	str       	=      2_5_0_0_0_4
snake_case__      :	Any       	=      2_5_0_0_2_0


@require_sentencepiece
@require_tokenizers
class 						_a       (  lowercase__						,       unittest.TestCase    ):






  """simple docstring"""





  A_						=   MBartaaTokenizer
  A_						=   MBartaaTokenizerFast
  A_						=   True
  A_						=   True




  def    _UpperCAmelCase     (     self )			-> str:
     super().setUp()

     # We have a SentencePiece fixture for testing
     UpperCamelCase_									=     MBartaaTokenizer(__lowercase		,		src_lang='en_XX'		,		tgt_lang='ro_RO'		,		keep_accents=__lowercase )
     tokenizer.save_pretrained(self.tmpdirname )




  def    _UpperCAmelCase     (     self )			-> Dict:
     UpperCamelCase_									=     '<s>'
     UpperCamelCase_									=     0

     self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowercase )		,		__lowercase )
     self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowercase )		,		__lowercase )




  def    _UpperCAmelCase     (     self )			-> Optional[int]:
     UpperCamelCase_									=     list(self.get_tokenizer().get_vocab().keys() )

     self.assertEqual(vocab_keys[0]		,		'<s>' )
     self.assertEqual(vocab_keys[1]		,		'<pad>' )
     self.assertEqual(vocab_keys[-1]		,		'<mask>' )
     self.assertEqual(len(__lowercase )		,		1054 )




  def    _UpperCAmelCase     (     self )			-> Dict:
     self.assertEqual(self.get_tokenizer().vocab_size		,		1054 )




  def    _UpperCAmelCase     (     self )			-> Any:
     UpperCamelCase_									=     MBartaaTokenizer(__lowercase		,		src_lang='en_XX'		,		tgt_lang='ro_RO'		,		keep_accents=__lowercase )

     UpperCamelCase_									=     tokenizer.tokenize('This is a test' )
     self.assertListEqual(__lowercase		,		['▁This', '▁is', '▁a', '▁t', 'est'] )

     self.assertListEqual(
         tokenizer.convert_tokens_to_ids(__lowercase )		,		[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]		,		)

     UpperCamelCase_									=     tokenizer.tokenize('I was born in 92000, and this is falsé.' )
     self.assertListEqual(
         __lowercase		,		[SPIECE_UNDERLINE + 'I', SPIECE_UNDERLINE + 'was', SPIECE_UNDERLINE + 'b', 'or', 'n', SPIECE_UNDERLINE + 'in', SPIECE_UNDERLINE + '', '9', '2', '0', '0', '0', ',', SPIECE_UNDERLINE + 'and', SPIECE_UNDERLINE + 'this', SPIECE_UNDERLINE + 'is', SPIECE_UNDERLINE + 'f', 'al', 's', 'é', '.']		,		)
     UpperCamelCase_									=     tokenizer.convert_tokens_to_ids(__lowercase )
     self.assertListEqual(
         __lowercase		,		[
             value + tokenizer.fairseq_offset
             for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
         ]		,		)

     UpperCamelCase_									=     tokenizer.convert_ids_to_tokens(__lowercase )
     self.assertListEqual(
         __lowercase		,		[SPIECE_UNDERLINE + 'I', SPIECE_UNDERLINE + 'was', SPIECE_UNDERLINE + 'b', 'or', 'n', SPIECE_UNDERLINE + 'in', SPIECE_UNDERLINE + '', '<unk>', '2', '0', '0', '0', ',', SPIECE_UNDERLINE + 'and', SPIECE_UNDERLINE + 'this', SPIECE_UNDERLINE + 'is', SPIECE_UNDERLINE + 'f', 'al', 's', '<unk>', '.']		,		)




  @slow
  def    _UpperCAmelCase     (     self )			-> str:
     # fmt: off
     UpperCamelCase_									=     {'input_ids': [[250004, 11062, 82772, 7, 15, 82772, 538, 51529, 237, 17198, 1290, 206, 9, 215175, 1314, 136, 17198, 1290, 206, 9, 56359, 42, 122009, 9, 16466, 16, 87344, 4537, 9, 4717, 78381, 6, 159958, 7, 15, 24480, 618, 4, 527, 22693, 5428, 4, 2777, 24480, 9874, 4, 43523, 594, 4, 803, 18392, 33189, 18, 4, 43523, 24447, 12399, 100, 24955, 83658, 9626, 144057, 15, 839, 22335, 16, 136, 24955, 83658, 83479, 15, 39102, 724, 16, 678, 645, 2789, 1328, 4589, 42, 122009, 115774, 23, 805, 1328, 46876, 7, 136, 53894, 1940, 42227, 41159, 17721, 823, 425, 4, 27512, 98722, 206, 136, 5531, 4970, 919, 17336, 5, 2], [250004, 20080, 618, 83, 82775, 47, 479, 9, 1517, 73, 53894, 333, 80581, 110117, 18811, 5256, 1295, 51, 152526, 297, 7986, 390, 124416, 538, 35431, 214, 98, 15044, 25737, 136, 7108, 43701, 23, 756, 135355, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [250004, 581, 63773, 119455, 6, 147797, 88203, 7, 645, 70, 21, 3285, 10269, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
     # fmt: on

     self.tokenizer_integration_test_util(
         expected_encoding=__lowercase		,		model_name='facebook/mbart-large-50'		,		revision='d3913889c59cd5c9e456b269c376325eabad57e2'		,		)




  def    _UpperCAmelCase     (     self )			-> Optional[Any]:
     if not self.test_slow_tokenizer:
        # as we don't have a slow version, we can't compare the outputs between slow and fast versions
        return

     UpperCamelCase_									=     (self.rust_tokenizer_class, 'hf-internal-testing/tiny-random-mbart50', {})
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})""" ):
           UpperCamelCase_									=     self.rust_tokenizer_class.from_pretrained(__lowercase		,		**__lowercase )
           UpperCamelCase_									=     self.tokenizer_class.from_pretrained(__lowercase		,		**__lowercase )

           UpperCamelCase_									=     tempfile.mkdtemp()

           UpperCamelCase_									=     tokenizer_r.save_pretrained(__lowercase )
           UpperCamelCase_									=     tokenizer_p.save_pretrained(__lowercase )

           # Checks it save with the same files + the tokenizer.json file for the fast one
           self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files ) )
           UpperCamelCase_									=     tuple(f for f in tokenizer_r_files if 'tokenizer.json' not in f )
           self.assertSequenceEqual(__lowercase		,		__lowercase )

           # Checks everything loads correctly in the same way
           UpperCamelCase_									=     tokenizer_r.from_pretrained(__lowercase )
           UpperCamelCase_									=     tokenizer_p.from_pretrained(__lowercase )

           # Check special tokens are set accordingly on Rust and Python
           for key in tokenizer_pp.special_tokens_map:
              self.assertTrue(hasattr(__lowercase		,		__lowercase ) )
              # self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
              # self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

           shutil.rmtree(__lowercase )

           # Save tokenizer rust, legacy_format=True
           UpperCamelCase_									=     tempfile.mkdtemp()

           UpperCamelCase_									=     tokenizer_r.save_pretrained(__lowercase		,		legacy_format=__lowercase )
           UpperCamelCase_									=     tokenizer_p.save_pretrained(__lowercase )

           # Checks it save with the same files
           self.assertSequenceEqual(__lowercase		,		__lowercase )

           # Checks everything loads correctly in the same way
           UpperCamelCase_									=     tokenizer_r.from_pretrained(__lowercase )
           UpperCamelCase_									=     tokenizer_p.from_pretrained(__lowercase )

           # Check special tokens are set accordingly on Rust and Python
           for key in tokenizer_pp.special_tokens_map:
              self.assertTrue(hasattr(__lowercase		,		__lowercase ) )

           shutil.rmtree(__lowercase )

           # Save tokenizer rust, legacy_format=False
           UpperCamelCase_									=     tempfile.mkdtemp()

           UpperCamelCase_									=     tokenizer_r.save_pretrained(__lowercase		,		legacy_format=__lowercase )
           UpperCamelCase_									=     tokenizer_p.save_pretrained(__lowercase )

           # Checks it saved the tokenizer.json file
           self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files ) )

           # Checks everything loads correctly in the same way
           UpperCamelCase_									=     tokenizer_r.from_pretrained(__lowercase )
           UpperCamelCase_									=     tokenizer_p.from_pretrained(__lowercase )

           # Check special tokens are set accordingly on Rust and Python
           for key in tokenizer_pp.special_tokens_map:
              self.assertTrue(hasattr(__lowercase		,		__lowercase ) )

           shutil.rmtree(__lowercase )




@require_torch
@require_sentencepiece
@require_tokenizers
class 						_a       (  unittest.TestCase    ):






  """simple docstring"""





  A_						=   """facebook/mbart-large-50-one-to-many-mmt"""
  A_						=   [
      """ UN Chief Says There Is No Military Solution in Syria""",
      """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.""",
  ]
  A_						=   [
      """Şeful ONU declară că nu există o soluţie militară în Siria""",
      """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"""
      """ pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"""
      """ face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
  ]
  A_						=   [EN_CODE, 8_274, 127_873, 25_916, 7, 8_622, 2_071, 438, 67_485, 53, 187_895, 23, 51_712, 2]




  @classmethod
  def    _UpperCAmelCase     (     cls )			-> Any:
     UpperCamelCase_									=     MBartaaTokenizer.from_pretrained(
         cls.checkpoint_name		,		src_lang='en_XX'		,		tgt_lang='ro_RO' )
     UpperCamelCase_									=     1
     return cls




  def    _UpperCAmelCase     (     self )			-> Optional[Any]:
     self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ar_AR']		,		250001 )
     self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['en_EN']		,		250004 )
     self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ro_RO']		,		250020 )
     self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['mr_IN']		,		250038 )




  def    _UpperCAmelCase     (     self )			-> Dict:
     UpperCamelCase_									=     self.tokenizer.batch_encode_plus(self.src_text ).input_ids[0]
     self.assertListEqual(self.expected_src_tokens		,		__lowercase )




  def    _UpperCAmelCase     (     self )			-> Optional[Any]:
     self.assertIn(__lowercase		,		self.tokenizer.all_special_ids )
     UpperCamelCase_									=     [RO_CODE, 884, 9019, 96, 9, 916, 86792, 36, 18743, 15596, 5, 2]
     UpperCamelCase_									=     self.tokenizer.decode(__lowercase		,		skip_special_tokens=__lowercase )
     UpperCamelCase_									=     self.tokenizer.decode(generated_ids[1:]		,		skip_special_tokens=__lowercase )
     self.assertEqual(__lowercase		,		__lowercase )
     self.assertNotIn(self.tokenizer.eos_token		,		__lowercase )




  def    _UpperCAmelCase     (     self )			-> Union[str, Any]:
     UpperCamelCase_									=     ['this is gunna be a long sentence ' * 20]
     assert isinstance(src_text[0]		,		__lowercase )
     UpperCamelCase_									=     10
     UpperCamelCase_									=     self.tokenizer(__lowercase		,		max_length=__lowercase		,		truncation=__lowercase ).input_ids[0]
     self.assertEqual(ids[0]		,		__lowercase )
     self.assertEqual(ids[-1]		,		2 )
     self.assertEqual(len(__lowercase )		,		__lowercase )




  def    _UpperCAmelCase     (     self )			-> Optional[Any]:
     self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', 'ar_AR'] )		,		[250053, 250001] )




  def    _UpperCAmelCase     (     self )			-> Dict:
     UpperCamelCase_									=     tempfile.mkdtemp()
     UpperCamelCase_									=     self.tokenizer.fairseq_tokens_to_ids
     self.tokenizer.save_pretrained(__lowercase )
     UpperCamelCase_									=     MBartaaTokenizer.from_pretrained(__lowercase )
     self.assertDictEqual(new_tok.fairseq_tokens_to_ids		,		__lowercase )




  @require_torch
  def    _UpperCAmelCase     (     self )			-> Any:
     UpperCamelCase_									=     self.tokenizer(self.src_text		,		text_target=self.tgt_text		,		padding=__lowercase		,		return_tensors='pt' )
     UpperCamelCase_									=     shift_tokens_right(batch['labels']		,		self.tokenizer.pad_token_id )

     # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
     assert batch.input_ids[1][0] == EN_CODE
     assert batch.input_ids[1][-1] == 2
     assert batch.labels[1][0] == RO_CODE
     assert batch.labels[1][-1] == 2
     assert batch.decoder_input_ids[1][:2].tolist() == [2, RO_CODE]




  @require_torch
  def    _UpperCAmelCase     (     self )			-> Dict:
     UpperCamelCase_									=     self.tokenizer(
         self.src_text		,		text_target=self.tgt_text		,		padding=__lowercase		,		truncation=__lowercase		,		max_length=len(self.expected_src_tokens )		,		return_tensors='pt'		,		)
     UpperCamelCase_									=     shift_tokens_right(batch['labels']		,		self.tokenizer.pad_token_id )

     self.assertIsInstance(__lowercase		,		__lowercase )

     self.assertEqual((2, 14)		,		batch.input_ids.shape )
     self.assertEqual((2, 14)		,		batch.attention_mask.shape )
     UpperCamelCase_									=     batch.input_ids.tolist()[0]
     self.assertListEqual(self.expected_src_tokens		,		__lowercase )
     self.assertEqual(2		,		batch.decoder_input_ids[0, 0] )  # decoder_start_token_id
     # Test that special tokens are reset
     self.assertEqual(self.tokenizer.prefix_tokens		,		[EN_CODE] )
     self.assertEqual(self.tokenizer.suffix_tokens		,		[self.tokenizer.eos_token_id] )




  def    _UpperCAmelCase     (     self )			-> Union[str, Any]:
     UpperCamelCase_									=     self.tokenizer(self.src_text		,		padding=__lowercase		,		truncation=__lowercase		,		max_length=3		,		return_tensors='pt' )
     UpperCamelCase_									=     self.tokenizer(
         text_target=self.tgt_text		,		padding=__lowercase		,		truncation=__lowercase		,		max_length=10		,		return_tensors='pt' )
     UpperCamelCase_									=     targets['input_ids']
     UpperCamelCase_									=     shift_tokens_right(__lowercase		,		self.tokenizer.pad_token_id )

     self.assertEqual(batch.input_ids.shape[1]		,		3 )
     self.assertEqual(batch.decoder_input_ids.shape[1]		,		10 )




  @require_torch
  def    _UpperCAmelCase     (     self )			-> List[str]:
     UpperCamelCase_									=     self.tokenizer._build_translation_inputs(
         'A test'		,		return_tensors='pt'		,		src_lang='en_XX'		,		tgt_lang='ar_AR' )

     self.assertEqual(
         nested_simplify(__lowercase )		,		{
             # en_XX, A, test, EOS
             'input_ids': [[250004, 62, 3034, 2]],
             'attention_mask': [[1, 1, 1, 1]],
             # ar_AR
             'forced_bos_token_id': 250001,
         }		,		)













import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


snake_case__      :	List[str]       	=      logging.get_logger(__name__)

snake_case__      :	Optional[int]       	=      {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

snake_case__      :	Optional[Any]       	=      {
    """vocab_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"""
        ),
        """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-openqa""": (
            """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-reader""": (
            """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-openqa""": (
            """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-reader""": (
            """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"""
        ),
    },
}

snake_case__      :	List[str]       	=      {
    """google/realm-cc-news-pretrained-embedder""": 5_1_2,
    """google/realm-cc-news-pretrained-encoder""": 5_1_2,
    """google/realm-cc-news-pretrained-scorer""": 5_1_2,
    """google/realm-cc-news-pretrained-openqa""": 5_1_2,
    """google/realm-orqa-nq-openqa""": 5_1_2,
    """google/realm-orqa-nq-reader""": 5_1_2,
    """google/realm-orqa-wq-openqa""": 5_1_2,
    """google/realm-orqa-wq-reader""": 5_1_2,
}

snake_case__      :	Tuple       	=      {
    """google/realm-cc-news-pretrained-embedder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-encoder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-scorer""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-reader""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-reader""": {"""do_lower_case""": True},
}


class 						_a       (  UpperCAmelCase__    ):






  """simple docstring"""





  A_						=   VOCAB_FILES_NAMES
  A_						=   PRETRAINED_VOCAB_FILES_MAP
  A_						=   PRETRAINED_INIT_CONFIGURATION
  A_						=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  A_						=   RealmTokenizer




  def __init__(     self		,		_UpperCAmelCase=None		,		_UpperCAmelCase=None		,		_UpperCAmelCase=True		,		_UpperCAmelCase="[UNK]"		,		_UpperCAmelCase="[SEP]"		,		_UpperCAmelCase="[PAD]"		,		_UpperCAmelCase="[CLS]"		,		_UpperCAmelCase="[MASK]"		,		_UpperCAmelCase=True		,		_UpperCAmelCase=None		,		**_UpperCAmelCase		,		)			-> Optional[Any]:
     super().__init__(
         _UpperCAmelCase		,		tokenizer_file=_UpperCAmelCase		,		do_lower_case=_UpperCAmelCase		,		unk_token=_UpperCAmelCase		,		sep_token=_UpperCAmelCase		,		pad_token=_UpperCAmelCase		,		cls_token=_UpperCAmelCase		,		mask_token=_UpperCAmelCase		,		tokenize_chinese_chars=_UpperCAmelCase		,		strip_accents=_UpperCAmelCase		,		**_UpperCAmelCase		,		)

     UpperCamelCase_									=     json.loads(self.backend_tokenizer.normalizer.__getstate__() )
     if (
         normalizer_state.get('lowercase'		,		_UpperCAmelCase ) != do_lower_case
         or normalizer_state.get('strip_accents'		,		_UpperCAmelCase ) != strip_accents
         or normalizer_state.get('handle_chinese_chars'		,		_UpperCAmelCase ) != tokenize_chinese_chars
     ):
        UpperCamelCase_									=     getattr(_UpperCAmelCase		,		normalizer_state.pop('type' ) )
        UpperCamelCase_									=     do_lower_case
        UpperCamelCase_									=     strip_accents
        UpperCamelCase_									=     tokenize_chinese_chars
        UpperCamelCase_									=     normalizer_class(**_UpperCAmelCase )

     UpperCamelCase_									=     do_lower_case




  def    _UpperCAmelCase     (     self		,		_UpperCAmelCase		,		**_UpperCAmelCase )			-> List[Any]:
     UpperCamelCase_									=     PaddingStrategy.MAX_LENGTH

     UpperCamelCase_									=     text
     UpperCamelCase_									=     kwargs.pop('text_pair'		,		_UpperCAmelCase )
     UpperCamelCase_									=     kwargs.pop('return_tensors'		,		_UpperCAmelCase )

     UpperCamelCase_									=     {
         'input_ids': [],
         'attention_mask': [],
         'token_type_ids': [],
     }

     for idx, candidate_text in enumerate(_UpperCAmelCase ):
        if batch_text_pair is not None:
           UpperCamelCase_									=     batch_text_pair[idx]
        else:
           UpperCamelCase_									=     None

        UpperCamelCase_									=     super().__call__(_UpperCAmelCase		,		_UpperCAmelCase		,		return_tensors=_UpperCAmelCase		,		**_UpperCAmelCase )

        UpperCamelCase_									=     encoded_candidates.get('input_ids' )
        UpperCamelCase_									=     encoded_candidates.get('attention_mask' )
        UpperCamelCase_									=     encoded_candidates.get('token_type_ids' )

        if encoded_input_ids is not None:
           output_data["input_ids"].append(_UpperCAmelCase )
        if encoded_attention_mask is not None:
           output_data["attention_mask"].append(_UpperCAmelCase )
        if encoded_token_type_ids is not None:
           output_data["token_type_ids"].append(_UpperCAmelCase )

     UpperCamelCase_									=     {key: item for key, item in output_data.items() if len(_UpperCAmelCase ) != 0}

     return BatchEncoding(_UpperCAmelCase		,		tensor_type=_UpperCAmelCase )




  def    _UpperCAmelCase     (     self		,		_UpperCAmelCase		,		_UpperCAmelCase=None )			-> Union[str, Any]:
     UpperCamelCase_									=     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

     if token_ids_a:
        output += token_ids_a + [self.sep_token_id]

     return output




  def    _UpperCAmelCase     (     self		,		_UpperCAmelCase		,		_UpperCAmelCase = None )			-> List[int]:
     UpperCamelCase_									=     [self.sep_token_id]
     UpperCamelCase_									=     [self.cls_token_id]
     if token_ids_a is None:
        return len(cls + token_ids_a + sep ) * [0]
     return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]




  def    _UpperCAmelCase     (     self		,		_UpperCAmelCase		,		_UpperCAmelCase = None )			-> Tuple[str]:
     UpperCamelCase_									=     self._tokenizer.model.save(_UpperCAmelCase		,		name=_UpperCAmelCase )
     return tuple(_UpperCAmelCase )





