



from unittest import TestCase

from datasets import Sequence, Value
from datasets.arrow_dataset import Dataset
class 			_lowerCAmelCase   ( _UpperCAmelCase			):






   """simple docstring"""





   def 		snake_case__			(				self	:       int   )						->					int:
         """simple docstring"""







         return [
             {"col_1": 3, "col_2": "a"},
             {"col_1": 2, "col_2": "b"},
             {"col_1": 1, "col_2": "c"},
             {"col_1": 0, "col_2": "d"},
         ]





   def 		snake_case__			(				self	:       Dict   )						->					Tuple:
         """simple docstring"""







         __lowercase					=				{"""col_1""": [3, 2, 1, 0], """col_2""": ["""a""", """b""", """c""", """d"""]}
         return Dataset.from_dict(lowercase   )





   def 		snake_case__			(				self	:       Optional[int]   )						->					Any:
         """simple docstring"""







         __lowercase					=				self._create_example_records()
         __lowercase					=				Dataset.from_list(lowercase   )
         self.assertListEqual(dset.column_names			,     ["""col_1""", """col_2"""]   )
         for i, r in enumerate(lowercase   ):
               self.assertDictEqual(lowercase			,     example_records[i]   )





   def 		snake_case__			(				self	:       List[str]   )						->					Optional[int]:
         """simple docstring"""







         __lowercase					=				self._create_example_records()
         __lowercase					=				Dataset.from_list(lowercase   )
         __lowercase					=				Dataset.from_dict({k: [r[k] for r in example_records] for k in example_records[0]}   )
         self.assertEqual(dset.info			,     dset_from_dict.info   )





   def 		snake_case__			(				self	:       Tuple   )						->					str:  # checks what happens with missing columns
         """simple docstring"""







         __lowercase					=				[{"""col_1""": 1}, {"""col_2""": """x"""}]
         __lowercase					=				Dataset.from_list(lowercase   )
         self.assertDictEqual(dset[0]			,     {"""col_1""": 1}   )
         self.assertDictEqual(dset[1]			,     {"""col_1""": None}   )  # NB: first record is used for columns





   def 		snake_case__			(				self	:       int   )						->					List[str]:  # checks if the type can be inferred from the second record
         """simple docstring"""







         __lowercase					=				[{"""col_1""": []}, {"""col_1""": [1, 2]}]
         __lowercase					=				Dataset.from_list(lowercase   )
         self.assertEqual(dset.info.features["""col_1"""]			,     Sequence(Value("""int64"""   )   )   )







   def 		snake_case__			(				self	:       Tuple   )						->					Dict:
         """simple docstring"""







         __lowercase					=				Dataset.from_list([]   )
         self.assertEqual(len(lowercase   )			,     0   )
         self.assertListEqual(dset.column_names			,     []   )










import os
import unittest

from transformers import LxmertTokenizer, LxmertTokenizerFast
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 			_lowerCAmelCase   ( _UpperCAmelCase    ,					unittest.TestCase			):






			"""simple docstring"""





			lowercase__      :  Tuple      =			LxmertTokenizer
			lowercase__      :  List[str]      =			LxmertTokenizerFast
			lowercase__      :  Optional[Any]      =			True
			lowercase__      :  List[Any]      =			True





			def 		snake_case__			(				self	:       Tuple   )						->					Tuple:
									"""simple docstring"""







									super().setUp()

									__lowercase					=				[
									    """[UNK]""",
									    """[CLS]""",
									    """[SEP]""",
									    """want""",
									    """##want""",
									    """##ed""",
									    """wa""",
									    """un""",
									    """runn""",
									    """##ing""",
									    """,""",
									    """low""",
									    """lowest""",
									]
									__lowercase					=				os.path.join(self.tmpdirname			,     VOCAB_FILES_NAMES["""vocab_file"""]   )
									with open(self.vocab_file			,     """w"""			,     encoding="""utf-8"""   ) as vocab_writer:
															vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]   )   )





			def 		snake_case__			(				self	:       Optional[int]			,     lowercase	:       int   )						->					List[Any]:
									"""simple docstring"""







									__lowercase					=				"""UNwant\u00E9d,running"""
									__lowercase					=				"""unwanted, running"""
									return input_text, output_text





			def 		snake_case__			(				self	:       str   )						->					Any:
									"""simple docstring"""







									__lowercase					=				self.tokenizer_class(self.vocab_file   )

									__lowercase					=				tokenizer.tokenize("""UNwant\u00E9d,running"""   )
									self.assertListEqual(lowercase			,     ["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]   )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase   )			,     [7, 4, 5, 10, 8, 9]   )







			def 		snake_case__			(				self	:       Union[str, Any]   )						->					int:
									"""simple docstring"""







									if not self.test_rust_tokenizer:
															return

									__lowercase					=				self.get_tokenizer()
									__lowercase					=				self.get_rust_tokenizer()

									__lowercase					=				"""I was born in 92000, and this is falsé."""

									__lowercase					=				tokenizer.tokenize(lowercase   )
									__lowercase					=				rust_tokenizer.tokenize(lowercase   )
									self.assertListEqual(lowercase			,     lowercase   )

									__lowercase					=				tokenizer.encode(lowercase			,     add_special_tokens=lowercase   )
									__lowercase					=				rust_tokenizer.encode(lowercase			,     add_special_tokens=lowercase   )
									self.assertListEqual(lowercase			,     lowercase   )

									__lowercase					=				self.get_rust_tokenizer()
									__lowercase					=				tokenizer.encode(lowercase   )
									__lowercase					=				rust_tokenizer.encode(lowercase   )
									self.assertListEqual(lowercase			,     lowercase   )





