






"""simple docstring"""




import json
import os
import unittest

from transformers.models.xlm.tokenization_xlm import VOCAB_FILES_NAMES, XLMTokenizer
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin




class    lowercase_    (_UpperCAmelCase,    unittest.TestCase     ):
						A__	:		Union[str, Any]       			=				XLMTokenizer
						A__	:		Optional[Any]       			=				False






						def 			lowerCamelCase__    (    self	)	->str:







								'''simple docstring'''


								super().setUp()

								# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
								_a							=	[
								    "l",
								    "o",
								    "w",
								    "e",
								    "r",
								    "s",
								    "t",
								    "i",
								    "d",
								    "n",
								    "w</w>",
								    "r</w>",
								    "t</w>",
								    "lo",
								    "low",
								    "er</w>",
								    "low</w>",
								    "lowest</w>",
								    "newer</w>",
								    "wider</w>",
								    "<unk>",
								]
								_a							=	dict(zip(a_     ,     range(len(a_	)	)	)	)
								_a							=	["l o 123", "lo w 1456", "e r</w> 1789", ""]

								_a							=	os.path.join(self.tmpdirname     ,     VOCAB_FILES_NAMES["vocab_file"]	)
								_a							=	os.path.join(self.tmpdirname     ,     VOCAB_FILES_NAMES["merges_file"]	)
								with open(self.vocab_file     ,     "w"	) as fp:
										fp.write(json.dumps(a_	)	)
								with open(self.merges_file     ,     "w"	) as fp:
										fp.write("\n".join(a_	)	)






						def 			lowerCamelCase__    (    self     ,     a_	)	->Dict:







								'''simple docstring'''


								_a							=	"lower newer"
								_a							=	"lower newer"
								return input_text, output_text






						def 			lowerCamelCase__    (    self	)	->List[Any]:







								'''simple docstring'''


								_a							=	XLMTokenizer(self.vocab_file     ,     self.merges_file	)

								_a							=	"lower"
								_a							=	["low", "er</w>"]
								_a							=	tokenizer.tokenize(a_	)
								self.assertListEqual(a_     ,     a_	)

								_a							=	tokens + ["<unk>"]
								_a							=	[1_4, 1_5, 2_0]
								self.assertListEqual(tokenizer.convert_tokens_to_ids(a_	)     ,     a_	)






						@slow
						def 			lowerCamelCase__    (    self	)	->int:







								'''simple docstring'''


								_a							=	XLMTokenizer.from_pretrained("xlm-mlm-en-2048"	)

								_a							=	tokenizer.encode("sequence builders"     ,     add_special_tokens=a_	)
								_a							=	tokenizer.encode("multi-sequence build"     ,     add_special_tokens=a_	)

								_a							=	tokenizer.build_inputs_with_special_tokens(a_	)
								_a							=	tokenizer.build_inputs_with_special_tokens(a_     ,     a_	)

								assert encoded_sentence == [0] + text + [1]
								assert encoded_pair == [0] + text + [1] + text_a + [1]










"""simple docstring"""




import warnings

from ...utils import logging
from .image_processing_yolos import YolosImageProcessor


UpperCamelCase							       =    logging.get_logger(__name__)




class    lowercase_    (_UpperCAmelCase     ):






      def __init__(    self     ,     *a_     ,     **a_	)	->None:







        '''simple docstring'''


        warnings.warn(
            "The class YolosFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please"
            " use YolosImageProcessor instead."     ,     a_     ,     )
        super().__init__(*a_     ,     **a_	)


