






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


A__			:    str         =				{
    'configuration_xlm': ['XLM_PRETRAINED_CONFIG_ARCHIVE_MAP', 'XLMConfig', 'XLMOnnxConfig'],
    'tokenization_xlm': ['XLMTokenizer'],
}

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							A__			:    int         =				[
							    'XLM_PRETRAINED_MODEL_ARCHIVE_LIST',
							    'XLMForMultipleChoice',
							    'XLMForQuestionAnswering',
							    'XLMForQuestionAnsweringSimple',
							    'XLMForSequenceClassification',
							    'XLMForTokenClassification',
							    'XLMModel',
							    'XLMPreTrainedModel',
							    'XLMWithLMHeadModel',
							]

try:
							if not is_tf_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							A__			:    str         =				[
							    'TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST',
							    'TFXLMForMultipleChoice',
							    'TFXLMForQuestionAnsweringSimple',
							    'TFXLMForSequenceClassification',
							    'TFXLMForTokenClassification',
							    'TFXLMMainLayer',
							    'TFXLMModel',
							    'TFXLMPreTrainedModel',
							    'TFXLMWithLMHeadModel',
							]


if TYPE_CHECKING:
							from .configuration_xlm import XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMConfig, XLMOnnxConfig
							from .tokenization_xlm import XLMTokenizer

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_xlm import (
														    XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
														    XLMForMultipleChoice,
														    XLMForQuestionAnswering,
														    XLMForQuestionAnsweringSimple,
														    XLMForSequenceClassification,
														    XLMForTokenClassification,
														    XLMModel,
														    XLMPreTrainedModel,
														    XLMWithLMHeadModel,
														)

							try:
														if not is_tf_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_tf_xlm import (
														    TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
														    TFXLMForMultipleChoice,
														    TFXLMForQuestionAnsweringSimple,
														    TFXLMForSequenceClassification,
														    TFXLMForTokenClassification,
														    TFXLMMainLayer,
														    TFXLMModel,
														    TFXLMPreTrainedModel,
														    TFXLMWithLMHeadModel,
														)

else:
							import sys

							A__			:    List[str]         =				_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)












from transformers import DistilBertTokenizer, DistilBertTokenizerFast
from transformers.testing_utils import require_tokenizers, slow

from ..bert.test_tokenization_bert import BertTokenizationTest
@require_tokenizers
class    _UpperCAmelCase      ( A__			):

				"""simple docstring"""




				lowercase__       							= DistilBertTokenizer
				lowercase__       							= DistilBertTokenizerFast
				lowercase__       							= True




				@slow
				def        lowercase__     (  self		:					str						):

								'''simple docstring'''
								lowercase__     				=    DistilBertTokenizer.from_pretrained('''distilbert-base-uncased'''						)

								lowercase__     				=    tokenizer.encode('''sequence builders''',						add_special_tokens=lowerCamelCase						)
								lowercase__     				=    tokenizer.encode('''multi-sequence build''',						add_special_tokens=lowerCamelCase						)

								lowercase__     				=    tokenizer.build_inputs_with_special_tokens(lowerCamelCase						)
								lowercase__     				=    tokenizer.build_inputs_with_special_tokens(lowerCamelCase,						lowerCamelCase						)

								assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
								assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
								    tokenizer.sep_token_id
								]




