






import os
import tempfile
import unittest

from transformers import FlaubertConfig, is_torch_available
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch

	from transformers import (
	    FlaubertForMultipleChoice,
	    FlaubertForQuestionAnswering,
	    FlaubertForQuestionAnsweringSimple,
	    FlaubertForSequenceClassification,
	    FlaubertForTokenClassification,
	    FlaubertModel,
	    FlaubertWithLMHeadModel,
	)
	from transformers.models.flaubert.modeling_flaubert import FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST

class 		lowerCAmelCase     ( a__			):

					'''simple docstring'''

					def __init__(  self				:	Any		,     __a				:	Union[str, Any]		,     __a				:	List[Any]=13		,     __a				:	Any=7		,     __a				:	Optional[int]=True		,     __a				:	Dict=True		,     __a				:	int=True		,     __a				:	Optional[Any]=True		,     __a				:	Any=True		,     __a				:	Union[str, Any]=False		,     __a				:	Union[str, Any]=False		,     __a				:	Optional[Any]=False		,     __a				:	List[str]=2		,     __a				:	Union[str, Any]=99		,     __a				:	Union[str, Any]=0		,     __a				:	List[Any]=32		,     __a				:	Any=5		,     __a				:	Tuple=4		,     __a				:	List[Any]=0.1		,     __a				:	str=0.1		,     __a				:	Optional[int]=512		,     __a				:	str=12		,     __a				:	str=2		,     __a				:	Optional[Any]=0.02		,     __a				:	Dict=3		,     __a				:	List[Any]=4		,     __a				:	List[Any]="last"		,     __a				:	Tuple=None		,     __a				:	List[str]=None		,     )			->  Union[str, Any]:


												"""simple docstring"""

												__lowercase      :							Union[str, Any]				=     parent
												__lowercase      :							int				=     batch_size
												__lowercase      :							Union[str, Any]				=     seq_length
												__lowercase      :							Tuple				=     is_training
												__lowercase      :							int				=     use_input_lengths
												__lowercase      :							List[str]				=     use_token_type_ids
												__lowercase      :							List[Any]				=     use_labels
												__lowercase      :							Union[str, Any]				=     gelu_activation
												__lowercase      :							int				=     sinusoidal_embeddings
												__lowercase      :							Union[str, Any]				=     causal
												__lowercase      :							Any				=     asm
												__lowercase      :							Union[str, Any]				=     n_langs
												__lowercase      :							int				=     vocab_size
												__lowercase      :							str				=     n_special
												__lowercase      :							Optional[int]				=     hidden_size
												__lowercase      :							Dict				=     num_hidden_layers
												__lowercase      :							Union[str, Any]				=     num_attention_heads
												__lowercase      :							Tuple				=     hidden_dropout_prob
												__lowercase      :							str				=     attention_probs_dropout_prob
												__lowercase      :							Tuple				=     max_position_embeddings
												__lowercase      :							List[str]				=     type_vocab_size
												__lowercase      :							Dict				=     type_sequence_label_size
												__lowercase      :							Dict				=     initializer_range
												__lowercase      :							Optional[int]				=     num_labels
												__lowercase      :							Tuple				=     num_choices
												__lowercase      :							List[str]				=     summary_type
												__lowercase      :							Union[str, Any]				=     use_proj
												__lowercase      :							List[Any]				=     scope



					def       lowerCAmelCase				(  self				:	Optional[Any]							)			->  Optional[Any]:


												"""simple docstring"""

												__lowercase      :							int				=     ids_tensor([self.batch_size, self.seq_length]		,     self.vocab_size							)
												__lowercase      :							str				=     random_attention_mask([self.batch_size, self.seq_length]							)

												__lowercase      :							Union[str, Any]				=     None
												if self.use_input_lengths:
																			__lowercase      :							Union[str, Any]				=     (
																			    ids_tensor([self.batch_size]		,     vocab_size=2							) + self.seq_length - 2
																			)  # small variation of seq_length

												__lowercase      :							Union[str, Any]				=     None
												if self.use_token_type_ids:
																			__lowercase      :							int				=     ids_tensor([self.batch_size, self.seq_length]		,     self.n_langs							)

												__lowercase      :							Any				=     None
												__lowercase      :							Union[str, Any]				=     None
												__lowercase      :							List[Any]				=     None
												if self.use_labels:
																			__lowercase      :							Tuple				=     ids_tensor([self.batch_size]		,     self.type_sequence_label_size							)
																			__lowercase      :							Dict				=     ids_tensor([self.batch_size, self.seq_length]		,     self.num_labels							)
																			__lowercase      :							Optional[Any]				=     ids_tensor([self.batch_size]		,     2							).float()
																			__lowercase      :							str				=     ids_tensor([self.batch_size]		,     self.num_choices							)

												__lowercase      :							Union[str, Any]				=     self.get_config()

												return (
												    config,
												    input_ids,
												    token_type_ids,
												    input_lengths,
												    sequence_labels,
												    token_labels,
												    is_impossible_labels,
												    choice_labels,
												    input_mask,
												)



					def       lowerCAmelCase				(  self				:	Optional[Any]							)			->  str:


												"""simple docstring"""

												return FlaubertConfig(
												    vocab_size=self.vocab_size		,     n_special=self.n_special		,     emb_dim=self.hidden_size		,     n_layers=self.num_hidden_layers		,     n_heads=self.num_attention_heads		,     dropout=self.hidden_dropout_prob		,     attention_dropout=self.attention_probs_dropout_prob		,     gelu_activation=self.gelu_activation		,     sinusoidal_embeddings=self.sinusoidal_embeddings		,     asm=self.asm		,     causal=self.causal		,     n_langs=self.n_langs		,     max_position_embeddings=self.max_position_embeddings		,     initializer_range=self.initializer_range		,     summary_type=self.summary_type		,     use_proj=self.use_proj		,     )



					def       lowerCAmelCase				(  self				:	Dict		,     __a				:	Tuple		,     __a				:	Optional[int]		,     __a				:	Optional[Any]		,     __a				:	str		,     __a				:	List[str]		,     __a				:	Dict		,     __a				:	List[str]		,     __a				:	int		,     __a				:	int		,     )			->  Tuple:


												"""simple docstring"""

												__lowercase      :							List[Any]				=     FlaubertModel(config=lowercase__							)
												model.to(lowercase__							)
												model.eval()
												__lowercase      :							Dict				=     model(lowercase__		,     lengths=lowercase__		,     langs=lowercase__							)
												__lowercase      :							Dict				=     model(lowercase__		,     langs=lowercase__							)
												__lowercase      :							Union[str, Any]				=     model(lowercase__							)
												self.parent.assertEqual(result.last_hidden_state.shape		,     (self.batch_size, self.seq_length, self.hidden_size)							)



					def       lowerCAmelCase				(  self				:	List[str]		,     __a				:	List[str]		,     __a				:	Any		,     __a				:	Union[str, Any]		,     __a				:	Union[str, Any]		,     __a				:	Optional[int]		,     __a				:	Any		,     __a				:	Optional[int]		,     __a				:	List[Any]		,     __a				:	List[Any]		,     )			->  Any:


												"""simple docstring"""

												__lowercase      :							List[str]				=     FlaubertWithLMHeadModel(lowercase__							)
												model.to(lowercase__							)
												model.eval()

												__lowercase      :							Optional[int]				=     model(lowercase__		,     token_type_ids=lowercase__		,     labels=lowercase__							)
												self.parent.assertEqual(result.loss.shape		,     ()							)
												self.parent.assertEqual(result.logits.shape		,     (self.batch_size, self.seq_length, self.vocab_size)							)



					def       lowerCAmelCase				(  self				:	int		,     __a				:	Any		,     __a				:	Dict		,     __a				:	Union[str, Any]		,     __a				:	Tuple		,     __a				:	Union[str, Any]		,     __a				:	int		,     __a				:	Dict		,     __a				:	Tuple		,     __a				:	Dict		,     )			->  Tuple:


												"""simple docstring"""

												__lowercase      :							List[Any]				=     FlaubertForQuestionAnsweringSimple(lowercase__							)
												model.to(lowercase__							)
												model.eval()

												__lowercase      :							List[str]				=     model(lowercase__							)

												__lowercase      :							Tuple				=     model(lowercase__		,     start_positions=lowercase__		,     end_positions=lowercase__							)
												self.parent.assertEqual(result.start_logits.shape		,     (self.batch_size, self.seq_length)							)
												self.parent.assertEqual(result.end_logits.shape		,     (self.batch_size, self.seq_length)							)



					def       lowerCAmelCase				(  self				:	Optional[int]		,     __a				:	Dict		,     __a				:	Optional[int]		,     __a				:	List[Any]		,     __a				:	str		,     __a				:	List[Any]		,     __a				:	Tuple		,     __a				:	Tuple		,     __a				:	str		,     __a				:	Optional[Any]		,     )			->  str:


												"""simple docstring"""

												__lowercase      :							Union[str, Any]				=     FlaubertForQuestionAnswering(lowercase__							)
												model.to(lowercase__							)
												model.eval()

												__lowercase      :							int				=     model(lowercase__							)

												__lowercase      :							Dict				=     model(
												    lowercase__		,     start_positions=lowercase__		,     end_positions=lowercase__		,     cls_index=lowercase__		,     is_impossible=lowercase__		,     p_mask=lowercase__		,     )

												__lowercase      :							str				=     model(
												    lowercase__		,     start_positions=lowercase__		,     end_positions=lowercase__		,     cls_index=lowercase__		,     is_impossible=lowercase__		,     )

												(__lowercase							)      :							Tuple				=     result_with_labels.to_tuple()

												__lowercase      :							List[Any]				=     model(lowercase__		,     start_positions=lowercase__		,     end_positions=lowercase__							)

												(__lowercase							)      :							Dict				=     result_with_labels.to_tuple()

												self.parent.assertEqual(result_with_labels.loss.shape		,     ()							)
												self.parent.assertEqual(result.start_top_log_probs.shape		,     (self.batch_size, model.config.start_n_top)							)
												self.parent.assertEqual(result.start_top_index.shape		,     (self.batch_size, model.config.start_n_top)							)
												self.parent.assertEqual(
												    result.end_top_log_probs.shape		,     (self.batch_size, model.config.start_n_top * model.config.end_n_top)							)
												self.parent.assertEqual(
												    result.end_top_index.shape		,     (self.batch_size, model.config.start_n_top * model.config.end_n_top)							)
												self.parent.assertEqual(result.cls_logits.shape		,     (self.batch_size,)							)



					def       lowerCAmelCase				(  self				:	List[Any]		,     __a				:	Optional[Any]		,     __a				:	Optional[int]		,     __a				:	Any		,     __a				:	Union[str, Any]		,     __a				:	List[str]		,     __a				:	Optional[int]		,     __a				:	Tuple		,     __a				:	List[Any]		,     __a				:	List[str]		,     )			->  Dict:


												"""simple docstring"""

												__lowercase      :							Any				=     FlaubertForSequenceClassification(lowercase__							)
												model.to(lowercase__							)
												model.eval()

												__lowercase      :							List[str]				=     model(lowercase__							)
												__lowercase      :							str				=     model(lowercase__		,     labels=lowercase__							)

												self.parent.assertEqual(result.loss.shape		,     ()							)
												self.parent.assertEqual(result.logits.shape		,     (self.batch_size, self.type_sequence_label_size)							)



					def       lowerCAmelCase				(  self				:	Tuple		,     __a				:	Union[str, Any]		,     __a				:	Optional[int]		,     __a				:	Any		,     __a				:	Tuple		,     __a				:	Optional[int]		,     __a				:	List[str]		,     __a				:	Dict		,     __a				:	Any		,     __a				:	Optional[Any]		,     )			->  Optional[Any]:


												"""simple docstring"""

												__lowercase      :							Tuple				=     self.num_labels
												__lowercase      :							Union[str, Any]				=     FlaubertForTokenClassification(lowercase__							)
												model.to(lowercase__							)
												model.eval()

												__lowercase      :							Optional[int]				=     model(lowercase__		,     attention_mask=lowercase__		,     labels=lowercase__							)
												self.parent.assertEqual(result.logits.shape		,     (self.batch_size, self.seq_length, self.num_labels)							)



					def       lowerCAmelCase				(  self				:	Union[str, Any]		,     __a				:	str		,     __a				:	List[str]		,     __a				:	str		,     __a				:	Optional[Any]		,     __a				:	str		,     __a				:	Union[str, Any]		,     __a				:	Tuple		,     __a				:	str		,     __a				:	Optional[int]		,     )			->  int:


												"""simple docstring"""

												__lowercase      :							str				=     self.num_choices
												__lowercase      :							List[Any]				=     FlaubertForMultipleChoice(config=lowercase__							)
												model.to(lowercase__							)
												model.eval()
												__lowercase      :							Union[str, Any]				=     input_ids.unsqueeze(1							).expand(-1		,     self.num_choices		,     -1							).contiguous()
												__lowercase      :							int				=     token_type_ids.unsqueeze(1							).expand(-1		,     self.num_choices		,     -1							).contiguous()
												__lowercase      :							Any				=     input_mask.unsqueeze(1							).expand(-1		,     self.num_choices		,     -1							).contiguous()
												__lowercase      :							Dict				=     model(
												    lowercase__		,     attention_mask=lowercase__		,     token_type_ids=lowercase__		,     labels=lowercase__		,     )
												self.parent.assertEqual(result.logits.shape		,     (self.batch_size, self.num_choices)							)



					def       lowerCAmelCase				(  self				:	Tuple							)			->  str:


												"""simple docstring"""

												__lowercase      :							Any				=     self.prepare_config_and_inputs()
												(
												    __lowercase							
												)      :							Optional[int]				=     config_and_inputs
												__lowercase      :							Dict				=     {
												    '''input_ids''': input_ids,
												    '''token_type_ids''': token_type_ids,
												    '''lengths''': input_lengths,
												    '''attention_mask''': input_mask,
												}
												return config, inputs_dict







@require_torch
class 		lowerCAmelCase     ( a__      ,       a__      ,       unittest.TestCase			):

					'''simple docstring'''

					_A						:			Optional[int]									=      (
					    (
					        FlaubertModel,
					        FlaubertWithLMHeadModel,
					        FlaubertForQuestionAnswering,
					        FlaubertForQuestionAnsweringSimple,
					        FlaubertForSequenceClassification,
					        FlaubertForTokenClassification,
					        FlaubertForMultipleChoice,
					    )
					    if is_torch_available()
					    else ()
					)
					_A						:			int									=      (
					    {
					        """feature-extraction""": FlaubertModel,
					        """fill-mask""": FlaubertWithLMHeadModel,
					        """question-answering""": FlaubertForQuestionAnsweringSimple,
					        """text-classification""": FlaubertForSequenceClassification,
					        """token-classification""": FlaubertForTokenClassification,
					        """zero-shot""": FlaubertForSequenceClassification,
					    }
					    if is_torch_available()
					    else {}
					)



					def       lowerCAmelCase				(  self				:	Dict		,     __a				:	Union[str, Any]		,     __a				:	Optional[int]		,     __a				:	List[str]		,     __a				:	Dict		,     __a				:	Optional[int]							)			->  List[str]:


												"""simple docstring"""

												if (
												    pipeline_test_casse_name == "QAPipelineTests"
												    and tokenizer_name is not None
												    and not tokenizer_name.endswith("""Fast"""							)
												):
																			# `QAPipelineTests` fails for a few models when the slower tokenizer are used.
																			# (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
																			# TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
																			return True

												return False



					def       lowerCAmelCase				(  self				:	Tuple		,     __a				:	Any		,     __a				:	List[str]		,     __a				:	str=False							)			->  List[Any]:


												"""simple docstring"""

												__lowercase      :							Optional[int]				=     super()._prepare_for_class(lowercase__		,     lowercase__		,     return_labels=lowercase__							)

												if return_labels:
																			if model_class.__name__ == "FlaubertForQuestionAnswering":
																										__lowercase      :							List[Any]				=     torch.zeros(
																										    self.model_tester.batch_size		,     dtype=torch.long		,     device=lowercase__							)
																										__lowercase      :							Dict				=     torch.zeros(
																										    self.model_tester.batch_size		,     dtype=torch.long		,     device=lowercase__							)

												return inputs_dict



					def       lowerCAmelCase				(  self				:	Union[str, Any]							)			->  str:


												"""simple docstring"""

												__lowercase      :							Dict				=     FlaubertModelTester(self							)
												__lowercase      :							Optional[int]				=     ConfigTester(self		,     config_class=lowercase__		,     emb_dim=37							)



					def       lowerCAmelCase				(  self				:	Any							)			->  Optional[int]:


												"""simple docstring"""

												self.config_tester.run_common_tests()



					def       lowerCAmelCase				(  self				:	int							)			->  Any:


												"""simple docstring"""

												__lowercase      :							Union[str, Any]				=     self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_flaubert_model(*lowercase__							)



					def       lowerCAmelCase				(  self				:	List[Any]							)			->  Union[str, Any]:


												"""simple docstring"""

												__lowercase      :							Optional[Any]				=     self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_flaubert_lm_head(*lowercase__							)



					def       lowerCAmelCase				(  self				:	Dict							)			->  Any:


												"""simple docstring"""

												__lowercase      :							List[str]				=     self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_flaubert_simple_qa(*lowercase__							)



					def       lowerCAmelCase				(  self				:	int							)			->  List[str]:


												"""simple docstring"""

												__lowercase      :							int				=     self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_flaubert_qa(*lowercase__							)



					def       lowerCAmelCase				(  self				:	List[str]							)			->  Optional[Any]:


												"""simple docstring"""

												__lowercase      :							str				=     self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_flaubert_sequence_classif(*lowercase__							)



					def       lowerCAmelCase				(  self				:	Any							)			->  str:


												"""simple docstring"""

												__lowercase      :							Any				=     self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_flaubert_token_classif(*lowercase__							)



					def       lowerCAmelCase				(  self				:	List[Any]							)			->  int:


												"""simple docstring"""

												__lowercase      :							Tuple				=     self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_flaubert_multiple_choice(*lowercase__							)



					@slow
					def       lowerCAmelCase				(  self				:	Tuple							)			->  Optional[int]:


												"""simple docstring"""

												for model_name in FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																			__lowercase      :							List[str]				=     FlaubertModel.from_pretrained(lowercase__							)
																			self.assertIsNotNone(lowercase__							)



					@slow
					@require_torch_gpu
					def       lowerCAmelCase				(  self				:	Dict							)			->  Union[str, Any]:


												"""simple docstring"""

												__lowercase      :							str				=     self.model_tester.prepare_config_and_inputs_for_common()
												for model_class in self.all_model_classes:
																			# FlauBertForMultipleChoice behaves incorrectly in JIT environments.
																			if model_class == FlaubertForMultipleChoice:
																										return

																			__lowercase      :							List[str]				=     True
																			__lowercase      :							List[Any]				=     model_class(config=lowercase__							)

																			__lowercase      :							str				=     self._prepare_for_class(lowercase__		,     lowercase__							)
																			__lowercase      :							int				=     torch.jit.trace(
																			    lowercase__		,     (inputs_dict["""input_ids"""].to("""cpu"""							), inputs_dict["""attention_mask"""].to("""cpu"""							))							)

																			with tempfile.TemporaryDirectory() as tmp:
																										torch.jit.save(lowercase__		,     os.path.join(lowercase__		,     """traced_model.pt"""							)							)
																										__lowercase      :							Dict				=     torch.jit.load(os.path.join(lowercase__		,     """traced_model.pt"""							)		,     map_location=lowercase__							)
																										loaded(inputs_dict["""input_ids"""].to(lowercase__							)		,     inputs_dict["""attention_mask"""].to(lowercase__							)							)







@require_torch
class 		lowerCAmelCase     ( unittest.TestCase			):

					'''simple docstring'''

					@slow
					def       lowerCAmelCase				(  self				:	Dict							)			->  List[str]:


												"""simple docstring"""

												__lowercase      :							Tuple				=     FlaubertModel.from_pretrained("""flaubert/flaubert_base_cased"""							)
												__lowercase      :							int				=     torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]							)
												with torch.no_grad():
																			__lowercase      :							str				=     model(lowercase__							)[0]
												__lowercase      :							int				=     torch.Size((1, 11, 768)							)
												self.assertEqual(output.shape		,     lowercase__							)
												__lowercase      :							List[Any]				=     torch.tensor(
												    [[[-2.6251, -1.4298, -0.0227], [-2.8510, -1.6387, 0.2258], [-2.8114, -1.1832, -0.3066]]]							)

												self.assertTrue(torch.allclose(output[:, :3, :3]		,     lowercase__		,     atol=1E-4							)							)







import itertools
import json
import os
import unittest

from transformers import AddedToken, LongformerTokenizer, LongformerTokenizerFast
from transformers.models.longformer.tokenization_longformer import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 		lowerCAmelCase     ( __a      ,       unittest.TestCase			):

					'''simple docstring'''

					_A						:			str									=      LongformerTokenizer
					_A						:			int									=      True
					_A						:			Optional[int]									=      LongformerTokenizerFast
					_A						:			int									=      True



					def       lowerCAmelCase				(  self				:	Any							)			->  List[str]:


												"""simple docstring"""

												super().setUp()

												# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
												__lowercase      :							Union[str, Any]				=     [
												    """l""",
												    """o""",
												    """w""",
												    """e""",
												    """r""",
												    """s""",
												    """t""",
												    """i""",
												    """d""",
												    """n""",
												    """\u0120""",
												    """\u0120l""",
												    """\u0120n""",
												    """\u0120lo""",
												    """\u0120low""",
												    """er""",
												    """\u0120lowest""",
												    """\u0120newer""",
												    """\u0120wider""",
												    """<unk>""",
												]
												__lowercase      :							Union[str, Any]				=     dict(zip(__a		,     range(len(__a							)							)							)							)
												__lowercase      :							Any				=     ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
												__lowercase      :							Optional[int]				=     {"""unk_token""": """<unk>"""}

												__lowercase      :							Dict				=     os.path.join(self.tmpdirname		,     VOCAB_FILES_NAMES["""vocab_file"""]							)
												__lowercase      :							str				=     os.path.join(self.tmpdirname		,     VOCAB_FILES_NAMES["""merges_file"""]							)
												with open(self.vocab_file		,     """w"""		,     encoding="""utf-8"""							) as fp:
																			fp.write(json.dumps(__a							) + """\n"""							)
												with open(self.merges_file		,     """w"""		,     encoding="""utf-8"""							) as fp:
																			fp.write("""\n""".join(__a							)							)



					def       lowerCAmelCase				(  self				:	Optional[int]		,     **__a				:	Optional[Any]							)			->  str:


												"""simple docstring"""

												kwargs.update(self.special_tokens_map							)
												return self.tokenizer_class.from_pretrained(self.tmpdirname		,     **__a							)



					def       lowerCAmelCase				(  self				:	Tuple		,     **__a				:	Tuple							)			->  str:


												"""simple docstring"""

												kwargs.update(self.special_tokens_map							)
												return self.rust_tokenizer_class.from_pretrained(self.tmpdirname		,     **__a							)



					def       lowerCAmelCase				(  self				:	str		,     __a				:	Optional[int]							)			->  Union[str, Any]:


												"""simple docstring"""

												__lowercase      :							Union[str, Any]				=     """lower newer"""
												__lowercase      :							int				=     """lower newer"""
												return input_text, output_text



					def       lowerCAmelCase				(  self				:	List[Any]							)			->  Optional[int]:


												"""simple docstring"""

												__lowercase      :							Union[str, Any]				=     self.tokenizer_class(self.vocab_file		,     self.merges_file		,     **self.special_tokens_map							)
												__lowercase      :							Dict				=     """lower newer"""
												__lowercase      :							Optional[Any]				=     ["""l""", """o""", """w""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
												__lowercase      :							str				=     tokenizer.tokenize(__a							)  # , add_prefix_space=True)
												self.assertListEqual(__a		,     __a							)

												__lowercase      :							int				=     tokens + [tokenizer.unk_token]
												__lowercase      :							str				=     [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
												self.assertListEqual(tokenizer.convert_tokens_to_ids(__a							)		,     __a							)



					def       lowerCAmelCase				(  self				:	Optional[Any]							)			->  List[Any]:


												"""simple docstring"""

												__lowercase      :							Optional[Any]				=     self.get_tokenizer()

												self.assertListEqual(tokenizer.encode("""Hello world!"""		,     add_special_tokens=__a							)		,     [0, 31414, 232, 328, 2]							)
												self.assertListEqual(
												    tokenizer.encode("""Hello world! cécé herlolip 418"""		,     add_special_tokens=__a							)		,     [0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]		,     )



					@slow
					def       lowerCAmelCase				(  self				:	Tuple							)			->  str:


												"""simple docstring"""

												__lowercase      :							Any				=     self.tokenizer_class.from_pretrained("""allenai/longformer-base-4096"""							)

												__lowercase      :							Optional[Any]				=     tokenizer.encode("""sequence builders"""		,     add_special_tokens=__a							)
												__lowercase      :							List[str]				=     tokenizer.encode("""multi-sequence build"""		,     add_special_tokens=__a							)

												__lowercase      :							Optional[Any]				=     tokenizer.encode(
												    """sequence builders"""		,     add_special_tokens=__a		,     add_prefix_space=__a							)
												__lowercase      :							Union[str, Any]				=     tokenizer.encode(
												    """sequence builders"""		,     """multi-sequence build"""		,     add_special_tokens=__a		,     add_prefix_space=__a							)

												__lowercase      :							List[Any]				=     tokenizer.build_inputs_with_special_tokens(__a							)
												__lowercase      :							Any				=     tokenizer.build_inputs_with_special_tokens(__a		,     __a							)

												assert encoded_sentence == encoded_text_from_decode
												assert encoded_pair == encoded_pair_from_decode



					def       lowerCAmelCase				(  self				:	Dict							)			->  Tuple:


												"""simple docstring"""

												__lowercase      :							Optional[Any]				=     self.get_tokenizer()

												__lowercase      :							Tuple				=     """Encode this sequence."""
												__lowercase      :							Optional[Any]				=     tokenizer.byte_encoder[""" """.encode("""utf-8"""							)[0]]

												# Testing encoder arguments
												__lowercase      :							Dict				=     tokenizer.encode(__a		,     add_special_tokens=__a		,     add_prefix_space=__a							)
												__lowercase      :							Tuple				=     tokenizer.convert_ids_to_tokens(encoded[0]							)[0]
												self.assertNotEqual(__a		,     __a							)

												__lowercase      :							List[str]				=     tokenizer.encode(__a		,     add_special_tokens=__a		,     add_prefix_space=__a							)
												__lowercase      :							Any				=     tokenizer.convert_ids_to_tokens(encoded[0]							)[0]
												self.assertEqual(__a		,     __a							)

												tokenizer.add_special_tokens({"""bos_token""": """<s>"""}							)
												__lowercase      :							str				=     tokenizer.encode(__a		,     add_special_tokens=__a							)
												__lowercase      :							Dict				=     tokenizer.convert_ids_to_tokens(encoded[1]							)[0]
												self.assertNotEqual(__a		,     __a							)

												# Testing spaces after special tokens
												__lowercase      :							List[Any]				=     """<mask>"""
												tokenizer.add_special_tokens(
												    {"""mask_token""": AddedToken(__a		,     lstrip=__a		,     rstrip=__a							)}							)  # mask token has a left space
												__lowercase      :							Dict				=     tokenizer.convert_tokens_to_ids(__a							)

												__lowercase      :							List[str]				=     """Encode <mask> sequence"""
												__lowercase      :							List[str]				=     """Encode <mask>sequence"""

												__lowercase      :							Union[str, Any]				=     tokenizer.encode(__a							)
												__lowercase      :							Dict				=     encoded.index(__a							)
												__lowercase      :							List[Any]				=     tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]							)[0]
												self.assertEqual(__a		,     __a							)

												__lowercase      :							int				=     tokenizer.encode(__a							)
												__lowercase      :							Union[str, Any]				=     encoded.index(__a							)
												__lowercase      :							List[Any]				=     tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]							)[0]
												self.assertNotEqual(__a		,     __a							)



					def       lowerCAmelCase				(  self				:	int							)			->  str:


												"""simple docstring"""

												pass



					def       lowerCAmelCase				(  self				:	int							)			->  Dict:


												"""simple docstring"""

												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																			with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"							):
																										__lowercase      :							List[str]				=     self.rust_tokenizer_class.from_pretrained(__a		,     **__a							)
																										__lowercase      :							List[Any]				=     self.tokenizer_class.from_pretrained(__a		,     **__a							)
																										__lowercase      :							Optional[Any]				=     """A, <mask> AllenNLP sentence."""
																										__lowercase      :							Union[str, Any]				=     tokenizer_r.encode_plus(__a		,     add_special_tokens=__a		,     return_token_type_ids=__a							)
																										__lowercase      :							Optional[Any]				=     tokenizer_p.encode_plus(__a		,     add_special_tokens=__a		,     return_token_type_ids=__a							)

																										# token_type_ids should put 0 everywhere
																										self.assertEqual(sum(tokens_r["""token_type_ids"""]							)		,     sum(tokens_p["""token_type_ids"""]							)							)

																										# attention_mask should put 1 everywhere, so sum over length should be 1
																										self.assertEqual(
																										    sum(tokens_r["""attention_mask"""]							) / len(tokens_r["""attention_mask"""]							)		,     sum(tokens_p["""attention_mask"""]							) / len(tokens_p["""attention_mask"""]							)		,     )

																										__lowercase      :							Dict				=     tokenizer_r.convert_ids_to_tokens(tokens_r["""input_ids"""]							)
																										__lowercase      :							str				=     tokenizer_p.convert_ids_to_tokens(tokens_p["""input_ids"""]							)

																										# Rust correctly handles the space before the mask while python doesnt
																										self.assertSequenceEqual(tokens_p["""input_ids"""]		,     [0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]							)
																										self.assertSequenceEqual(tokens_r["""input_ids"""]		,     [0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]							)

																										self.assertSequenceEqual(
																										    __a		,     ["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]							)
																										self.assertSequenceEqual(
																										    __a		,     ["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]							)



					def       lowerCAmelCase				(  self				:	List[Any]							)			->  Dict:


												"""simple docstring"""

												for trim_offsets, add_prefix_space in itertools.product([True, False]		,     repeat=2							):
																			__lowercase      :							Dict				=     self.rust_tokenizer_class.from_pretrained(
																			    self.tmpdirname		,     use_fast=__a		,     add_prefix_space=__a		,     trim_offsets=__a							)

																			__lowercase      :							Optional[int]				=     json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()							)
																			__lowercase      :							Any				=     json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()							)

																			self.assertEqual(pre_tokenizer_state["""add_prefix_space"""]		,     __a							)

																			self.assertEqual(post_processor_state["""add_prefix_space"""]		,     __a							)
																			self.assertEqual(post_processor_state["""trim_offsets"""]		,     __a							)



					def       lowerCAmelCase				(  self				:	int							)			->  Tuple:


												"""simple docstring"""

												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																			with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"							):
																										__lowercase      :							List[str]				=     """hello"""  # `hello` is a token in the vocabulary of `pretrained_name`
																										__lowercase      :							int				=     F"{text_of_1_token} {text_of_1_token}"

																										__lowercase      :							List[str]				=     self.rust_tokenizer_class.from_pretrained(
																										    __a		,     use_fast=__a		,     add_prefix_space=__a		,     trim_offsets=__a							)
																										__lowercase      :							Any				=     tokenizer_r(__a		,     return_offsets_mapping=__a		,     add_special_tokens=__a							)
																										self.assertEqual(encoding.offset_mapping[0]		,     (0, len(__a							))							)
																										self.assertEqual(
																										    encoding.offset_mapping[1]		,     (len(__a							) + 1, len(__a							) + 1 + len(__a							))		,     )

																										__lowercase      :							str				=     self.rust_tokenizer_class.from_pretrained(
																										    __a		,     use_fast=__a		,     add_prefix_space=__a		,     trim_offsets=__a							)
																										__lowercase      :							Tuple				=     tokenizer_r(__a		,     return_offsets_mapping=__a		,     add_special_tokens=__a							)
																										self.assertEqual(encoding.offset_mapping[0]		,     (0, len(__a							))							)
																										self.assertEqual(
																										    encoding.offset_mapping[1]		,     (len(__a							) + 1, len(__a							) + 1 + len(__a							))		,     )

																										__lowercase      :							Optional[int]				=     self.rust_tokenizer_class.from_pretrained(
																										    __a		,     use_fast=__a		,     add_prefix_space=__a		,     trim_offsets=__a							)
																										__lowercase      :							str				=     tokenizer_r(__a		,     return_offsets_mapping=__a		,     add_special_tokens=__a							)
																										self.assertEqual(encoding.offset_mapping[0]		,     (0, len(__a							))							)
																										self.assertEqual(
																										    encoding.offset_mapping[1]		,     (len(__a							), len(__a							) + 1 + len(__a							))		,     )

																										__lowercase      :							str				=     self.rust_tokenizer_class.from_pretrained(
																										    __a		,     use_fast=__a		,     add_prefix_space=__a		,     trim_offsets=__a							)
																										__lowercase      :							int				=     tokenizer_r(__a		,     return_offsets_mapping=__a		,     add_special_tokens=__a							)
																										self.assertEqual(encoding.offset_mapping[0]		,     (0, len(__a							))							)
																										self.assertEqual(
																										    encoding.offset_mapping[1]		,     (len(__a							), len(__a							) + 1 + len(__a							))		,     )

																										__lowercase      :							Any				=     F" {text}"

																										# tokenizer_r = self.rust_tokenizer_class.from_pretrained(
																										#     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
																										# )
																										# encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
																										# self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
																										# self.assertEqual(
																										#     encoding.offset_mapping[1],
																										#     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
																										# )

																										__lowercase      :							Optional[Any]				=     self.rust_tokenizer_class.from_pretrained(
																										    __a		,     use_fast=__a		,     add_prefix_space=__a		,     trim_offsets=__a							)
																										__lowercase      :							str				=     tokenizer_r(__a		,     return_offsets_mapping=__a		,     add_special_tokens=__a							)
																										self.assertEqual(encoding.offset_mapping[0]		,     (1, 1 + len(__a							))							)
																										self.assertEqual(
																										    encoding.offset_mapping[1]		,     (1 + len(__a							) + 1, 1 + len(__a							) + 1 + len(__a							))		,     )

																										__lowercase      :							int				=     self.rust_tokenizer_class.from_pretrained(
																										    __a		,     use_fast=__a		,     add_prefix_space=__a		,     trim_offsets=__a							)
																										__lowercase      :							Dict				=     tokenizer_r(__a		,     return_offsets_mapping=__a		,     add_special_tokens=__a							)
																										self.assertEqual(encoding.offset_mapping[0]		,     (0, 1 + len(__a							))							)
																										self.assertEqual(
																										    encoding.offset_mapping[1]		,     (1 + len(__a							), 1 + len(__a							) + 1 + len(__a							))		,     )

																										__lowercase      :							int				=     self.rust_tokenizer_class.from_pretrained(
																										    __a		,     use_fast=__a		,     add_prefix_space=__a		,     trim_offsets=__a							)
																										__lowercase      :							Tuple				=     tokenizer_r(__a		,     return_offsets_mapping=__a		,     add_special_tokens=__a							)
																										self.assertEqual(encoding.offset_mapping[0]		,     (0, 1 + len(__a							))							)
																										self.assertEqual(
																										    encoding.offset_mapping[1]		,     (1 + len(__a							), 1 + len(__a							) + 1 + len(__a							))		,     )