


'''simple docstring'''




import unittest

from transformers import XLMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
				import torch

				from transformers import (
				    XLMForMultipleChoice,
				    XLMForQuestionAnswering,
				    XLMForQuestionAnsweringSimple,
				    XLMForSequenceClassification,
				    XLMForTokenClassification,
				    XLMModel,
				    XLMWithLMHeadModel,
				)
				from transformers.models.xlm.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_LIST



class  lowerCAmelCase				:



	def __init__(					self ,					snake_case__ ,					snake_case__=13 ,					snake_case__=7 ,					snake_case__=True ,					snake_case__=True ,					snake_case__=True ,					snake_case__=True ,					snake_case__=True ,					snake_case__=False ,					snake_case__=False ,					snake_case__=False ,					snake_case__=2 ,					snake_case__=99 ,					snake_case__=0 ,					snake_case__=32 ,					snake_case__=5 ,					snake_case__=4 ,					snake_case__=0.1 ,					snake_case__=0.1 ,					snake_case__=512 ,					snake_case__=2 ,					snake_case__=0.0_2 ,					snake_case__=2 ,					snake_case__=4 ,					snake_case__="last" ,					snake_case__=True ,					snake_case__=None ,					snake_case__=0 ,					):
		lowerCAmelCase					:    List[Any]      							= parent
		lowerCAmelCase					:    Optional[Any]      							= batch_size
		lowerCAmelCase					:    int      							= seq_length
		lowerCAmelCase					:    Optional[int]      							= is_training
		lowerCAmelCase					:    Optional[int]      							= use_input_lengths
		lowerCAmelCase					:    Any      							= use_token_type_ids
		lowerCAmelCase					:    List[str]      							= use_labels
		lowerCAmelCase					:    List[Any]      							= gelu_activation
		lowerCAmelCase					:    Tuple      							= sinusoidal_embeddings
		lowerCAmelCase					:    Union[str, Any]      							= causal
		lowerCAmelCase					:    int      							= asm
		lowerCAmelCase					:    Union[str, Any]      							= n_langs
		lowerCAmelCase					:    Union[str, Any]      							= vocab_size
		lowerCAmelCase					:    int      							= n_special
		lowerCAmelCase					:    Tuple      							= hidden_size
		lowerCAmelCase					:    List[Any]      							= num_hidden_layers
		lowerCAmelCase					:    Union[str, Any]      							= num_attention_heads
		lowerCAmelCase					:    Optional[int]      							= hidden_dropout_prob
		lowerCAmelCase					:    Dict      							= attention_probs_dropout_prob
		lowerCAmelCase					:    Tuple      							= max_position_embeddings
		lowerCAmelCase					:    Dict      							= type_sequence_label_size
		lowerCAmelCase					:    List[Any]      							= initializer_range
		lowerCAmelCase					:    int      							= num_labels
		lowerCAmelCase					:    Union[str, Any]      							= num_choices
		lowerCAmelCase					:    str      							= summary_type
		lowerCAmelCase					:    Optional[Any]      							= use_proj
		lowerCAmelCase					:    Optional[Any]      							= scope
		lowerCAmelCase					:    Optional[int]      							= bos_token_id



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    Dict      							= ids_tensor([self.batch_size, self.seq_length] ,					self.vocab_size    )
		lowerCAmelCase					:    Union[str, Any]      							= random_attention_mask([self.batch_size, self.seq_length]    )

		lowerCAmelCase					:    Any      							= None
		if self.use_input_lengths:
			lowerCAmelCase					:    Tuple      							= (
			    ids_tensor([self.batch_size] ,					vocab_size=2    ) + self.seq_length - 2
			)  # small variation of seq_length

		lowerCAmelCase					:    Any      							= None
		if self.use_token_type_ids:
			lowerCAmelCase					:    int      							= ids_tensor([self.batch_size, self.seq_length] ,					self.n_langs    )

		lowerCAmelCase					:    List[Any]      							= None
		lowerCAmelCase					:    List[str]      							= None
		lowerCAmelCase					:    Optional[int]      							= None
		if self.use_labels:
			lowerCAmelCase					:    Any      							= ids_tensor([self.batch_size] ,					self.type_sequence_label_size    )
			lowerCAmelCase					:    Union[str, Any]      							= ids_tensor([self.batch_size, self.seq_length] ,					self.num_labels    )
			lowerCAmelCase					:    List[Any]      							= ids_tensor([self.batch_size] ,					2    ).float()
			lowerCAmelCase					:    Dict      							= ids_tensor([self.batch_size] ,					self.num_choices    )

		lowerCAmelCase					:    Union[str, Any]      							= self.get_config()

		return (
		    config,
		    input_ids,
		    token_type_ids,
		    input_lengths,
		    sequence_labels,
		    token_labels,
		    is_impossible_labels,
		    choice_labels,
		    input_mask,
		)



	def 				lowercase     (					self    ):
		return XLMConfig(
		    vocab_size=self.vocab_size ,					n_special=self.n_special ,					emb_dim=self.hidden_size ,					n_layers=self.num_hidden_layers ,					n_heads=self.num_attention_heads ,					dropout=self.hidden_dropout_prob ,					attention_dropout=self.attention_probs_dropout_prob ,					gelu_activation=self.gelu_activation ,					sinusoidal_embeddings=self.sinusoidal_embeddings ,					asm=self.asm ,					causal=self.causal ,					n_langs=self.n_langs ,					max_position_embeddings=self.max_position_embeddings ,					initializer_range=self.initializer_range ,					summary_type=self.summary_type ,					use_proj=self.use_proj ,					num_labels=self.num_labels ,					bos_token_id=self.bos_token_id ,					)



	def 				lowercase     (					self ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					):
		lowerCAmelCase					:    Union[str, Any]      							= XLMModel(config=snake_case__    )
		model.to(snake_case__    )
		model.eval()
		lowerCAmelCase					:    Union[str, Any]      							= model(snake_case__ ,					lengths=snake_case__ ,					langs=snake_case__    )
		lowerCAmelCase					:    List[Any]      							= model(snake_case__ ,					langs=snake_case__    )
		lowerCAmelCase					:    List[str]      							= model(snake_case__    )
		self.parent.assertEqual(result.last_hidden_state.shape ,					(self.batch_size, self.seq_length, self.hidden_size)    )



	def 				lowercase     (					self ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					):
		lowerCAmelCase					:    List[str]      							= XLMWithLMHeadModel(snake_case__    )
		model.to(snake_case__    )
		model.eval()

		lowerCAmelCase					:    Union[str, Any]      							= model(snake_case__ ,					token_type_ids=snake_case__ ,					labels=snake_case__    )
		self.parent.assertEqual(result.loss.shape ,					()    )
		self.parent.assertEqual(result.logits.shape ,					(self.batch_size, self.seq_length, self.vocab_size)    )



	def 				lowercase     (					self ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					):
		lowerCAmelCase					:    Optional[int]      							= XLMForQuestionAnsweringSimple(snake_case__    )
		model.to(snake_case__    )
		model.eval()

		lowerCAmelCase					:    List[Any]      							= model(snake_case__    )

		lowerCAmelCase					:    Union[str, Any]      							= model(snake_case__ ,					start_positions=snake_case__ ,					end_positions=snake_case__    )
		lowerCAmelCase					:    Dict      							= outputs
		self.parent.assertEqual(result.start_logits.shape ,					(self.batch_size, self.seq_length)    )
		self.parent.assertEqual(result.end_logits.shape ,					(self.batch_size, self.seq_length)    )



	def 				lowercase     (					self ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					):
		lowerCAmelCase					:    List[Any]      							= XLMForQuestionAnswering(snake_case__    )
		model.to(snake_case__    )
		model.eval()

		lowerCAmelCase					:    Tuple      							= model(snake_case__    )

		lowerCAmelCase					:    int      							= model(
		    snake_case__ ,					start_positions=snake_case__ ,					end_positions=snake_case__ ,					cls_index=snake_case__ ,					is_impossible=snake_case__ ,					p_mask=snake_case__ ,					)

		lowerCAmelCase					:    List[str]      							= model(
		    snake_case__ ,					start_positions=snake_case__ ,					end_positions=snake_case__ ,					cls_index=snake_case__ ,					is_impossible=snake_case__ ,					)

		((lowerCAmelCase)							,   )					:    Any      							= result_with_labels.to_tuple()

		lowerCAmelCase					:    Any      							= model(snake_case__ ,					start_positions=snake_case__ ,					end_positions=snake_case__    )

		((lowerCAmelCase)							,   )					:    Optional[int]      							= result_with_labels.to_tuple()

		self.parent.assertEqual(result_with_labels.loss.shape ,					()    )
		self.parent.assertEqual(result.start_top_log_probs.shape ,					(self.batch_size, model.config.start_n_top)    )
		self.parent.assertEqual(result.start_top_index.shape ,					(self.batch_size, model.config.start_n_top)    )
		self.parent.assertEqual(
		    result.end_top_log_probs.shape ,					(self.batch_size, model.config.start_n_top * model.config.end_n_top)    )
		self.parent.assertEqual(
		    result.end_top_index.shape ,					(self.batch_size, model.config.start_n_top * model.config.end_n_top)    )
		self.parent.assertEqual(result.cls_logits.shape ,					(self.batch_size,)    )



	def 				lowercase     (					self ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					):
		lowerCAmelCase					:    Optional[Any]      							= XLMForSequenceClassification(snake_case__    )
		model.to(snake_case__    )
		model.eval()

		lowerCAmelCase					:    List[Any]      							= model(snake_case__    )
		lowerCAmelCase					:    List[Any]      							= model(snake_case__ ,					labels=snake_case__    )
		self.parent.assertEqual(result.loss.shape ,					()    )
		self.parent.assertEqual(result.logits.shape ,					(self.batch_size, self.type_sequence_label_size)    )



	def 				lowercase     (					self ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					):
		lowerCAmelCase					:    str      							= self.num_labels
		lowerCAmelCase					:    Union[str, Any]      							= XLMForTokenClassification(snake_case__    )
		model.to(snake_case__    )
		model.eval()

		lowerCAmelCase					:    Union[str, Any]      							= model(snake_case__ ,					attention_mask=snake_case__ ,					labels=snake_case__    )
		self.parent.assertEqual(result.logits.shape ,					(self.batch_size, self.seq_length, self.num_labels)    )



	def 				lowercase     (					self ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					):
		lowerCAmelCase					:    str      							= self.num_choices
		lowerCAmelCase					:    Dict      							= XLMForMultipleChoice(config=snake_case__    )
		model.to(snake_case__    )
		model.eval()
		lowerCAmelCase					:    List[Any]      							= input_ids.unsqueeze(1    ).expand(-1 ,					self.num_choices ,					-1    ).contiguous()
		lowerCAmelCase					:    List[str]      							= token_type_ids.unsqueeze(1    ).expand(-1 ,					self.num_choices ,					-1    ).contiguous()
		lowerCAmelCase					:    Optional[Any]      							= input_mask.unsqueeze(1    ).expand(-1 ,					self.num_choices ,					-1    ).contiguous()
		lowerCAmelCase					:    List[Any]      							= model(
		    snake_case__ ,					attention_mask=snake_case__ ,					token_type_ids=snake_case__ ,					labels=snake_case__ ,					)
		self.parent.assertEqual(result.logits.shape ,					(self.batch_size, self.num_choices)    )



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    Dict      							= self.prepare_config_and_inputs()
		(
		    (
		    lowerCAmelCase
		)							,   (
		    lowerCAmelCase
		)							,   (
		    lowerCAmelCase
		)							,   (
		    lowerCAmelCase
		)							,   (
		    lowerCAmelCase
		)							,   (
		    lowerCAmelCase
		)							,   (
		    lowerCAmelCase
		)							,   (
		    lowerCAmelCase
		)							,   (
		    lowerCAmelCase
		)							,   
		)					:    Any      							= config_and_inputs
		lowerCAmelCase					:    int      							= {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'lengths': input_lengths}
		return config, inputs_dict



@require_torch
class  lowerCAmelCase				(				a      ,				a      ,				a      ,				unittest.TestCase							):
	_lowerCamelCase   :					int			      = (
	    (
	        XLMModel,
	        XLMWithLMHeadModel,
	        XLMForQuestionAnswering,
	        XLMForSequenceClassification,
	        XLMForQuestionAnsweringSimple,
	        XLMForTokenClassification,
	        XLMForMultipleChoice,
	    )
	    if is_torch_available()
	    else ()
	)
	_lowerCamelCase   :					List[str]			      = (
	    (XLMWithLMHeadModel,) if is_torch_available() else ()
	)  # TODO (PVP): Check other models whether language generation is also applicable
	_lowerCamelCase   :					Optional[Any]			      = (
	    {
	        """feature-extraction""": XLMModel,
	        """fill-mask""": XLMWithLMHeadModel,
	        """question-answering""": XLMForQuestionAnsweringSimple,
	        """text-classification""": XLMForSequenceClassification,
	        """text-generation""": XLMWithLMHeadModel,
	        """token-classification""": XLMForTokenClassification,
	        """zero-shot""": XLMForSequenceClassification,
	    }
	    if is_torch_available()
	    else {}
	)



	def 				lowercase     (					self ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__    ):
		if (
		    pipeline_test_casse_name == "QAPipelineTests"
		    and tokenizer_name is not None
		    and not tokenizer_name.endswith('Fast'    )
		):
			# `QAPipelineTests` fails for a few models when the slower tokenizer are used.
			# (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
			# TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
			return True

		return False



	def 				lowercase     (					self ,					snake_case__ ,					snake_case__ ,					snake_case__=False    ):
		lowerCAmelCase					:    str      							= super()._prepare_for_class(snake_case__ ,					snake_case__ ,					return_labels=snake_case__    )

		if return_labels:
			if model_class.__name__ == "XLMForQuestionAnswering":
				lowerCAmelCase					:    Union[str, Any]      							= torch.zeros(
				    self.model_tester.batch_size ,					dtype=torch.long ,					device=snake_case__    )
				lowerCAmelCase					:    Tuple      							= torch.zeros(
				    self.model_tester.batch_size ,					dtype=torch.long ,					device=snake_case__    )

		return inputs_dict



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    Optional[int]      							= XLMModelTester(self    )
		lowerCAmelCase					:    int      							= ConfigTester(self ,					config_class=snake_case__ ,					emb_dim=37    )



	def 				lowercase     (					self    ):
		self.config_tester.run_common_tests()



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    Tuple      							= self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_xlm_model(*snake_case__    )



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    Optional[Any]      							= self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_xlm_lm_head(*snake_case__    )



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    str      							= self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_xlm_simple_qa(*snake_case__    )



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    Optional[int]      							= self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_xlm_qa(*snake_case__    )



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    Dict      							= self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_xlm_sequence_classif(*snake_case__    )



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    Union[str, Any]      							= self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_xlm_token_classif(*snake_case__    )



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    List[str]      							= self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_xlm_for_multiple_choice(*snake_case__    )



	def 				lowercase     (					self ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__=False ,					snake_case__=1    ):
		self.assertIsInstance(snake_case__ ,					snake_case__    )
		self.assertListEqual(
		    [isinstance(snake_case__ ,					snake_case__    ) for iter_attentions in attentions] ,					[True] * len(snake_case__    )    )
		self.assertEqual(len(snake_case__    ) ,					(max_length - min_length) * num_beam_groups    )

		for idx, iter_attentions in enumerate(snake_case__    ):
			# adds PAD dummy token
			lowerCAmelCase					:    Union[str, Any]      							= min_length + idx + 1
			lowerCAmelCase					:    str      							= min_length + idx + 1

			lowerCAmelCase					:    Optional[int]      							= (
			    batch_size * num_beam_groups,
			    config.num_attention_heads,
			    tgt_len,
			    src_len,
			)
			# check attn size
			self.assertListEqual(
			    [layer_attention.shape for layer_attention in iter_attentions] ,					[expected_shape] * len(snake_case__    )    )



	def 				lowercase     (					self ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__=False ,					snake_case__=1    ):
		self.assertIsInstance(snake_case__ ,					snake_case__    )
		self.assertListEqual(
		    [isinstance(snake_case__ ,					snake_case__    ) for iter_hidden_states in hidden_states] ,					[True] * len(snake_case__    ) ,					)
		self.assertEqual(len(snake_case__    ) ,					(max_length - min_length) * num_beam_groups    )

		for idx, iter_hidden_states in enumerate(snake_case__    ):
			# adds PAD dummy token
			lowerCAmelCase					:    Dict      							= min_length + idx + 1
			lowerCAmelCase					:    List[str]      							= (batch_size * num_beam_groups, seq_len, config.hidden_size)
			# check hidden size
			self.assertListEqual(
			    [layer_hidden_states.shape for layer_hidden_states in iter_hidden_states] ,					[expected_shape] * len(snake_case__    ) ,					)
		pass



	@slow
	def 				lowercase     (					self    ):
		for model_name in XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
			lowerCAmelCase					:    List[str]      							= XLMModel.from_pretrained(snake_case__    )
			self.assertIsNotNone(snake_case__    )



@require_torch
class  lowerCAmelCase				(				unittest.TestCase							):



	@slow
	def 				lowercase     (					self    ):
		lowerCAmelCase					:    Optional[Any]      							= XLMWithLMHeadModel.from_pretrained('xlm-mlm-en-2048'    )
		model.to(snake_case__    )
		lowerCAmelCase					:    int      							= torch.tensor([[14, 447]] ,					dtype=torch.long ,					device=snake_case__    )  # the president
		lowerCAmelCase					:    Any      							= [
		    14,
		    447,
		    14,
		    447,
		    14,
		    447,
		    14,
		    447,
		    14,
		    447,
		    14,
		    447,
		    14,
		    447,
		    14,
		    447,
		    14,
		    447,
		    14,
		    447,
		]  # the president the president the president the president the president the president the president the president the president the president
		# TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
		lowerCAmelCase					:    Any      							= model.generate(snake_case__ ,					do_sample=snake_case__    )
		self.assertListEqual(output_ids[0].cpu().numpy().tolist() ,					snake_case__    )




'''simple docstring'''




import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_lowerCAmelCase  :		Union[str, Any]		     =	get_tests_dir('fixtures/test_sentencepiece_with_bytefallback.model')



@require_sentencepiece
@require_tokenizers
class  lowerCAmelCase				(				a      ,				unittest.TestCase							):
	_lowerCamelCase   :					Tuple			      = GPTSwaTokenizer
	_lowerCamelCase   :					str			      = False
	_lowerCamelCase   :					Dict			      = True
	_lowerCamelCase   :					Optional[Any]			      = False



	def 				lowercase     (					self    ):
		super().setUp()

		# We have a SentencePiece fixture for testing
		lowerCAmelCase					:    Tuple      							= GPTSwaTokenizer(snake_case__ ,					eos_token='<unk>' ,					bos_token='<unk>' ,					pad_token='<unk>'    )

		tokenizer.save_pretrained(self.tmpdirname    )



	def 				lowercase     (					self ,					snake_case__    ):
		lowerCAmelCase					:    List[Any]      							= 'This is a test'
		lowerCAmelCase					:    List[Any]      							= 'This is a test'
		return input_text, output_text



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    Tuple      							= '<s>'
		lowerCAmelCase					:    Optional[int]      							= 1

		self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case__    ) ,					snake_case__    )
		self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case__    ) ,					snake_case__    )



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    List[Any]      							= list(self.get_tokenizer().get_vocab().keys()    )

		self.assertEqual(vocab_keys[0] ,					'<unk>'    )
		self.assertEqual(vocab_keys[1] ,					'<s>'    )
		self.assertEqual(vocab_keys[-1] ,					'j'    )
		self.assertEqual(len(snake_case__    ) ,					2000    )



	def 				lowercase     (					self    ):
		self.assertEqual(self.get_tokenizer().vocab_size ,					2000    )



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    List[Any]      							= GPTSwaTokenizer(snake_case__    )

		lowerCAmelCase					:    Optional[Any]      							= tokenizer.tokenize('This is a test'    )
		self.assertListEqual(snake_case__ ,					['▁This', '▁is', '▁a', '▁t', 'est']    )

		self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__    ) ,					[465, 287, 265, 631, 842]    )

		lowerCAmelCase					:    Tuple      							= tokenizer.tokenize('I was born in 92000, and this is falsé.'    )
		# fmt: off
		self.assertListEqual(
		    snake_case__ ,					['▁I', '▁was', '▁bor', 'n', '▁in', '▁', '<0x39>', '2', '0', '0', '0', ',', '▁and', '▁this', '▁is', '▁f', 'al', 's', '<0xC3>', '<0xA9>', '.'] ,					)
		# fmt: on

		lowerCAmelCase					:    Optional[Any]      							= tokenizer.convert_tokens_to_ids(snake_case__    )
		self.assertListEqual(
		    snake_case__ ,					[262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260] ,					)

		lowerCAmelCase					:    int      							= tokenizer.convert_ids_to_tokens(snake_case__    )
		# fmt: off
		self.assertListEqual(
		    snake_case__ ,					['▁I', '▁was', '▁bor', 'n', '▁in', '▁', '<0x39>', '2', '0', '0', '0', ',', '▁and', '▁this', '▁is', '▁f', 'al', 's', '<0xC3>', '<0xA9>', '.']    )
		# fmt: on



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    str      							= GPTSwaTokenizer(snake_case__    )
		lowerCAmelCase					:    Optional[int]      							= ['This is a test', 'I was born in 92000, and this is falsé.']
		lowerCAmelCase					:    Tuple      							= [
		    [465, 287, 265, 631, 842],
		    [262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260],
		]

		# Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
		for text, expected_ids in zip(snake_case__ ,					snake_case__    ):
			self.assertListEqual(tokenizer.encode_fast(snake_case__    ) ,					snake_case__    )

		# Test that decode_fast returns the input text
		for text, token_ids in zip(snake_case__ ,					snake_case__    ):
			self.assertEqual(tokenizer.decode_fast(snake_case__    ) ,					snake_case__    )



	@slow
	def 				lowercase     (					self    ):
		lowerCAmelCase					:    str      							= [
		    '<|python|>def fibonacci(n)\n    if n < 0:\n        print(\'Incorrect input\')',
		    'Hey there, how are you doing this fine day?',
		    'This is a text with a trailing spaces followed by a dot     .',
		    'Häj sväjs lillebrör! =)',
		    'Det är inget fel på Mr. Cool',
		]

		# fmt: off
		lowerCAmelCase					:    Tuple      							= {'input_ids': [[6_3423, 5, 6811, 1_4954, 282, 816, 3821, 6_3466, 6_3425, 6_3462, 18, 6_3978, 678, 301, 1320, 6_3423, 6_3455, 6_3458, 18, 6_3982, 4246, 3940, 1901, 4_7789, 5547, 1_8994], [1_9630, 1100, 6_3446, 1342, 633, 544, 4488, 593, 5102, 2416, 6_3495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1652, 428, 268, 1936, 515, 268, 5_8593, 2_2413, 9106, 546, 268, 3_3213, 6_3979, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_5130, 6_3450, 924, 6_3449, 2249, 4062, 1558, 318, 6_3504, 2_1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [509, 377, 2827, 2559, 332, 6575, 6_3443, 2_6801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
		# fmt: on
		self.tokenizer_integration_test_util(
		    expected_encoding=snake_case__ ,					model_name='AI-Sweden/gpt-sw3-126m' ,					sequences=snake_case__ ,					)
