





import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase_						   =    get_tests_dir("""fixtures/test_sentencepiece_with_bytefallback.model""")



@require_sentencepiece
@require_tokenizers
class        _lowerCAmelCase    (			UpperCAmelCase_							,    unittest.TestCase							):


  '''simple docstring'''
  a_							:       Any      					=GPTSwaTokenizer
  a_							:       Dict      					=False
  a_							:       Any      					=True
  a_							:       Optional[int]      					=False







  def UpperCamelCase_ (      self      :					List[str]							):


    '''simple docstring'''





    super().setUp()

    # We have a SentencePiece fixture for testing
    _snake_case   :				Union[str, Any]														=	GPTSwaTokenizer(UpperCamelCase      ,				eos_token='<unk>'      ,				bos_token='<unk>'      ,				pad_token='<unk>'							)

    tokenizer.save_pretrained(self.tmpdirname							)







  def UpperCamelCase_ (      self      :					Optional[Any]      ,				UpperCamelCase      :					Tuple							):


    '''simple docstring'''





    _snake_case   :				str														=	'This is a test'
    _snake_case   :				Tuple														=	'This is a test'
    return input_text, output_text







  def UpperCamelCase_ (      self      :					List[str]							):


    '''simple docstring'''





    _snake_case   :				Any														=	'<s>'
    _snake_case   :				Optional[Any]														=	1

    self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase							)      ,				UpperCamelCase							)
    self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase							)      ,				UpperCamelCase							)







  def UpperCamelCase_ (      self      :					List[Any]							):


    '''simple docstring'''





    _snake_case   :				Tuple														=	list(self.get_tokenizer().get_vocab().keys()							)

    self.assertEqual(vocab_keys[0]      ,				'<unk>'							)
    self.assertEqual(vocab_keys[1]      ,				'<s>'							)
    self.assertEqual(vocab_keys[-1]      ,				'j'							)
    self.assertEqual(len(UpperCamelCase							)      ,				20_00							)







  def UpperCamelCase_ (      self      :					str							):


    '''simple docstring'''





    self.assertEqual(self.get_tokenizer().vocab_size      ,				20_00							)







  def UpperCamelCase_ (      self      :					int							):


    '''simple docstring'''





    _snake_case   :				List[Any]														=	GPTSwaTokenizer(UpperCamelCase							)

    _snake_case   :				Optional[int]														=	tokenizer.tokenize('This is a test'							)
    self.assertListEqual(UpperCamelCase      ,				['▁This', '▁is', '▁a', '▁t', 'est']							)

    self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase							)      ,				[4_65, 2_87, 2_65, 6_31, 8_42]							)

    _snake_case   :				Dict														=	tokenizer.tokenize('I was born in 92000, and this is falsé.'							)
    # fmt: off
    self.assertListEqual(
        UpperCamelCase      ,				['▁I', '▁was', '▁bor', 'n', '▁in', '▁', '<0x39>', '2', '0', '0', '0', ',', '▁and', '▁this', '▁is', '▁f', 'al', 's', '<0xC3>', '<0xA9>', '.']      ,				)
    # fmt: on

    _snake_case   :				Tuple														=	tokenizer.convert_tokens_to_ids(UpperCamelCase							)
    self.assertListEqual(
        UpperCamelCase      ,				[2_62, 2_72, 15_25, 2_86, 2_71, 2_68, 60, 9_16, 6_33, 6_33, 6_33, 2_59, 2_66, 3_01, 2_87, 3_84, 3_67, 2_63, 1_98, 1_72, 2_60]      ,				)

    _snake_case   :				Optional[int]														=	tokenizer.convert_ids_to_tokens(UpperCamelCase							)
    # fmt: off
    self.assertListEqual(
        UpperCamelCase      ,				['▁I', '▁was', '▁bor', 'n', '▁in', '▁', '<0x39>', '2', '0', '0', '0', ',', '▁and', '▁this', '▁is', '▁f', 'al', 's', '<0xC3>', '<0xA9>', '.']							)
    # fmt: on







  def UpperCamelCase_ (      self      :					List[str]							):


    '''simple docstring'''





    _snake_case   :				Any														=	GPTSwaTokenizer(UpperCamelCase							)
    _snake_case   :				Tuple														=	['This is a test', 'I was born in 92000, and this is falsé.']
    _snake_case   :				Dict														=	[
        [4_65, 2_87, 2_65, 6_31, 8_42],
        [2_62, 2_72, 15_25, 2_86, 2_71, 2_68, 60, 9_16, 6_33, 6_33, 6_33, 2_59, 2_66, 3_01, 2_87, 3_84, 3_67, 2_63, 1_98, 1_72, 2_60],
    ]

    # Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
    for text, expected_ids in zip(UpperCamelCase      ,				UpperCamelCase							):
      self.assertListEqual(tokenizer.encode_fast(UpperCamelCase							)      ,				UpperCamelCase							)

    # Test that decode_fast returns the input text
    for text, token_ids in zip(UpperCamelCase      ,				UpperCamelCase							):
      self.assertEqual(tokenizer.decode_fast(UpperCamelCase							)      ,				UpperCamelCase							)







  @slow
  def UpperCamelCase_ (      self      :					Any							):


    '''simple docstring'''





    _snake_case   :				str														=	[
        '<|python|>def fibonacci(n)\n    if n < 0:\n        print(\'Incorrect input\')',
        'Hey there, how are you doing this fine day?',
        'This is a text with a trailing spaces followed by a dot     .',
        'Häj sväjs lillebrör! =)',
        'Det är inget fel på Mr. Cool',
    ]

    # fmt: off
    _snake_case   :				int														=	{'input_ids': [[6_34_23, 5, 68_11, 1_49_54, 2_82, 8_16, 38_21, 6_34_66, 6_34_25, 6_34_62, 18, 6_39_78, 6_78, 3_01, 13_20, 6_34_23, 6_34_55, 6_34_58, 18, 6_39_82, 42_46, 39_40, 19_01, 4_77_89, 55_47, 1_89_94], [1_96_30, 11_00, 6_34_46, 13_42, 6_33, 5_44, 44_88, 5_93, 51_02, 24_16, 6_34_95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [16_52, 4_28, 2_68, 19_36, 5_15, 2_68, 5_85_93, 2_24_13, 91_06, 5_46, 2_68, 3_32_13, 6_39_79, 6_98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_51_30, 6_34_50, 9_24, 6_34_49, 22_49, 40_62, 15_58, 3_18, 6_35_04, 2_14_98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_09, 3_77, 28_27, 25_59, 3_32, 65_75, 6_34_43, 2_68_01, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
    # fmt: on
    self.tokenizer_integration_test_util(
        expected_encoding=UpperCamelCase      ,				model_name='AI-Sweden/gpt-sw3-126m'      ,				sequences=UpperCamelCase      ,				)











import warnings

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding



class        _lowerCAmelCase    (			UpperCAmelCase_							):


  '''simple docstring'''
  a_							:       Union[str, Any]      					=["""image_processor""", """tokenizer"""]
  a_							:       Optional[int]      					="""CLIPImageProcessor"""
  a_							:       Optional[Any]      					=("""XLMRobertaTokenizer""", """XLMRobertaTokenizerFast""")







  def __init__(      self      :					List[str]      ,				UpperCamelCase      :					Optional[int]=None      ,				UpperCamelCase      :					Optional[Any]=None      ,				**UpperCamelCase      :					Dict							):


    '''simple docstring'''





    _snake_case   :				int														=	None
    if "feature_extractor" in kwargs:
      warnings.warn(
          'The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'
          ' instead.'      ,				UpperCamelCase      ,				)
      _snake_case   :				Optional[Any]														=	kwargs.pop('feature_extractor'							)

    _snake_case   :				Dict														=	image_processor if image_processor is not None else feature_extractor
    if image_processor is None:
      raise ValueError('You need to specify an `image_processor`.'							)
    if tokenizer is None:
      raise ValueError('You need to specify a `tokenizer`.'							)

    super().__init__(UpperCamelCase      ,				UpperCamelCase							)







  def __call__(      self      :					Dict      ,				UpperCamelCase      :					Optional[Any]=None      ,				UpperCamelCase      :					Optional[Any]=None      ,				UpperCamelCase      :					Optional[int]=None      ,				**UpperCamelCase      :					Dict							):


    '''simple docstring'''





    if text is None and images is None:
      raise ValueError('You have to specify either text or images. Both cannot be none.'							)

    if text is not None:
      _snake_case   :				Optional[int]														=	self.tokenizer(UpperCamelCase      ,				return_tensors=UpperCamelCase      ,				**UpperCamelCase							)

    if images is not None:
      _snake_case   :				Optional[int]														=	self.image_processor(UpperCamelCase      ,				return_tensors=UpperCamelCase      ,				**UpperCamelCase							)

    if text is not None and images is not None:
      _snake_case   :				Optional[int]														=	image_features.pixel_values
      return encoding
    elif text is not None:
      return encoding
    else:
      return BatchEncoding(data=dict(**UpperCamelCase							)      ,				tensor_type=UpperCamelCase							)







  def UpperCamelCase_ (      self      :					Union[str, Any]      ,				*UpperCamelCase      :					Any      ,				**UpperCamelCase      :					Union[str, Any]							):


    '''simple docstring'''





    return self.tokenizer.batch_decode(*UpperCamelCase      ,				**UpperCamelCase							)







  def UpperCamelCase_ (      self      :					Union[str, Any]      ,				*UpperCamelCase      :					Union[str, Any]      ,				**UpperCamelCase      :					Optional[Any]							):


    '''simple docstring'''





    return self.tokenizer.decode(*UpperCamelCase      ,				**UpperCamelCase							)







  @property
  def UpperCamelCase_ (      self      :					Optional[int]							):


    '''simple docstring'''





    _snake_case   :				Any														=	self.tokenizer.model_input_names
    _snake_case   :				List[Any]														=	self.image_processor.model_input_names
    return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names							)							)




