





import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class        _lowerCAmelCase    (			UpperCAmelCase_							,    unittest.TestCase							):


  '''simple docstring'''
  a_							:       Dict      					=CLIPTokenizer
  a_							:       str      					=CLIPTokenizerFast
  a_							:       str      					=True
  a_							:       Union[str, Any]      					={}
  a_							:       List[str]      					=False







  def UpperCamelCase_ (      self      :					int							):


    '''simple docstring'''





    super().setUp()

    # fmt: off
    _snake_case   :				Union[str, Any]														=	['l', 'o', 'w', 'e', 'r', 's', 't', 'i', 'd', 'n', 'lo', 'l</w>', 'w</w>', 'r</w>', 't</w>', 'low</w>', 'er</w>', 'lowest</w>', 'newer</w>', 'wider', '<unk>', '<|startoftext|>', '<|endoftext|>']
    # fmt: on
    _snake_case   :				Optional[Any]														=	dict(zip(UpperCamelCase      ,				range(len(UpperCamelCase							)							)							)							)
    _snake_case   :				Optional[int]														=	['#version: 0.2', 'l o', 'lo w</w>', 'e r</w>']
    _snake_case   :				Tuple														=	{'unk_token': '<unk>'}

    _snake_case   :				Optional[Any]														=	os.path.join(self.tmpdirname      ,				VOCAB_FILES_NAMES['vocab_file']							)
    _snake_case   :				List[Any]														=	os.path.join(self.tmpdirname      ,				VOCAB_FILES_NAMES['merges_file']							)
    with open(self.vocab_file      ,				'w'      ,				encoding='utf-8'							) as fp:
      fp.write(json.dumps(UpperCamelCase							) + '\n'							)
    with open(self.merges_file      ,				'w'      ,				encoding='utf-8'							) as fp:
      fp.write('\n'.join(UpperCamelCase							)							)







  def UpperCamelCase_ (      self      :					Optional[int]      ,				**UpperCamelCase      :					int							):


    '''simple docstring'''





    kwargs.update(self.special_tokens_map							)
    return CLIPTokenizer.from_pretrained(self.tmpdirname      ,				**UpperCamelCase							)







  def UpperCamelCase_ (      self      :					Union[str, Any]      ,				**UpperCamelCase      :					Optional[Any]							):


    '''simple docstring'''





    kwargs.update(self.special_tokens_map							)
    return CLIPTokenizerFast.from_pretrained(self.tmpdirname      ,				**UpperCamelCase							)







  def UpperCamelCase_ (      self      :					Any      ,				UpperCamelCase      :					str							):


    '''simple docstring'''





    _snake_case   :				Optional[Any]														=	'lower newer'
    _snake_case   :				List[str]														=	'lower newer'
    return input_text, output_text







  def UpperCamelCase_ (      self      :					Dict							):


    '''simple docstring'''





    _snake_case   :				int														=	CLIPTokenizer(self.vocab_file      ,				self.merges_file      ,				**self.special_tokens_map							)
    _snake_case   :				Tuple														=	'lower newer'
    _snake_case   :				List[Any]														=	['lo', 'w', 'er</w>', 'n', 'e', 'w', 'er</w>']
    _snake_case   :				Union[str, Any]														=	tokenizer.tokenize(UpperCamelCase							)
    self.assertListEqual(UpperCamelCase      ,				UpperCamelCase							)

    _snake_case   :				List[str]														=	tokens + [tokenizer.unk_token]
    _snake_case   :				Any														=	[10, 2, 16, 9, 3, 2, 16, 20]
    self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase							)      ,				UpperCamelCase							)







  @require_ftfy
  def UpperCamelCase_ (      self      :					Tuple							):


    '''simple docstring'''





    for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
      with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
        _snake_case   :				str														=	self.tokenizer_class.from_pretrained(UpperCamelCase      ,				**UpperCamelCase							)
        _snake_case   :				Union[str, Any]														=	self.rust_tokenizer_class.from_pretrained(UpperCamelCase      ,				**UpperCamelCase							)

        _snake_case   :				int														=	'A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'
        _snake_case   :				Dict														=	tokenizer_s.tokenize(UpperCamelCase							)
        _snake_case   :				Dict														=	tokenizer_r.tokenize(UpperCamelCase							)

        self.assertListEqual(UpperCamelCase      ,				UpperCamelCase							)

        # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
        # with Tilde) encoded in 2 different ways
        _snake_case   :				Union[str, Any]														=	'xa\u0303y' + ' ' + 'x\xe3y'
        _snake_case   :				Any														=	tokenizer_s.tokenize(UpperCamelCase							)
        _snake_case   :				int														=	tokenizer_r.tokenize(UpperCamelCase							)

        self.assertListEqual(UpperCamelCase      ,				UpperCamelCase							)

        # Test that the tokenization is identical on unicode of space type
        _snake_case   :				int														=	[
            '\u0009',  # (horizontal tab, '\t')
            '\u000B',  # (vertical tab)
            '\u000C',  # (form feed)
            '\u0020',  # (space, ' ')
            '\u200E',  # (left-to-right mark):w
            '\u200F',  # (right-to-left mark)
        ]
        for unicode_seq in spaces_unicodes:
          _snake_case   :				Tuple														=	tokenizer_s.tokenize(UpperCamelCase							)
          _snake_case   :				Optional[Any]														=	tokenizer_r.tokenize(UpperCamelCase							)

          self.assertListEqual(UpperCamelCase      ,				UpperCamelCase							)

        # Test that the tokenization is identical on unicode of line break type
        _snake_case   :				Optional[int]														=	[
            '\u000A',  # (line feed, '\n')
            '\r\n',  # (carriage return and line feed, '\r\n')
            '\u000D',  # (carriage return, '\r')
            '\r',  # (carriage return, '\r')
            '\u000D',  # (carriage return, '\r')
            '\u2028',  # (line separator)
            '\u2029',  # (paragraph separator)
            # "\u0085", # (next line)
        ]

        # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
        # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
        # space (and thus into an empty list).

        for unicode_seq in line_break_unicodes:
          _snake_case   :				List[str]														=	tokenizer_s.tokenize(UpperCamelCase							)
          _snake_case   :				List[str]														=	tokenizer_r.tokenize(UpperCamelCase							)

          self.assertListEqual(UpperCamelCase      ,				UpperCamelCase							)







  def UpperCamelCase_ (      self      :					Optional[Any]							):


    '''simple docstring'''





    for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
      with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
        _snake_case   :				Tuple														=	'hello'  # `hello` is a token in the vocabulary of `pretrained_name`
        _snake_case   :				int														=	f"""{text_of_1_token} {text_of_1_token}"""

        _snake_case   :				List[str]														=	self.rust_tokenizer_class.from_pretrained(
            UpperCamelCase      ,				use_fast=UpperCamelCase      ,				)
        _snake_case   :				int														=	tokenizer_r(UpperCamelCase      ,				return_offsets_mapping=UpperCamelCase      ,				add_special_tokens=UpperCamelCase							)
        self.assertEqual(encoding.offset_mapping[0]      ,				(0, len(UpperCamelCase							))							)
        self.assertEqual(
            encoding.offset_mapping[1]      ,				(len(UpperCamelCase							) + 1, len(UpperCamelCase							) + 1 + len(UpperCamelCase							))      ,				)

        _snake_case   :				Tuple														=	f""" {text}"""

        _snake_case   :				str														=	self.rust_tokenizer_class.from_pretrained(
            UpperCamelCase      ,				use_fast=UpperCamelCase      ,				)
        _snake_case   :				Optional[Any]														=	tokenizer_r(UpperCamelCase      ,				return_offsets_mapping=UpperCamelCase      ,				add_special_tokens=UpperCamelCase							)
        self.assertEqual(encoding.offset_mapping[0]      ,				(1, 1 + len(UpperCamelCase							))							)
        self.assertEqual(
            encoding.offset_mapping[1]      ,				(1 + len(UpperCamelCase							) + 1, 1 + len(UpperCamelCase							) + 1 + len(UpperCamelCase							))      ,				)







  def UpperCamelCase_ (      self      :					List[str]							):


    '''simple docstring'''





    with self.assertRaises(UpperCamelCase							) as context:
      self.rust_tokenizer_class.from_pretrained('robot-test/old-clip-tokenizer'							)

    self.assertTrue(
        context.exception.args[0].startswith(
            'The `backend_tokenizer` provided does not match the expected format.'							)							)







  @require_ftfy
  def UpperCamelCase_ (      self      :					Tuple							):


    '''simple docstring'''





    super().test_tokenization_python_rust_equals()







  def UpperCamelCase_ (      self      :					Optional[int]							):


    '''simple docstring'''





    pass











import warnings

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding



class        _lowerCAmelCase    (			UpperCAmelCase_							):


  '''simple docstring'''
  a_							:       Union[str, Any]      					=["""image_processor""", """tokenizer"""]
  a_							:       Optional[int]      					="""CLIPImageProcessor"""
  a_							:       Optional[Any]      					=("""XLMRobertaTokenizer""", """XLMRobertaTokenizerFast""")







  def __init__(      self      :					List[str]      ,				UpperCamelCase      :					Optional[int]=None      ,				UpperCamelCase      :					Optional[Any]=None      ,				**UpperCamelCase      :					Dict							):


    '''simple docstring'''





    _snake_case   :				int														=	None
    if "feature_extractor" in kwargs:
      warnings.warn(
          'The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'
          ' instead.'      ,				UpperCamelCase      ,				)
      _snake_case   :				Optional[Any]														=	kwargs.pop('feature_extractor'							)

    _snake_case   :				Dict														=	image_processor if image_processor is not None else feature_extractor
    if image_processor is None:
      raise ValueError('You need to specify an `image_processor`.'							)
    if tokenizer is None:
      raise ValueError('You need to specify a `tokenizer`.'							)

    super().__init__(UpperCamelCase      ,				UpperCamelCase							)







  def __call__(      self      :					Dict      ,				UpperCamelCase      :					Optional[Any]=None      ,				UpperCamelCase      :					Optional[Any]=None      ,				UpperCamelCase      :					Optional[int]=None      ,				**UpperCamelCase      :					Dict							):


    '''simple docstring'''





    if text is None and images is None:
      raise ValueError('You have to specify either text or images. Both cannot be none.'							)

    if text is not None:
      _snake_case   :				Optional[int]														=	self.tokenizer(UpperCamelCase      ,				return_tensors=UpperCamelCase      ,				**UpperCamelCase							)

    if images is not None:
      _snake_case   :				Optional[int]														=	self.image_processor(UpperCamelCase      ,				return_tensors=UpperCamelCase      ,				**UpperCamelCase							)

    if text is not None and images is not None:
      _snake_case   :				Optional[int]														=	image_features.pixel_values
      return encoding
    elif text is not None:
      return encoding
    else:
      return BatchEncoding(data=dict(**UpperCamelCase							)      ,				tensor_type=UpperCamelCase							)







  def UpperCamelCase_ (      self      :					Union[str, Any]      ,				*UpperCamelCase      :					Any      ,				**UpperCamelCase      :					Union[str, Any]							):


    '''simple docstring'''





    return self.tokenizer.batch_decode(*UpperCamelCase      ,				**UpperCamelCase							)







  def UpperCamelCase_ (      self      :					Union[str, Any]      ,				*UpperCamelCase      :					Union[str, Any]      ,				**UpperCamelCase      :					Optional[Any]							):


    '''simple docstring'''





    return self.tokenizer.decode(*UpperCamelCase      ,				**UpperCamelCase							)







  @property
  def UpperCamelCase_ (      self      :					Optional[int]							):


    '''simple docstring'''





    _snake_case   :				Any														=	self.tokenizer.model_input_names
    _snake_case   :				List[Any]														=	self.image_processor.model_input_names
    return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names							)							)




