





import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class        _lowerCAmelCase    (			UpperCAmelCase_							,    unittest.TestCase							):


		'''simple docstring'''
		a_							:       Dict      					=CLIPTokenizer
		a_							:       str      					=CLIPTokenizerFast
		a_							:       str      					=True
		a_							:       Union[str, Any]      					={}
		a_							:       List[str]      					=False







		def UpperCamelCase_ (      self      :					int							):


				'''simple docstring'''





				super().setUp()

				# fmt: off
				_snake_case   :				Union[str, Any]														=	['l', 'o', 'w', 'e', 'r', 's', 't', 'i', 'd', 'n', 'lo', 'l</w>', 'w</w>', 'r</w>', 't</w>', 'low</w>', 'er</w>', 'lowest</w>', 'newer</w>', 'wider', '<unk>', '<|startoftext|>', '<|endoftext|>']
				# fmt: on
				_snake_case   :				Optional[Any]														=	dict(zip(UpperCamelCase      ,				range(len(UpperCamelCase							)							)							)							)
				_snake_case   :				Optional[int]														=	['#version: 0.2', 'l o', 'lo w</w>', 'e r</w>']
				_snake_case   :				Tuple														=	{'unk_token': '<unk>'}

				_snake_case   :				Optional[Any]														=	os.path.join(self.tmpdirname      ,				VOCAB_FILES_NAMES['vocab_file']							)
				_snake_case   :				List[Any]														=	os.path.join(self.tmpdirname      ,				VOCAB_FILES_NAMES['merges_file']							)
				with open(self.vocab_file      ,				'w'      ,				encoding='utf-8'							) as fp:
						fp.write(json.dumps(UpperCamelCase							) + '\n'							)
				with open(self.merges_file      ,				'w'      ,				encoding='utf-8'							) as fp:
						fp.write('\n'.join(UpperCamelCase							)							)







		def UpperCamelCase_ (      self      :					Optional[int]      ,				**UpperCamelCase      :					int							):


				'''simple docstring'''





				kwargs.update(self.special_tokens_map							)
				return CLIPTokenizer.from_pretrained(self.tmpdirname      ,				**UpperCamelCase							)







		def UpperCamelCase_ (      self      :					Union[str, Any]      ,				**UpperCamelCase      :					Optional[Any]							):


				'''simple docstring'''





				kwargs.update(self.special_tokens_map							)
				return CLIPTokenizerFast.from_pretrained(self.tmpdirname      ,				**UpperCamelCase							)







		def UpperCamelCase_ (      self      :					Any      ,				UpperCamelCase      :					str							):


				'''simple docstring'''





				_snake_case   :				Optional[Any]														=	'lower newer'
				_snake_case   :				List[str]														=	'lower newer'
				return input_text, output_text







		def UpperCamelCase_ (      self      :					Dict							):


				'''simple docstring'''





				_snake_case   :				int														=	CLIPTokenizer(self.vocab_file      ,				self.merges_file      ,				**self.special_tokens_map							)
				_snake_case   :				Tuple														=	'lower newer'
				_snake_case   :				List[Any]														=	['lo', 'w', 'er</w>', 'n', 'e', 'w', 'er</w>']
				_snake_case   :				Union[str, Any]														=	tokenizer.tokenize(UpperCamelCase							)
				self.assertListEqual(UpperCamelCase      ,				UpperCamelCase							)

				_snake_case   :				List[str]														=	tokens + [tokenizer.unk_token]
				_snake_case   :				Any														=	[10, 2, 16, 9, 3, 2, 16, 20]
				self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase							)      ,				UpperCamelCase							)







		@require_ftfy
		def UpperCamelCase_ (      self      :					Tuple							):


				'''simple docstring'''





				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
						with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
								_snake_case   :				str														=	self.tokenizer_class.from_pretrained(UpperCamelCase      ,				**UpperCamelCase							)
								_snake_case   :				Union[str, Any]														=	self.rust_tokenizer_class.from_pretrained(UpperCamelCase      ,				**UpperCamelCase							)

								_snake_case   :				int														=	'A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'
								_snake_case   :				Dict														=	tokenizer_s.tokenize(UpperCamelCase							)
								_snake_case   :				Dict														=	tokenizer_r.tokenize(UpperCamelCase							)

								self.assertListEqual(UpperCamelCase      ,				UpperCamelCase							)

								# Test that the tokenization is identical on an example containing a character (Latin Small Letter A
								# with Tilde) encoded in 2 different ways
								_snake_case   :				Union[str, Any]														=	'xa\u0303y' + ' ' + 'x\xe3y'
								_snake_case   :				Any														=	tokenizer_s.tokenize(UpperCamelCase							)
								_snake_case   :				int														=	tokenizer_r.tokenize(UpperCamelCase							)

								self.assertListEqual(UpperCamelCase      ,				UpperCamelCase							)

								# Test that the tokenization is identical on unicode of space type
								_snake_case   :				int														=	[
								    '\u0009',  # (horizontal tab, '\t')
								    '\u000B',  # (vertical tab)
								    '\u000C',  # (form feed)
								    '\u0020',  # (space, ' ')
								    '\u200E',  # (left-to-right mark):w
								    '\u200F',  # (right-to-left mark)
								]
								for unicode_seq in spaces_unicodes:
										_snake_case   :				Tuple														=	tokenizer_s.tokenize(UpperCamelCase							)
										_snake_case   :				Optional[Any]														=	tokenizer_r.tokenize(UpperCamelCase							)

										self.assertListEqual(UpperCamelCase      ,				UpperCamelCase							)

								# Test that the tokenization is identical on unicode of line break type
								_snake_case   :				Optional[int]														=	[
								    '\u000A',  # (line feed, '\n')
								    '\r\n',  # (carriage return and line feed, '\r\n')
								    '\u000D',  # (carriage return, '\r')
								    '\r',  # (carriage return, '\r')
								    '\u000D',  # (carriage return, '\r')
								    '\u2028',  # (line separator)
								    '\u2029',  # (paragraph separator)
								    # "\u0085", # (next line)
								]

								# The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
								# it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
								# space (and thus into an empty list).

								for unicode_seq in line_break_unicodes:
										_snake_case   :				List[str]														=	tokenizer_s.tokenize(UpperCamelCase							)
										_snake_case   :				List[str]														=	tokenizer_r.tokenize(UpperCamelCase							)

										self.assertListEqual(UpperCamelCase      ,				UpperCamelCase							)







		def UpperCamelCase_ (      self      :					Optional[Any]							):


				'''simple docstring'''





				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
						with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
								_snake_case   :				Tuple														=	'hello'  # `hello` is a token in the vocabulary of `pretrained_name`
								_snake_case   :				int														=	f"""{text_of_1_token} {text_of_1_token}"""

								_snake_case   :				List[str]														=	self.rust_tokenizer_class.from_pretrained(
								    UpperCamelCase      ,				use_fast=UpperCamelCase      ,				)
								_snake_case   :				int														=	tokenizer_r(UpperCamelCase      ,				return_offsets_mapping=UpperCamelCase      ,				add_special_tokens=UpperCamelCase							)
								self.assertEqual(encoding.offset_mapping[0]      ,				(0, len(UpperCamelCase							))							)
								self.assertEqual(
								    encoding.offset_mapping[1]      ,				(len(UpperCamelCase							) + 1, len(UpperCamelCase							) + 1 + len(UpperCamelCase							))      ,				)

								_snake_case   :				Tuple														=	f""" {text}"""

								_snake_case   :				str														=	self.rust_tokenizer_class.from_pretrained(
								    UpperCamelCase      ,				use_fast=UpperCamelCase      ,				)
								_snake_case   :				Optional[Any]														=	tokenizer_r(UpperCamelCase      ,				return_offsets_mapping=UpperCamelCase      ,				add_special_tokens=UpperCamelCase							)
								self.assertEqual(encoding.offset_mapping[0]      ,				(1, 1 + len(UpperCamelCase							))							)
								self.assertEqual(
								    encoding.offset_mapping[1]      ,				(1 + len(UpperCamelCase							) + 1, 1 + len(UpperCamelCase							) + 1 + len(UpperCamelCase							))      ,				)







		def UpperCamelCase_ (      self      :					List[str]							):


				'''simple docstring'''





				with self.assertRaises(UpperCamelCase							) as context:
						self.rust_tokenizer_class.from_pretrained('robot-test/old-clip-tokenizer'							)

				self.assertTrue(
				    context.exception.args[0].startswith(
				        'The `backend_tokenizer` provided does not match the expected format.'							)							)







		@require_ftfy
		def UpperCamelCase_ (      self      :					Tuple							):


				'''simple docstring'''





				super().test_tokenization_python_rust_equals()







		def UpperCamelCase_ (      self      :					Optional[int]							):


				'''simple docstring'''





				pass











import unittest
from pathlib import Path
from tempfile import TemporaryDirectory

from transformers import AutoConfig, TFGPTaLMHeadModel, is_keras_nlp_available, is_tf_available
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import require_keras_nlp, require_tf, slow


if is_tf_available():
      import tensorflow as tf

if is_keras_nlp_available():
      from transformers.models.gpta import TFGPTaTokenizer


lowerCAmelCase_						   =    ["""gpt2"""]
lowerCAmelCase_						   =    """gpt2"""

if is_tf_available():



      class        _lowerCAmelCase    (			tf.Module							):


        '''simple docstring'''







        def __init__(      self      :					List[str]      ,				UpperCamelCase      :					Dict							):


          '''simple docstring'''





          super().__init__()
          _snake_case   :				Optional[int]														=	tokenizer
          _snake_case   :				Union[str, Any]														=	AutoConfig.from_pretrained(UpperCamelCase							)
          _snake_case   :				int														=	TFGPTaLMHeadModel.from_config(UpperCamelCase							)







        @tf.function(input_signature=(tf.TensorSpec((None,)      ,				tf.string      ,				name='text'							),)							)
        def UpperCamelCase_ (      self      :					Optional[int]      ,				UpperCamelCase      :					Optional[int]							):


          '''simple docstring'''





          _snake_case   :				Dict														=	self.tokenizer(UpperCamelCase							)
          _snake_case   :				Union[str, Any]														=	tokenized['input_ids'].to_tensor()

          _snake_case   :				Any														=	tf.cast(input_ids_dense > 0      ,				tf.intaa							)
          # input_mask = tf.reshape(input_mask, [-1, MAX_SEQ_LEN])

          _snake_case   :				Tuple														=	self.model(input_ids=UpperCamelCase      ,				attention_mask=UpperCamelCase							)['logits']

          return outputs



@require_tf
@require_keras_nlp
class        _lowerCAmelCase    (			unittest.TestCase							):


  '''simple docstring'''







  def UpperCamelCase_ (      self      :					Any							):


    '''simple docstring'''





    super().setUp()

    _snake_case   :				Optional[int]														=	[GPTaTokenizer.from_pretrained(UpperCamelCase							) for checkpoint in (TOKENIZER_CHECKPOINTS)]
    _snake_case   :				Tuple														=	[TFGPTaTokenizer.from_pretrained(UpperCamelCase							) for checkpoint in TOKENIZER_CHECKPOINTS]
    assert len(self.tokenizers							) == len(self.tf_tokenizers							)

    _snake_case   :				Any														=	[
        'This is a straightforward English test sentence.',
        'This one has some weird characters\rto\nsee\r\nif  those\u00E9break things.',
        'Now we\'re going to add some Chinese: 一 二 三 一二三',
        'And some much more rare Chinese: 齉 堃 齉堃',
        'Je vais aussi écrire en français pour tester les accents',
        'Classical Irish also has some unusual characters, so in they go: Gaelaċ, ꝼ',
    ]
    _snake_case   :				Tuple														=	list(zip(self.test_sentences      ,				self.test_sentences[::-1]							)							)







  def UpperCamelCase_ (      self      :					Union[str, Any]							):


    '''simple docstring'''





    for tokenizer, tf_tokenizer in zip(self.tokenizers      ,				self.tf_tokenizers							):
      for test_inputs in self.test_sentences:
        _snake_case   :				Optional[int]														=	tokenizer([test_inputs]      ,				return_tensors='tf'							)
        _snake_case   :				Tuple														=	tf_tokenizer([test_inputs]							)

        for key in python_outputs.keys():
          # convert them to numpy to avoid messing with ragged tensors
          _snake_case   :				Dict														=	python_outputs[key].numpy()
          _snake_case   :				Optional[Any]														=	tf_outputs[key].numpy()

          self.assertTrue(tf.reduce_all(python_outputs_values.shape == tf_outputs_values.shape							)							)
          self.assertTrue(tf.reduce_all(tf.cast(UpperCamelCase      ,				tf.intaa							) == tf_outputs_values							)							)







  @slow
  def UpperCamelCase_ (      self      :					Optional[Any]							):


    '''simple docstring'''





    for tf_tokenizer in self.tf_tokenizers:
      _snake_case   :				str														=	tf.function(UpperCamelCase							)
      for test_inputs in self.test_sentences:
        _snake_case   :				int														=	tf.constant(UpperCamelCase							)
        _snake_case   :				Tuple														=	compiled_tokenizer(UpperCamelCase							)
        _snake_case   :				int														=	tf_tokenizer(UpperCamelCase							)

        for key in eager_outputs.keys():
          self.assertTrue(tf.reduce_all(eager_outputs[key] == compiled_outputs[key]							)							)







  @slow
  def UpperCamelCase_ (      self      :					Tuple							):


    '''simple docstring'''





    for tf_tokenizer in self.tf_tokenizers:
      _snake_case   :				Union[str, Any]														=	ModelToSave(tokenizer=UpperCamelCase							)
      _snake_case   :				Optional[Any]														=	tf.convert_to_tensor([self.test_sentences[0]]							)
      _snake_case   :				Tuple														=	model.serving(UpperCamelCase							)  # Build model with some sample inputs
      with TemporaryDirectory() as tempdir:
        _snake_case   :				str														=	Path(UpperCamelCase							) / 'saved.model'
        tf.saved_model.save(UpperCamelCase      ,				UpperCamelCase      ,				signatures={'serving_default': model.serving}							)
        _snake_case   :				Optional[int]														=	tf.saved_model.load(UpperCamelCase							)
      _snake_case   :				List[str]														=	loaded_model.signatures['serving_default'](UpperCamelCase							)['output_0']
      # We may see small differences because the loaded model is compiled, so we need an epsilon for the test
      self.assertTrue(tf.reduce_all(out == loaded_output							)							)







  @slow
  def UpperCamelCase_ (      self      :					Optional[int]							):


    '''simple docstring'''





    for tf_tokenizer in self.tf_tokenizers:
      _snake_case   :				Optional[Any]														=	tf.convert_to_tensor([self.test_sentences[0]]							)
      _snake_case   :				Any														=	tf_tokenizer(UpperCamelCase							)  # Build model with some sample inputs

      _snake_case   :				Optional[Any]														=	tf_tokenizer.get_config()
      _snake_case   :				Tuple														=	TFGPTaTokenizer.from_config(UpperCamelCase							)
      _snake_case   :				Optional[Any]														=	model_from_config(UpperCamelCase							)

      for key in from_config_output.keys():
        self.assertTrue(tf.reduce_all(from_config_output[key] == out[key]							)							)







  @slow
  def UpperCamelCase_ (      self      :					Any							):


    '''simple docstring'''





    for tf_tokenizer in self.tf_tokenizers:
      # for the test to run
      _snake_case   :				Union[str, Any]														=	12_31_23

      for max_length in [3, 5, 10_24]:
        _snake_case   :				Union[str, Any]														=	tf.convert_to_tensor([self.test_sentences[0]]							)
        _snake_case   :				List[str]														=	tf_tokenizer(UpperCamelCase      ,				max_length=UpperCamelCase							)

        _snake_case   :				int														=	out['input_ids'].numpy().shape[1]

        assert out_length == max_length




