





import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
						from transformers.pipelines.conversational import Conversation


lowerCAmelCase_						   =    logging.get_logger(__name__)

lowerCAmelCase_						   =    {"""vocab_file""": """vocab.txt""", """emoji_file""": """emoji.json"""}

lowerCAmelCase_						   =    {
    """vocab_file""": {
        """abeja/gpt-neox-japanese-2.7b""": """https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt""",
    },
    """emoji_file""": {
        """abeja/gpt-neox-japanese-2.7b""": """https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json""",
    },
}

lowerCAmelCase_						   =    {
    """abeja/gpt-neox-japanese-2.7b""": 2048,
}


def     lowerCamelCase_	(      lowerCAmelCase:	Union[str, Any]    ,							lowerCAmelCase:	Union[str, Any]				)->		Tuple:
		with open(lowerCAmelCase    ,							'r'    ,							encoding='utf-8'				) as f:
				_snake_case   :				Optional[Any]														=	json.loads(f.read()				)

		_snake_case   :				List[Any]														=	collections.OrderedDict()
		_snake_case   :				Optional[int]														=	collections.OrderedDict()
		_snake_case   :				Tuple														=	collections.OrderedDict()
		with open(lowerCAmelCase    ,							'r'    ,							encoding='utf-8'				) as f:
				_snake_case   :				Tuple														=	f.readlines()
		_snake_case   :				Union[str, Any]														=	[[t.rstrip('\n'				)] if (t == ',' or ',' not in t) else t.rstrip('\n'				).split(','				) for t in token]
		for idx, b in enumerate(lowerCAmelCase				):
				_snake_case   :				List[str]														=	b
				_snake_case   :				int														=	idx
				for wd in b:
						_snake_case   :				List[str]														=	idx

		return vocab, raw_vocab, ids_to_tokens, emoji



class        _lowerCAmelCase    (			UpperCAmelCase_							):


		'''simple docstring'''
		a_							:       Optional[int]      					=VOCAB_FILES_NAMES
		a_							:       List[Any]      					=PRETRAINED_VOCAB_FILES_MAP
		a_							:       Dict      					=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		a_							:       Dict      					=["""input_ids""", """attention_mask"""]







		def __init__(      self      :					List[str]      ,				UpperCamelCase      :					int      ,				UpperCamelCase      :					Dict      ,				UpperCamelCase      :					int="<|endoftext|>"      ,				UpperCamelCase      :					List[Any]="<|endoftext|>"      ,				UpperCamelCase      :					Optional[Any]="<|startoftext|>"      ,				UpperCamelCase      :					Tuple="<|endoftext|>"      ,				UpperCamelCase      :					Optional[Any]=False      ,				**UpperCamelCase      :					Union[str, Any]      ,				):


				'''simple docstring'''





				super().__init__(
				    unk_token=UpperCamelCase      ,				pad_token=UpperCamelCase      ,				bos_token=UpperCamelCase      ,				eos_token=UpperCamelCase      ,				do_clean_text=UpperCamelCase      ,				**UpperCamelCase      ,				)
				if not os.path.isfile(UpperCamelCase							):
						raise ValueError(
						    f"""Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"""
						    ' model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'							)
				if not os.path.isfile(UpperCamelCase							):
						raise ValueError(
						    f"""Can't find a emoji file at path '{emoji_file}'. To load the emoji information from a Google"""
						    ' pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'							)
				_snake_case   :				Dict														=	do_clean_text
				_snake_case   :				str														=	load_vocab_and_emoji(UpperCamelCase      ,				UpperCamelCase							)
				_snake_case   :				Union[str, Any]														=	SubWordJapaneseTokenizer(
				    vocab=self.vocab      ,				ids_to_tokens=self.ids_to_tokens      ,				emoji=self.emoji							)







		@property
		def UpperCamelCase_ (      self      :					List[Any]							):


				'''simple docstring'''





				return len(self.raw_vocab							)







		def UpperCamelCase_ (      self      :					Union[str, Any]							):


				'''simple docstring'''





				return dict(self.raw_vocab      ,				**self.added_tokens_encoder							)







		def UpperCamelCase_ (      self      :					int      ,				UpperCamelCase      :					List[str]							):


				'''simple docstring'''





				return self.subword_tokenizer.tokenize(UpperCamelCase      ,				clean=self.do_clean_text							)







		def UpperCamelCase_ (      self      :					List[str]      ,				UpperCamelCase      :					Optional[int]							):


				'''simple docstring'''





				return self.vocab.get(UpperCamelCase      ,				self.vocab.get(self.unk_token							)							)







		def UpperCamelCase_ (      self      :					Union[str, Any]      ,				UpperCamelCase      :					int							):


				'''simple docstring'''





				return self.subword_tokenizer.convert_id_to_token(UpperCamelCase							)







		def UpperCamelCase_ (      self      :					List[Any]      ,				UpperCamelCase      :					Union[str, Any]							):


				'''simple docstring'''





				_snake_case   :				List[str]														=	''.join(UpperCamelCase							).strip()
				return out_string







		def UpperCamelCase_ (      self      :					Union[str, Any]      ,				UpperCamelCase      :					"Conversation"							):


				'''simple docstring'''





				_snake_case   :				List[Any]														=	[]
				for is_user, text in conversation.iter_texts():
						input_ids.extend(self.encode(UpperCamelCase      ,				add_special_tokens=UpperCamelCase							) + [self.eos_token_id]							)

				if len(UpperCamelCase							) > self.model_max_length:
						_snake_case   :				str														=	input_ids[-self.model_max_length :]
				return input_ids







		def UpperCamelCase_ (      self      :					Union[str, Any]      ,				UpperCamelCase      :					str      ,				UpperCamelCase      :					Optional[str] = None							):


				'''simple docstring'''





				_snake_case   :				Tuple														=	0
				if os.path.isdir(UpperCamelCase							):
						_snake_case   :				Tuple														=	os.path.join(
						    UpperCamelCase      ,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']							)
						_snake_case   :				Any														=	os.path.join(
						    UpperCamelCase      ,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['emoji_file']							)
				else:
						_snake_case   :				str														=	(
						    (filename_prefix + '-' if filename_prefix else '') + save_directory + VOCAB_FILES_NAMES['vocab_file']
						)
						_snake_case   :				Dict														=	(
						    (filename_prefix + '-' if filename_prefix else '') + save_directory + VOCAB_FILES_NAMES['emoji_file']
						)
				with open(UpperCamelCase      ,				'w'      ,				encoding='utf-8'							) as writer:
						for token_index, token in self.ids_to_tokens.items():
								if index != token_index:
										logger.warning(
										    f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
										    ' Please check that the vocabulary is not corrupted!'							)
										_snake_case   :				int														=	token_index
								writer.write(','.join(UpperCamelCase							) + '\n'							)
								index += 1
				with open(UpperCamelCase      ,				'w'      ,				encoding='utf-8'							) as writer:
						json.dump(self.emoji      ,				UpperCamelCase							)
				return vocab_file, emoji_file




class        _lowerCAmelCase    (			UpperCAmelCase_							):


		'''simple docstring'''







		def __init__(      self      :					Optional[int]      ,				UpperCamelCase      :					Any      ,				UpperCamelCase      :					Optional[Any]      ,				UpperCamelCase      :					Any							):


				'''simple docstring'''





				_snake_case   :				Any														=	vocab  # same as swe
				_snake_case   :				Any														=	ids_to_tokens  # same as bpe
				_snake_case   :				Any														=	emoji
				_snake_case   :				Tuple														=	np.max([len(UpperCamelCase							) for w in self.vocab.keys()]							)
				_snake_case   :				Dict														=	re.compile(R'(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)'							)
				_snake_case   :				Optional[int]														=	re.compile(R'[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*'							)
				_snake_case   :				List[str]														=	re.compile(R'[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}'							)
				_snake_case   :				Optional[int]														=	re.compile(
				    R'([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'							)
				_snake_case   :				List[Any]														=	re.compile(
				    R'(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'							)
				_snake_case   :				str														=	re.compile(
				    R'((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*'							)
				_snake_case   :				List[str]														=	'─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿'
				_snake_case   :				Any														=	'▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟'
				_snake_case   :				Optional[int]														=	str.maketrans({k: '<BLOCK>' for k in keisen + blocks}							)







		def __len__(      self      :					Optional[Any]							):


				'''simple docstring'''





				return len(self.ids_to_tokens							)







		def UpperCamelCase_ (      self      :					Tuple      ,				UpperCamelCase      :					str							):


				'''simple docstring'''





				_snake_case   :				List[str]														=	self.content_repattera.sub('<URL>'      ,				UpperCamelCase							)
				_snake_case   :				int														=	self.content_repattera.sub('<EMAIL>'      ,				UpperCamelCase							)
				_snake_case   :				Optional[int]														=	self.content_repattera.sub('<TEL>'      ,				UpperCamelCase							)
				_snake_case   :				Tuple														=	self.content_repattera.sub('<DATE>'      ,				UpperCamelCase							)
				_snake_case   :				Union[str, Any]														=	self.content_repattera.sub('<DATE>'      ,				UpperCamelCase							)
				_snake_case   :				Optional[int]														=	self.content_repattera.sub('<PRICE>'      ,				UpperCamelCase							)
				_snake_case   :				str														=	content.translate(self.content_transa							)
				while "<BLOCK><BLOCK>" in content:
						_snake_case   :				Tuple														=	content.replace('<BLOCK><BLOCK>'      ,				'<BLOCK>'							)
				return content







		def UpperCamelCase_ (      self      :					Dict      ,				UpperCamelCase      :					Tuple      ,				UpperCamelCase      :					Optional[int]=False							):


				'''simple docstring'''





				_snake_case   :				Optional[int]														=	text.replace(' '      ,				'<SP>'							)
				_snake_case   :				Tuple														=	text.replace('　'      ,				'<SP>'							)
				_snake_case   :				Optional[Any]														=	text.replace('\r\n'      ,				'<BR>'							)
				_snake_case   :				int														=	text.replace('\n'      ,				'<BR>'							)
				_snake_case   :				Union[str, Any]														=	text.replace('\r'      ,				'<BR>'							)
				_snake_case   :				List[Any]														=	text.replace('\t'      ,				'<TAB>'							)
				_snake_case   :				Union[str, Any]														=	text.replace('—'      ,				'ー'							)
				_snake_case   :				List[Any]														=	text.replace('−'      ,				'ー'							)
				for k, v in self.emoji["emoji"].items():
						if k in text:
								_snake_case   :				Optional[int]														=	text.replace(UpperCamelCase      ,				UpperCamelCase							)
				if clean:
						_snake_case   :				List[str]														=	self.clean_text(UpperCamelCase							)

				def check_simbol(UpperCamelCase      :					Dict							):
						_snake_case   :				Any														=	x.encode()
						if len(UpperCamelCase							) == 1 and len(UpperCamelCase							) == 2:
								_snake_case   :				Any														=	(int(e[0]							) << 8) + int(e[1]							)
								if (
								    (c >= 0xc2a1 and c <= 0xc2bf)
								    or (c >= 0xc780 and c <= 0xc783)
								    or (c >= 0xcab9 and c <= 0xcbbf)
								    or (c >= 0xcc80 and c <= 0xcda2)
								):
										return True
						return False

				def checkuae(UpperCamelCase      :					Optional[Any]							):
						_snake_case   :				Optional[int]														=	x.encode()
						if len(UpperCamelCase							) == 1 and len(UpperCamelCase							) == 3:
								_snake_case   :				List[Any]														=	(int(e[0]							) << 16) + (int(e[1]							) << 8) + int(e[2]							)
								if c >= 0xe28080 and c <= 0xe2b07f:
										return True
						return False

				_snake_case   :				List[str]														=	0
				_snake_case   :				Dict														=	[]
				while pos < len(UpperCamelCase							):
						_snake_case   :				List[str]														=	min(len(UpperCamelCase							)      ,				pos + self.maxlen + 1							) if text[pos] == '<' else pos + 3
						_snake_case   :				Union[str, Any]														=	[]  # (token_id, token, pos)
						for e in range(UpperCamelCase      ,				UpperCamelCase      ,				-1							):
								_snake_case   :				Dict														=	text[pos:e]
								if wd in self.vocab:
										if wd[0] == "<" and len(UpperCamelCase							) > 2:
												_snake_case   :				Dict														=	[(self.vocab[wd], wd, e)]
												break
										else:
												candidates.append((self.vocab[wd], wd, e)							)
						if len(UpperCamelCase							) > 0:
								# the smallest token_id is adopted
								_snake_case   :				Any														=	sorted(UpperCamelCase      ,				key=lambda UpperCamelCase							: x[0]							)[0]
								result.append(UpperCamelCase							)
								_snake_case   :				Optional[Any]														=	e
						else:
								_snake_case   :				List[str]														=	pos + 1
								_snake_case   :				List[Any]														=	text[pos:end]
								if check_simbol(UpperCamelCase							):
										result.append('<KIGOU>'							)
								elif checkuae(UpperCamelCase							):
										result.append('<U2000U2BFF>'							)
								else:
										for i in wd.encode('utf-8'							):
												result.append('<|byte%d|>' % i							)
								_snake_case   :				Optional[int]														=	end
				return result







		def UpperCamelCase_ (      self      :					Optional[int]      ,				UpperCamelCase      :					Optional[Any]      ,				UpperCamelCase      :					int="\n"							):


				'''simple docstring'''





				_snake_case   :				str														=	[]
				_snake_case   :				str														=	[]
				_snake_case   :				List[str]														=	self.ids_to_tokens[index][0]
				if word[:6] == "<|byte" and word[-2:] == "|>":
						byte_tokens.append(int(word[6:-2]							)							)
				else:
						if len(UpperCamelCase							) > 0:
								words.append(bytearray(UpperCamelCase							).decode('utf-8'      ,				errors='replace'							)							)
								_snake_case   :				int														=	[]
						if word[:7] == "<|emoji" and word[-2:] == "|>":
								words.append(self.emoji['emoji_inv'][word]							)
						elif word == "<SP>":
								words.append(' '							)
						elif word == "<BR>":
								words.append(UpperCamelCase							)
						elif word == "<TAB>":
								words.append('\t'							)
						elif word == "<BLOCK>":
								words.append('▀'							)
						elif word == "<KIGOU>":
								words.append('ǀ'							)
						elif word == "<U2000U2BFF>":
								words.append('‖'							)
						else:
								words.append(UpperCamelCase							)
				if len(UpperCamelCase							) > 0:
						words.append(bytearray(UpperCamelCase							).decode('utf-8'      ,				errors='replace'							)							)
				_snake_case   :				List[Any]														=	''.join(UpperCamelCase							)
				return text











import unittest
from pathlib import Path
from tempfile import TemporaryDirectory

from transformers import AutoConfig, TFGPTaLMHeadModel, is_keras_nlp_available, is_tf_available
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import require_keras_nlp, require_tf, slow


if is_tf_available():
      import tensorflow as tf

if is_keras_nlp_available():
      from transformers.models.gpta import TFGPTaTokenizer


lowerCAmelCase_						   =    ["""gpt2"""]
lowerCAmelCase_						   =    """gpt2"""

if is_tf_available():



      class        _lowerCAmelCase    (			tf.Module							):


        '''simple docstring'''







        def __init__(      self      :					List[str]      ,				UpperCamelCase      :					Dict							):


          '''simple docstring'''





          super().__init__()
          _snake_case   :				Optional[int]														=	tokenizer
          _snake_case   :				Union[str, Any]														=	AutoConfig.from_pretrained(UpperCamelCase							)
          _snake_case   :				int														=	TFGPTaLMHeadModel.from_config(UpperCamelCase							)







        @tf.function(input_signature=(tf.TensorSpec((None,)      ,				tf.string      ,				name='text'							),)							)
        def UpperCamelCase_ (      self      :					Optional[int]      ,				UpperCamelCase      :					Optional[int]							):


          '''simple docstring'''





          _snake_case   :				Dict														=	self.tokenizer(UpperCamelCase							)
          _snake_case   :				Union[str, Any]														=	tokenized['input_ids'].to_tensor()

          _snake_case   :				Any														=	tf.cast(input_ids_dense > 0      ,				tf.intaa							)
          # input_mask = tf.reshape(input_mask, [-1, MAX_SEQ_LEN])

          _snake_case   :				Tuple														=	self.model(input_ids=UpperCamelCase      ,				attention_mask=UpperCamelCase							)['logits']

          return outputs



@require_tf
@require_keras_nlp
class        _lowerCAmelCase    (			unittest.TestCase							):


  '''simple docstring'''







  def UpperCamelCase_ (      self      :					Any							):


    '''simple docstring'''





    super().setUp()

    _snake_case   :				Optional[int]														=	[GPTaTokenizer.from_pretrained(UpperCamelCase							) for checkpoint in (TOKENIZER_CHECKPOINTS)]
    _snake_case   :				Tuple														=	[TFGPTaTokenizer.from_pretrained(UpperCamelCase							) for checkpoint in TOKENIZER_CHECKPOINTS]
    assert len(self.tokenizers							) == len(self.tf_tokenizers							)

    _snake_case   :				Any														=	[
        'This is a straightforward English test sentence.',
        'This one has some weird characters\rto\nsee\r\nif  those\u00E9break things.',
        'Now we\'re going to add some Chinese: 一 二 三 一二三',
        'And some much more rare Chinese: 齉 堃 齉堃',
        'Je vais aussi écrire en français pour tester les accents',
        'Classical Irish also has some unusual characters, so in they go: Gaelaċ, ꝼ',
    ]
    _snake_case   :				Tuple														=	list(zip(self.test_sentences      ,				self.test_sentences[::-1]							)							)







  def UpperCamelCase_ (      self      :					Union[str, Any]							):


    '''simple docstring'''





    for tokenizer, tf_tokenizer in zip(self.tokenizers      ,				self.tf_tokenizers							):
      for test_inputs in self.test_sentences:
        _snake_case   :				Optional[int]														=	tokenizer([test_inputs]      ,				return_tensors='tf'							)
        _snake_case   :				Tuple														=	tf_tokenizer([test_inputs]							)

        for key in python_outputs.keys():
          # convert them to numpy to avoid messing with ragged tensors
          _snake_case   :				Dict														=	python_outputs[key].numpy()
          _snake_case   :				Optional[Any]														=	tf_outputs[key].numpy()

          self.assertTrue(tf.reduce_all(python_outputs_values.shape == tf_outputs_values.shape							)							)
          self.assertTrue(tf.reduce_all(tf.cast(UpperCamelCase      ,				tf.intaa							) == tf_outputs_values							)							)







  @slow
  def UpperCamelCase_ (      self      :					Optional[Any]							):


    '''simple docstring'''





    for tf_tokenizer in self.tf_tokenizers:
      _snake_case   :				str														=	tf.function(UpperCamelCase							)
      for test_inputs in self.test_sentences:
        _snake_case   :				int														=	tf.constant(UpperCamelCase							)
        _snake_case   :				Tuple														=	compiled_tokenizer(UpperCamelCase							)
        _snake_case   :				int														=	tf_tokenizer(UpperCamelCase							)

        for key in eager_outputs.keys():
          self.assertTrue(tf.reduce_all(eager_outputs[key] == compiled_outputs[key]							)							)







  @slow
  def UpperCamelCase_ (      self      :					Tuple							):


    '''simple docstring'''





    for tf_tokenizer in self.tf_tokenizers:
      _snake_case   :				Union[str, Any]														=	ModelToSave(tokenizer=UpperCamelCase							)
      _snake_case   :				Optional[Any]														=	tf.convert_to_tensor([self.test_sentences[0]]							)
      _snake_case   :				Tuple														=	model.serving(UpperCamelCase							)  # Build model with some sample inputs
      with TemporaryDirectory() as tempdir:
        _snake_case   :				str														=	Path(UpperCamelCase							) / 'saved.model'
        tf.saved_model.save(UpperCamelCase      ,				UpperCamelCase      ,				signatures={'serving_default': model.serving}							)
        _snake_case   :				Optional[int]														=	tf.saved_model.load(UpperCamelCase							)
      _snake_case   :				List[str]														=	loaded_model.signatures['serving_default'](UpperCamelCase							)['output_0']
      # We may see small differences because the loaded model is compiled, so we need an epsilon for the test
      self.assertTrue(tf.reduce_all(out == loaded_output							)							)







  @slow
  def UpperCamelCase_ (      self      :					Optional[int]							):


    '''simple docstring'''





    for tf_tokenizer in self.tf_tokenizers:
      _snake_case   :				Optional[Any]														=	tf.convert_to_tensor([self.test_sentences[0]]							)
      _snake_case   :				Any														=	tf_tokenizer(UpperCamelCase							)  # Build model with some sample inputs

      _snake_case   :				Optional[Any]														=	tf_tokenizer.get_config()
      _snake_case   :				Tuple														=	TFGPTaTokenizer.from_config(UpperCamelCase							)
      _snake_case   :				Optional[Any]														=	model_from_config(UpperCamelCase							)

      for key in from_config_output.keys():
        self.assertTrue(tf.reduce_all(from_config_output[key] == out[key]							)							)







  @slow
  def UpperCamelCase_ (      self      :					Any							):


    '''simple docstring'''





    for tf_tokenizer in self.tf_tokenizers:
      # for the test to run
      _snake_case   :				Union[str, Any]														=	12_31_23

      for max_length in [3, 5, 10_24]:
        _snake_case   :				Union[str, Any]														=	tf.convert_to_tensor([self.test_sentences[0]]							)
        _snake_case   :				List[str]														=	tf_tokenizer(UpperCamelCase      ,				max_length=UpperCamelCase							)

        _snake_case   :				int														=	out['input_ids'].numpy().shape[1]

        assert out_length == max_length




