


'''simple docstring'''




import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
							from test_module.custom_tokenization_fast import CustomTokenizerFast



class  lowerCAmelCase				(				unittest.TestCase							):



	def 				lowercase     (					self    ):
		# A mock response for an HTTP head request to emulate server down
		lowerCAmelCase					:    Optional[int]      							= mock.Mock()
		lowerCAmelCase					:    Union[str, Any]      							= 500
		lowerCAmelCase					:    str      							= {}
		lowerCAmelCase					:    List[str]      							= HTTPError
		lowerCAmelCase					:    Optional[Any]      							= {}

		# Download this model to make sure it's in the cache.
		lowerCAmelCase					:    List[str]      							= BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'    )

		# Under the mock environment we get a 500 error when trying to reach the tokenizer.
		with mock.patch('requests.Session.request' ,					return_value=a_    ) as mock_head:
			lowerCAmelCase					:    Optional[Any]      							= BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'    )
			# This check we did call the fake head request
			mock_head.assert_called()



	@require_tokenizers
	def 				lowercase     (					self    ):
		# A mock response for an HTTP head request to emulate server down
		lowerCAmelCase					:    Union[str, Any]      							= mock.Mock()
		lowerCAmelCase					:    Any      							= 500
		lowerCAmelCase					:    int      							= {}
		lowerCAmelCase					:    int      							= HTTPError
		lowerCAmelCase					:    Any      							= {}

		# Download this model to make sure it's in the cache.
		lowerCAmelCase					:    List[str]      							= GPTaTokenizerFast.from_pretrained('gpt2'    )

		# Under the mock environment we get a 500 error when trying to reach the tokenizer.
		with mock.patch('requests.Session.request' ,					return_value=a_    ) as mock_head:
			lowerCAmelCase					:    Optional[int]      							= GPTaTokenizerFast.from_pretrained('gpt2'    )
			# This check we did call the fake head request
			mock_head.assert_called()



	def 				lowercase     (					self    ):
		# This test is for deprecated behavior and can be removed in v5
		try:
			lowerCAmelCase					:    Union[str, Any]      							= tempfile.mktemp()
			with open(a_ ,					'wb'    ) as f:
				http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model' ,					a_    )

			lowerCAmelCase					:    Dict      							= AlbertTokenizer.from_pretrained(a_    )
		finally:
			os.remove(a_    )

		# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
		# the current folder and have the right name.
		if os.path.isfile('tokenizer.json'    ):
			# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
			return
		try:
			with open('tokenizer.json' ,					'wb'    ) as f:
				http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json' ,					a_    )
			lowerCAmelCase					:    Tuple      							= AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2'    )
			# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
			self.assertEqual(tokenizer.vocab_size ,					1000    )
			# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

		finally:
			os.remove('tokenizer.json'    )



	def 				lowercase     (					self    ):
		# This test is for deprecated behavior and can be removed in v5
		lowerCAmelCase					:    Tuple      							= AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'    )



@is_staging_test
class  lowerCAmelCase				(				unittest.TestCase							):
	_lowerCamelCase   :					str			      = ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']



	@classmethod
	def 				lowercase     (					cls    ):
		lowerCAmelCase					:    Tuple      							= TOKEN
		HfFolder.save_token(a_    )



	@classmethod
	def 				lowercase     (					cls    ):
		try:
			delete_repo(token=cls._token ,					repo_id='test-tokenizer'    )
		except HTTPError:
			pass

		try:
			delete_repo(token=cls._token ,					repo_id='valid_org/test-tokenizer-org'    )
		except HTTPError:
			pass

		try:
			delete_repo(token=cls._token ,					repo_id='test-dynamic-tokenizer'    )
		except HTTPError:
			pass



	def 				lowercase     (					self    ):
		with tempfile.TemporaryDirectory() as tmp_dir:
			lowerCAmelCase					:    int      							= os.path.join(a_ ,					'vocab.txt'    )
			with open(a_ ,					'w' ,					encoding='utf-8'    ) as vocab_writer:
				vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]    )    )
			lowerCAmelCase					:    str      							= BertTokenizer(a_    )

		tokenizer.push_to_hub('test-tokenizer' ,					use_auth_token=self._token    )
		lowerCAmelCase					:    List[Any]      							= BertTokenizer.from_pretrained(f"{USER}/test-tokenizer"    )
		self.assertDictEqual(new_tokenizer.vocab ,					tokenizer.vocab    )

		# Reset repo
		delete_repo(token=self._token ,					repo_id='test-tokenizer'    )

		# Push to hub via save_pretrained
		with tempfile.TemporaryDirectory() as tmp_dir:
			tokenizer.save_pretrained(a_ ,					repo_id='test-tokenizer' ,					push_to_hub=a_ ,					use_auth_token=self._token    )

		lowerCAmelCase					:    str      							= BertTokenizer.from_pretrained(f"{USER}/test-tokenizer"    )
		self.assertDictEqual(new_tokenizer.vocab ,					tokenizer.vocab    )



	def 				lowercase     (					self    ):
		with tempfile.TemporaryDirectory() as tmp_dir:
			lowerCAmelCase					:    List[Any]      							= os.path.join(a_ ,					'vocab.txt'    )
			with open(a_ ,					'w' ,					encoding='utf-8'    ) as vocab_writer:
				vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]    )    )
			lowerCAmelCase					:    Any      							= BertTokenizer(a_    )

		tokenizer.push_to_hub('valid_org/test-tokenizer-org' ,					use_auth_token=self._token    )
		lowerCAmelCase					:    Dict      							= BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'    )
		self.assertDictEqual(new_tokenizer.vocab ,					tokenizer.vocab    )

		# Reset repo
		delete_repo(token=self._token ,					repo_id='valid_org/test-tokenizer-org'    )

		# Push to hub via save_pretrained
		with tempfile.TemporaryDirectory() as tmp_dir:
			tokenizer.save_pretrained(
			    a_ ,					repo_id='valid_org/test-tokenizer-org' ,					push_to_hub=a_ ,					use_auth_token=self._token    )

		lowerCAmelCase					:    int      							= BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'    )
		self.assertDictEqual(new_tokenizer.vocab ,					tokenizer.vocab    )



	@require_tokenizers
	def 				lowercase     (					self    ):
		CustomTokenizer.register_for_auto_class()
		with tempfile.TemporaryDirectory() as tmp_dir:
			lowerCAmelCase					:    Optional[int]      							= os.path.join(a_ ,					'vocab.txt'    )
			with open(a_ ,					'w' ,					encoding='utf-8'    ) as vocab_writer:
				vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]    )    )
			lowerCAmelCase					:    Tuple      							= CustomTokenizer(a_    )

		# No fast custom tokenizer
		tokenizer.push_to_hub('test-dynamic-tokenizer' ,					use_auth_token=self._token    )

		lowerCAmelCase					:    int      							= AutoTokenizer.from_pretrained(f"{USER}/test-dynamic-tokenizer" ,					trust_remote_code=a_    )
		# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
		self.assertEqual(tokenizer.__class__.__name__ ,					'CustomTokenizer'    )

		# Fast and slow custom tokenizer
		CustomTokenizerFast.register_for_auto_class()
		with tempfile.TemporaryDirectory() as tmp_dir:
			lowerCAmelCase					:    Optional[Any]      							= os.path.join(a_ ,					'vocab.txt'    )
			with open(a_ ,					'w' ,					encoding='utf-8'    ) as vocab_writer:
				vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]    )    )

			lowerCAmelCase					:    List[Any]      							= BertTokenizerFast.from_pretrained(a_    )
			bert_tokenizer.save_pretrained(a_    )
			lowerCAmelCase					:    List[str]      							= CustomTokenizerFast.from_pretrained(a_    )

		tokenizer.push_to_hub('test-dynamic-tokenizer' ,					use_auth_token=self._token    )

		lowerCAmelCase					:    Dict      							= AutoTokenizer.from_pretrained(f"{USER}/test-dynamic-tokenizer" ,					trust_remote_code=a_    )
		# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
		self.assertEqual(tokenizer.__class__.__name__ ,					'CustomTokenizerFast'    )
		lowerCAmelCase					:    Dict      							= AutoTokenizer.from_pretrained(
		    f"{USER}/test-dynamic-tokenizer" ,					use_fast=a_ ,					trust_remote_code=a_    )
		# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
		self.assertEqual(tokenizer.__class__.__name__ ,					'CustomTokenizer'    )



class  lowerCAmelCase				(				unittest.TestCase							):



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    Optional[Any]      							= Trie()
		trie.add('Hello 友達'    )
		self.assertEqual(trie.data ,					{'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}}    )
		trie.add('Hello'    )
		trie.data
		self.assertEqual(trie.data ,					{'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}}    )



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    int      							= Trie()
		self.assertEqual(trie.split('[CLS] This is a extra_id_100'    ) ,					['[CLS] This is a extra_id_100']    )
		trie.add('[CLS]'    )
		trie.add('extra_id_1'    )
		trie.add('extra_id_100'    )
		self.assertEqual(trie.split('[CLS] This is a extra_id_100'    ) ,					['[CLS]', ' This is a ', 'extra_id_100']    )



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    Any      							= Trie()
		trie.add('A'    )
		self.assertEqual(trie.split('ABC'    ) ,					['A', 'BC']    )
		self.assertEqual(trie.split('BCA'    ) ,					['BC', 'A']    )



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    Tuple      							= Trie()
		trie.add('TOKEN]'    )
		trie.add('[SPECIAL_TOKEN]'    )
		self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'    ) ,					['This is something ', '[SPECIAL_TOKEN]']    )



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    List[str]      							= Trie()
		trie.add('A'    )
		trie.add('P'    )
		trie.add('[SPECIAL_TOKEN]'    )
		self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'    ) ,					['This is something ', '[SPECIAL_TOKEN]']    )



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    Optional[Any]      							= Trie()
		trie.add('AB'    )
		trie.add('B'    )
		trie.add('C'    )
		self.assertEqual(trie.split('ABC'    ) ,					['AB', 'C']    )



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    str      							= Trie()
		trie.add('ABC'    )
		trie.add('B'    )
		trie.add('CD'    )
		self.assertEqual(trie.split('ABCD'    ) ,					['ABC', 'D']    )



	def 				lowercase     (					self    ):
		# Even if the offsets are wrong, we necessarily output correct string
		# parts.
		lowerCAmelCase					:    Dict      							= Trie()
		lowerCAmelCase					:    str      							= trie.cut_text('ABC' ,					[0, 0, 2, 1, 2, 3]    )
		self.assertEqual(a_ ,					['AB', 'C']    )




'''simple docstring'''




def 			__UpperCamelCase    (    _A    :				List[str]				)    ->      Optional[Any]:




	"""simple docstring"""





	if not head:
		return True
	# split the list to two parts
	lowerCAmelCase							,   lowerCAmelCase					:    str      							= head.next, head
	while fast and fast.next:
		lowerCAmelCase					:    Optional[int]      							= fast.next.next
		lowerCAmelCase					:    int      							= slow.next
	lowerCAmelCase					:    int      							= slow.next
	lowerCAmelCase					:    Optional[Any]      							= None  # Don't forget here! But forget still works!
	# reverse the second part
	lowerCAmelCase					:    List[Any]      							= None
	while second:
		lowerCAmelCase					:    List[Any]      							= second.next
		lowerCAmelCase					:    Union[str, Any]      							= node
		lowerCAmelCase					:    Optional[Any]      							= second
		lowerCAmelCase					:    Any      							= nxt
	# compare two parts
	# second part has the same or one less node
	while node:
		if node.val != head.val:
			return False
		lowerCAmelCase					:    Optional[Any]      							= node.next
		lowerCAmelCase					:    Tuple      							= head.next
	return True

def 			__UpperCamelCase    (    _A    :				Optional[Any]				)    ->      Optional[int]:




	"""simple docstring"""





	if not head or not head.next:
		return True

	# 1. Get the midpoint (slow)
	lowerCAmelCase					:    Optional[int]      							= head
	while fast and fast.next:
		lowerCAmelCase							,   lowerCAmelCase					:    Optional[Any]      							= fast.next.next, slow.next

	# 2. Push the second half into the stack
	lowerCAmelCase					:    Tuple      							= [slow.val]
	while slow.next:
		lowerCAmelCase					:    Tuple      							= slow.next
		stack.append(slow.val				)

	# 3. Comparison
	while stack:
		if stack.pop() != cur.val:
			return False
		lowerCAmelCase					:    Union[str, Any]      							= cur.next

	return True






def 			__UpperCamelCase    (    _A    :				Tuple				)    ->      Optional[int]:




	"""simple docstring"""





	if not head or not head.next:
		return True
	lowerCAmelCase					:    Optional[int]      							= {}
	lowerCAmelCase					:    int      							= 0
	while head:
		if head.val in d:
			d[head.val].append(_A				)
		else:
			lowerCAmelCase					:    Any      							= [pos]
		lowerCAmelCase					:    int      							= head.next
		pos += 1
	lowerCAmelCase					:    str      							= pos - 1
	lowerCAmelCase					:    Optional[Any]      							= 0
	for v in d.values():
		if len(_A				) % 2 != 0:
			middle += 1
		else:
			lowerCAmelCase					:    Any      							= 0
			for i in range(0					,   len(_A				)				):
				if v[i] + v[len(_A				) - 1 - step] != checksum:
					return False
				step += 1
		if middle > 1:
			return False
	return True
