"""simple docstring"""


import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
 from test_module.custom_tokenization_fast import CustomTokenizerFast






class    a__    (   unittest.TestCase	):


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        lowercase__				 =							mock.Mock()
        lowercase__				 =							500
        lowercase__				 =							{}
        lowercase__				 =							HTTPError
        lowercase__				 =							{}

        # Download this model to make sure it's in the cache.
        lowercase__				 =							BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"  )

        # Under the mock environment we get a 500 error when trying to reach the tokenizer.
        with mock.patch("requests.Session.request",				return_value=_UpperCAmelCase  ) as mock_head:
             lowercase__				 =							BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"  )
             # This check we did call the fake head request
             mock_head.assert_called()


   @require_tokenizers
   def 	snake_case__	(     self  ):
        '''simple docstring'''



        lowercase__				 =							mock.Mock()
        lowercase__				 =							500
        lowercase__				 =							{}
        lowercase__				 =							HTTPError
        lowercase__				 =							{}

        # Download this model to make sure it's in the cache.
        lowercase__				 =							GPTaTokenizerFast.from_pretrained("gpt2"  )

        # Under the mock environment we get a 500 error when trying to reach the tokenizer.
        with mock.patch("requests.Session.request",				return_value=_UpperCAmelCase  ) as mock_head:
             lowercase__				 =							GPTaTokenizerFast.from_pretrained("gpt2"  )
             # This check we did call the fake head request
             mock_head.assert_called()


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        try:
             lowercase__				 =							tempfile.mktemp()
             with open(_UpperCAmelCase,				"wb"  ) as f:
                  http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model",				_UpperCAmelCase  )

             lowercase__				 =							AlbertTokenizer.from_pretrained(_UpperCAmelCase  )
        finally:
             os.remove(_UpperCAmelCase  )

        # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
        # the current folder and have the right name.
        if os.path.isfile("tokenizer.json"  ):
             # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
             return
        try:
             with open("tokenizer.json",				"wb"  ) as f:
                  http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json",				_UpperCAmelCase  )
             lowercase__				 =							AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2"  )
             # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
             self.assertEqual(tokenizer.vocab_size,				1000  )
             # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

        finally:
             os.remove("tokenizer.json"  )


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        lowercase__				 =							AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"  )






@is_staging_test
class    a__    (   unittest.TestCase	):
   snake_case_							=      ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]


   @classmethod
   def 	snake_case__	(     cls  ):
        '''simple docstring'''



        lowercase__				 =							TOKEN
        HfFolder.save_token(_UpperCAmelCase  )


   @classmethod
   def 	snake_case__	(     cls  ):
        '''simple docstring'''



        try:
             delete_repo(token=cls._token,				repo_id="test-tokenizer"  )
        except HTTPError:
             pass

        try:
             delete_repo(token=cls._token,				repo_id="valid_org/test-tokenizer-org"  )
        except HTTPError:
             pass

        try:
             delete_repo(token=cls._token,				repo_id="test-dynamic-tokenizer"  )
        except HTTPError:
             pass


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        with tempfile.TemporaryDirectory() as tmp_dir:
             lowercase__				 =							os.path.join(_UpperCAmelCase,				"vocab.txt"  )
             with open(_UpperCAmelCase,				"w",				encoding="utf-8"  ) as vocab_writer:
                  vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]  )  )
             lowercase__				 =							BertTokenizer(_UpperCAmelCase  )

        tokenizer.push_to_hub("test-tokenizer",				use_auth_token=self._token  )
        lowercase__				 =							BertTokenizer.from_pretrained(F'''{USER}/test-tokenizer'''  )
        self.assertDictEqual(new_tokenizer.vocab,				tokenizer.vocab  )

        # Reset repo
        delete_repo(token=self._token,				repo_id="test-tokenizer"  )

        # Push to hub via save_pretrained
        with tempfile.TemporaryDirectory() as tmp_dir:
             tokenizer.save_pretrained(_UpperCAmelCase,				repo_id="test-tokenizer",				push_to_hub=_UpperCAmelCase,				use_auth_token=self._token  )

        lowercase__				 =							BertTokenizer.from_pretrained(F'''{USER}/test-tokenizer'''  )
        self.assertDictEqual(new_tokenizer.vocab,				tokenizer.vocab  )


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        with tempfile.TemporaryDirectory() as tmp_dir:
             lowercase__				 =							os.path.join(_UpperCAmelCase,				"vocab.txt"  )
             with open(_UpperCAmelCase,				"w",				encoding="utf-8"  ) as vocab_writer:
                  vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]  )  )
             lowercase__				 =							BertTokenizer(_UpperCAmelCase  )

        tokenizer.push_to_hub("valid_org/test-tokenizer-org",				use_auth_token=self._token  )
        lowercase__				 =							BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"  )
        self.assertDictEqual(new_tokenizer.vocab,				tokenizer.vocab  )

        # Reset repo
        delete_repo(token=self._token,				repo_id="valid_org/test-tokenizer-org"  )

        # Push to hub via save_pretrained
        with tempfile.TemporaryDirectory() as tmp_dir:
             tokenizer.save_pretrained(
                 _UpperCAmelCase,				repo_id="valid_org/test-tokenizer-org",				push_to_hub=_UpperCAmelCase,				use_auth_token=self._token  )

        lowercase__				 =							BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"  )
        self.assertDictEqual(new_tokenizer.vocab,				tokenizer.vocab  )


   @require_tokenizers
   def 	snake_case__	(     self  ):
        '''simple docstring'''



        CustomTokenizer.register_for_auto_class()
        with tempfile.TemporaryDirectory() as tmp_dir:
             lowercase__				 =							os.path.join(_UpperCAmelCase,				"vocab.txt"  )
             with open(_UpperCAmelCase,				"w",				encoding="utf-8"  ) as vocab_writer:
                  vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]  )  )
             lowercase__				 =							CustomTokenizer(_UpperCAmelCase  )

        # No fast custom tokenizer
        tokenizer.push_to_hub("test-dynamic-tokenizer",				use_auth_token=self._token  )

        lowercase__				 =							AutoTokenizer.from_pretrained(F'''{USER}/test-dynamic-tokenizer''',				trust_remote_code=_UpperCAmelCase  )
        # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
        self.assertEqual(tokenizer.__class__.__name__,				"CustomTokenizer"  )

        # Fast and slow custom tokenizer
        CustomTokenizerFast.register_for_auto_class()
        with tempfile.TemporaryDirectory() as tmp_dir:
             lowercase__				 =							os.path.join(_UpperCAmelCase,				"vocab.txt"  )
             with open(_UpperCAmelCase,				"w",				encoding="utf-8"  ) as vocab_writer:
                  vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]  )  )

             lowercase__				 =							BertTokenizerFast.from_pretrained(_UpperCAmelCase  )
             bert_tokenizer.save_pretrained(_UpperCAmelCase  )
             lowercase__				 =							CustomTokenizerFast.from_pretrained(_UpperCAmelCase  )

        tokenizer.push_to_hub("test-dynamic-tokenizer",				use_auth_token=self._token  )

        lowercase__				 =							AutoTokenizer.from_pretrained(F'''{USER}/test-dynamic-tokenizer''',				trust_remote_code=_UpperCAmelCase  )
        # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
        self.assertEqual(tokenizer.__class__.__name__,				"CustomTokenizerFast"  )
        lowercase__				 =							AutoTokenizer.from_pretrained(
            F'''{USER}/test-dynamic-tokenizer''',				use_fast=_UpperCAmelCase,				trust_remote_code=_UpperCAmelCase  )
        # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
        self.assertEqual(tokenizer.__class__.__name__,				"CustomTokenizer"  )






class    a__    (   unittest.TestCase	):


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        lowercase__				 =							Trie()
        trie.add("Hello 友達"  )
        self.assertEqual(trie.data,				{"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}  )
        trie.add("Hello"  )
        trie.data
        self.assertEqual(trie.data,				{"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}  )


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        lowercase__				 =							Trie()
        self.assertEqual(trie.split("[CLS] This is a extra_id_100"  ),				["[CLS] This is a extra_id_100"]  )
        trie.add("[CLS]"  )
        trie.add("extra_id_1"  )
        trie.add("extra_id_100"  )
        self.assertEqual(trie.split("[CLS] This is a extra_id_100"  ),				["[CLS]", " This is a ", "extra_id_100"]  )


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        lowercase__				 =							Trie()
        trie.add("A"  )
        self.assertEqual(trie.split("ABC"  ),				["A", "BC"]  )
        self.assertEqual(trie.split("BCA"  ),				["BC", "A"]  )


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        lowercase__				 =							Trie()
        trie.add("TOKEN]"  )
        trie.add("[SPECIAL_TOKEN]"  )
        self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"  ),				["This is something ", "[SPECIAL_TOKEN]"]  )


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        lowercase__				 =							Trie()
        trie.add("A"  )
        trie.add("P"  )
        trie.add("[SPECIAL_TOKEN]"  )
        self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"  ),				["This is something ", "[SPECIAL_TOKEN]"]  )


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        lowercase__				 =							Trie()
        trie.add("AB"  )
        trie.add("B"  )
        trie.add("C"  )
        self.assertEqual(trie.split("ABC"  ),				["AB", "C"]  )


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        lowercase__				 =							Trie()
        trie.add("ABC"  )
        trie.add("B"  )
        trie.add("CD"  )
        self.assertEqual(trie.split("ABCD"  ),				["ABC", "D"]  )


   def 	snake_case__	(     self  ):
        '''simple docstring'''



        lowercase__				 =							Trie()
        lowercase__				 =							trie.cut_text("ABC",				[0, 0, 2, 1, 2, 3]  )
        self.assertEqual(_UpperCAmelCase,				["AB", "C"]  )





"""simple docstring"""


from __future__ import annotations







def 					__a   (		A    ,				A      ):






     '''simple docstring'''



     if partitions <= 0:
          raise ValueError("partitions must be a positive number!"      )
     if partitions > number_of_bytes:
          raise ValueError("partitions can not > number_of_bytes!"      )
     lowercase__				 =							number_of_bytes // partitions
     lowercase__				 =							[]
     for i in range(A      ):
          lowercase__				 =							i * bytes_per_partition + 1
          lowercase__				 =							(
              number_of_bytes if i == partitions - 1 else (i + 1) * bytes_per_partition
          )
          allocation_list.append(f'''{start_bytes}-{end_bytes}'''      )
     return allocation_list


if __name__ == "__main__":
 import doctest

 doctest.testmod()




