





import unittest

from transformers import BertGenerationTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


_lowerCamelCase			       =      '▁'

_lowerCamelCase			       =      get_tests_dir('fixtures/test_sentencepiece.model')







@require_sentencepiece
class 			__A       (      lowerCamelCase__ ,unittest.TestCase  ):




      """simple docstring"""







      UpperCAmelCase__						=      BertGenerationTokenizer
      UpperCAmelCase__						=      False
      UpperCAmelCase__						=      True


      def 				__snake_case			(		self):



             """simple docstring"""





             super().setUp()

             _lowerCamelCase							:     Dict	=		BertGenerationTokenizer(a__      ,							keep_accents=a__)
             tokenizer.save_pretrained(self.tmpdirname)


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Union[str, Any]	=		'''<s>'''
             _lowerCamelCase							:     Union[str, Any]	=		1

             self.assertEqual(self.get_tokenizer()._convert_token_to_id(a__)      ,							a__)
             self.assertEqual(self.get_tokenizer()._convert_id_to_token(a__)      ,							a__)


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     List[str]	=		list(self.get_tokenizer().get_vocab().keys())

             self.assertEqual(vocab_keys[0]      ,							'''<unk>''')
             self.assertEqual(vocab_keys[1]      ,							'''<s>''')
             self.assertEqual(vocab_keys[-1]      ,							'''<pad>''')
             self.assertEqual(len(a__)      ,							1002)


      def 				__snake_case			(		self):



             """simple docstring"""





             self.assertEqual(self.get_tokenizer().vocab_size      ,							1000)


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     int	=		BertGenerationTokenizer(a__      ,							keep_accents=a__)

             _lowerCamelCase							:     Union[str, Any]	=		tokenizer.tokenize('''This is a test''')
             self.assertListEqual(a__      ,							['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est'''])

             self.assertListEqual(
                 tokenizer.convert_tokens_to_ids(a__)      ,							[285, 46, 10, 170, 382]      ,							)

             _lowerCamelCase							:     Optional[Any]	=		tokenizer.tokenize('''I was born in 92000, and this is falsé.''')
             self.assertListEqual(
                 a__      ,							[
                     SPIECE_UNDERLINE + '''I''',
                     SPIECE_UNDERLINE + '''was''',
                     SPIECE_UNDERLINE + '''b''',
                     '''or''',
                     '''n''',
                     SPIECE_UNDERLINE + '''in''',
                     SPIECE_UNDERLINE + '''''',
                     '''9''',
                     '''2''',
                     '''0''',
                     '''0''',
                     '''0''',
                     ''',''',
                     SPIECE_UNDERLINE + '''and''',
                     SPIECE_UNDERLINE + '''this''',
                     SPIECE_UNDERLINE + '''is''',
                     SPIECE_UNDERLINE + '''f''',
                     '''al''',
                     '''s''',
                     '''é''',
                     '''.''',
                 ]      ,							)
             _lowerCamelCase							:     Union[str, Any]	=		tokenizer.convert_tokens_to_ids(a__)
             self.assertListEqual(
                 a__      ,							[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]      ,							)

             _lowerCamelCase							:     Optional[Any]	=		tokenizer.convert_ids_to_tokens(a__)
             self.assertListEqual(
                 a__      ,							[
                     SPIECE_UNDERLINE + '''I''',
                     SPIECE_UNDERLINE + '''was''',
                     SPIECE_UNDERLINE + '''b''',
                     '''or''',
                     '''n''',
                     SPIECE_UNDERLINE + '''in''',
                     SPIECE_UNDERLINE + '''''',
                     '''<unk>''',
                     '''2''',
                     '''0''',
                     '''0''',
                     '''0''',
                     ''',''',
                     SPIECE_UNDERLINE + '''and''',
                     SPIECE_UNDERLINE + '''this''',
                     SPIECE_UNDERLINE + '''is''',
                     SPIECE_UNDERLINE + '''f''',
                     '''al''',
                     '''s''',
                     '''<unk>''',
                     '''.''',
                 ]      ,							)


      @cached_property
      def 				__snake_case			(		self):



             """simple docstring"""





             return BertGenerationTokenizer.from_pretrained('''google/bert_for_seq_generation_L-24_bbc_encoder''')


      @slow
      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Optional[int]	=		'''Hello World!'''
             _lowerCamelCase							:     Dict	=		[1_8536, 2260, 101]

             self.assertListEqual(a__      ,							self.big_tokenizer.encode(a__))


      @slow
      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Optional[Any]	=		(
                 '''This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'''
                 ''' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'''
             )
             _lowerCamelCase							:     Any	=		[
                 871,
                 419,
                 358,
                 946,
                 991,
                 2521,
                 452,
                 358,
                 1357,
                 387,
                 7751,
                 3536,
                 112,
                 985,
                 456,
                 126,
                 865,
                 938,
                 5400,
                 5734,
                 458,
                 1368,
                 467,
                 786,
                 2462,
                 5246,
                 1159,
                 633,
                 865,
                 4519,
                 457,
                 582,
                 852,
                 2557,
                 427,
                 916,
                 508,
                 405,
                 3_4324,
                 497,
                 391,
                 408,
                 1_1342,
                 1244,
                 385,
                 100,
                 938,
                 985,
                 456,
                 574,
                 362,
                 1_2597,
                 3200,
                 3129,
                 1172,
             ]

             self.assertListEqual(a__      ,							self.big_tokenizer.encode(a__))


      @require_torch
      @slow
      def 				__snake_case			(		self):



             """simple docstring"""





             import torch

             from transformers import BertGenerationConfig, BertGenerationEncoder

             # Build sequence
             _lowerCamelCase							:     int	=		list(self.big_tokenizer.get_vocab().keys())[:10]
             _lowerCamelCase							:     Optional[Any]	=		''' '''.join(a__)
             _lowerCamelCase							:     Tuple	=		self.big_tokenizer.encode_plus(a__      ,							return_tensors='''pt'''      ,							return_token_type_ids=a__)
             _lowerCamelCase							:     Optional[int]	=		self.big_tokenizer.batch_encode_plus(
                 [sequence + ''' ''' + sequence]      ,							return_tensors='''pt'''      ,							return_token_type_ids=a__)

             _lowerCamelCase							:     Union[str, Any]	=		BertGenerationConfig()
             _lowerCamelCase							:     Tuple	=		BertGenerationEncoder(a__)

             assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

             with torch.no_grad():
                    model(**a__)
                    model(**a__)



      @slow
      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Any	=		{'''input_ids''': [[3_9286, 458, 3_6335, 2001, 456, 1_3073, 1_3266, 455, 113, 7746, 1741, 1_1157, 391, 1_3073, 1_3266, 455, 113, 3967, 3_5412, 113, 4936, 109, 3870, 2377, 113, 3_0084, 4_5720, 458, 134, 1_7496, 112, 503, 1_1672, 113, 118, 112, 5665, 1_3347, 3_8687, 112, 1496, 3_1389, 112, 3268, 4_7264, 134, 962, 112, 1_6377, 8035, 2_3130, 430, 1_2169, 1_5518, 2_8592, 458, 146, 4_1697, 109, 391, 1_2169, 1_5518, 1_6689, 458, 146, 4_1358, 109, 452, 726, 4034, 111, 763, 3_5412, 5082, 388, 1903, 111, 9051, 391, 2870, 4_8918, 1900, 1123, 550, 998, 112, 9586, 1_5985, 455, 391, 410, 2_2955, 3_7636, 114], [448, 1_7496, 419, 3663, 385, 763, 113, 2_7533, 2870, 3283, 1_3043, 1639, 2_4713, 523, 656, 2_4013, 1_8550, 2521, 517, 2_7014, 2_1244, 420, 1212, 1465, 391, 927, 4833, 388, 578, 1_1786, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [484, 2169, 7687, 2_1932, 1_8146, 726, 363, 1_7032, 3391, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
             # fmt: on

             self.tokenizer_integration_test_util(
                 expected_encoding=a__      ,							model_name='''google/bert_for_seq_generation_L-24_bbc_encoder'''      ,							revision='''c817d1fd1be2ffa69431227a1fe320544943d4db'''      ,							)








_lowerCamelCase			       =      [4, 1, 7, 4, 2, 6, 4, 1, 5, 3, 7, 5]
_lowerCamelCase			       =      [3, 7, 7, 4, 2, 6, 4, 1, 5, 3, 7, 5]
_lowerCamelCase			       =      {
    0: 'Sunday',
    1: 'Monday',
    2: 'Tuesday',
    3: 'Wednesday',
    4: 'Thursday',
    5: 'Friday',
    6: 'Saturday',
}




def    __UpperCAmelCase(      lowercase_ ,    lowercase_ ,    lowercase_      ):
       assert len(str(lowercase_      )      ) > 2, "year should be in YYYY format"
       assert 1 <= month <= 12, "month should be between 1 to 12"
       assert 1 <= day <= 31, "day should be between 1 to 31"

       # Doomsday algorithm:
       _lowerCamelCase							:     List[Any]	=		year // 1_00
       _lowerCamelCase							:     Dict	=		(5 * (century % 4) + 2) % 7
       _lowerCamelCase							:     Tuple	=		year % 1_00
       _lowerCamelCase							:     int	=		centurian % 12
       _lowerCamelCase							:     List[Any]	=		(
           (centurian // 12) + centurian_m + (centurian_m // 4) + century_anchor
       ) % 7
       _lowerCamelCase							:     List[Any]	=		(
           DOOMSDAY_NOT_LEAP[month - 1]
           if (year % 4 != 0) or (centurian == 0 and (year % 4_00) == 0)
           else DOOMSDAY_LEAP[month - 1]
       )
       _lowerCamelCase							:     Any	=		(dooms_day + day - day_anchor) % 7
       return WEEK_DAY_NAMES[week_day]


if __name__ == "__main__":
       import doctest

       doctest.testmod()

