


"""simple docstring"""





import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


snake_case__		:     int    				=				get_tests_dir('''fixtures/test_sentencepiece.model''')



@require_sentencepiece
class 		snake_case_(       a__							,					unittest.TestCase			):
     __UpperCamelCase  =					XLMProphetNetTokenizer
     __UpperCamelCase  =					False
     __UpperCamelCase  =					True






     def      lowerCamelCase__     (	self  :					Union[str, Any]				):
           super().setUp()

           # We have a SentencePiece fixture for testing
           lowerCAmelCase					:	str       =   XLMProphetNetTokenizer(UpperCamelCase_					,				keep_accents=UpperCamelCase_				)
           tokenizer.save_pretrained(self.tmpdirname				)






     def      lowerCamelCase__     (	self  :					Union[str, Any]				):
           lowerCAmelCase					:	Any       =   '''[PAD]'''
           lowerCAmelCase					:	List[Any]       =   0

           self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase_				)					,				UpperCamelCase_				)
           self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase_				)					,				UpperCamelCase_				)






     def      lowerCamelCase__     (	self  :					Any				):
           lowerCAmelCase					:	List[str]       =   list(self.get_tokenizer().get_vocab().keys()				)

           self.assertEqual(vocab_keys[0]					,				'''[PAD]'''				)
           self.assertEqual(vocab_keys[1]					,				'''[CLS]'''				)
           self.assertEqual(vocab_keys[-1]					,				'''j'''				)
           self.assertEqual(len(UpperCamelCase_				)					,				1_0_1_2				)






     def      lowerCamelCase__     (	self  :					Union[str, Any]				):
           self.assertEqual(self.get_tokenizer().vocab_size					,				1_0_1_2				)






     def      lowerCamelCase__     (	self  :					List[Any]				):
           lowerCAmelCase					:	str       =   XLMProphetNetTokenizer(UpperCamelCase_					,				keep_accents=UpperCamelCase_				)

           lowerCAmelCase					:	str       =   tokenizer.tokenize('''This is a test'''				)
           self.assertListEqual(UpperCamelCase_					,				['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']				)

           self.assertListEqual(
               tokenizer.convert_tokens_to_ids(UpperCamelCase_				)					,				[value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]					,				)

           lowerCAmelCase					:	Any       =   tokenizer.tokenize('''I was born in 92000, and this is falsé.'''				)
           self.assertListEqual(
               UpperCamelCase_					,				[
                   SPIECE_UNDERLINE + '''I''',
                   SPIECE_UNDERLINE + '''was''',
                   SPIECE_UNDERLINE + '''b''',
                   '''or''',
                   '''n''',
                   SPIECE_UNDERLINE + '''in''',
                   SPIECE_UNDERLINE + '''''',
                   '''9''',
                   '''2''',
                   '''0''',
                   '''0''',
                   '''0''',
                   ''',''',
                   SPIECE_UNDERLINE + '''and''',
                   SPIECE_UNDERLINE + '''this''',
                   SPIECE_UNDERLINE + '''is''',
                   SPIECE_UNDERLINE + '''f''',
                   '''al''',
                   '''s''',
                   '''é''',
                   '''.''',
               ]					,				)
           lowerCAmelCase					:	Tuple       =   tokenizer.convert_tokens_to_ids(UpperCamelCase_				)
           self.assertListEqual(
               UpperCamelCase_					,				[
                   value + tokenizer.fairseq_offset
                   for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, -9, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, -9, 4]
               ]					,				)

           lowerCAmelCase					:	Any       =   tokenizer.convert_ids_to_tokens(UpperCamelCase_				)
           self.assertListEqual(
               UpperCamelCase_					,				[
                   SPIECE_UNDERLINE + '''I''',
                   SPIECE_UNDERLINE + '''was''',
                   SPIECE_UNDERLINE + '''b''',
                   '''or''',
                   '''n''',
                   SPIECE_UNDERLINE + '''in''',
                   SPIECE_UNDERLINE + '''''',
                   '''[UNK]''',
                   '''2''',
                   '''0''',
                   '''0''',
                   '''0''',
                   ''',''',
                   SPIECE_UNDERLINE + '''and''',
                   SPIECE_UNDERLINE + '''this''',
                   SPIECE_UNDERLINE + '''is''',
                   SPIECE_UNDERLINE + '''f''',
                   '''al''',
                   '''s''',
                   '''[UNK]''',
                   '''.''',
               ]					,				)






     @cached_property
     def      lowerCamelCase__     (	self  :					Tuple				):
           return XLMProphetNetTokenizer.from_pretrained('''microsoft/xprophetnet-large-wiki100-cased'''				)






     @slow
     def      lowerCamelCase__     (	self  :					Optional[Any]				):
           lowerCAmelCase					:	Union[str, Any]       =   '''Hello World!'''
           lowerCAmelCase					:	Any       =   [3_5_3_8_9, 6_6_7_2, 4_9, 2]
           self.assertListEqual(UpperCamelCase_					,				self.big_tokenizer.encode(UpperCamelCase_				)				)






     @slow
     def      lowerCamelCase__     (	self  :					Union[str, Any]				):
           # fmt: off
           lowerCAmelCase					:	Dict       =   {'''input_ids''': [[1_1_0_7_3, 8_2_7_8_3, 1_8, 2_6, 8_2_7_8_3, 5_4_9, 5_1_5_4_0, 2_4_8, 1_7_2_0_9, 1_3_0_1, 2_1_7, 2_0, 2_1_5_1_8_6, 1_3_2_5, 1_4_7, 1_7_2_0_9, 1_3_0_1, 2_1_7, 2_0, 5_6_3_7_0, 5_3, 1_2_2_0_2_0, 2_0, 1_6_4_7_7, 2_7, 8_7_3_5_5, 4_5_4_8, 2_0, 4_7_2_8, 7_8_3_9_2, 1_7, 1_5_9_9_6_9, 1_8, 2_6, 2_4_4_9_1, 6_2_9, 1_5, 5_3_8, 2_2_7_0_4, 5_4_3_9, 1_5, 2_7_8_8, 2_4_4_9_1, 9_8_8_5, 1_5, 4_3_5_3_4, 6_0_5, 1_5, 8_1_4, 1_8_4_0_3, 3_3_2_0_0, 2_9, 1_5, 4_3_5_3_4, 2_4_4_5_8, 1_2_4_1_0, 1_1_1, 2_4_9_6_6, 8_3_6_6_9, 9_6_3_7, 1_4_4_0_6_8, 2_6, 8_5_0, 2_2_3_4_6, 2_7, 1_4_7, 2_4_9_6_6, 8_3_6_6_9, 8_3_4_9_0, 2_6, 3_9_1_1_3, 7_3_5, 2_7, 6_8_9, 6_5_6, 2_8_0_0, 1_3_3_9, 4_6_0_0, 5_3, 1_2_2_0_2_0, 1_1_5_7_8_5, 3_4, 8_1_6, 1_3_3_9, 4_6_8_8_7, 1_8, 1_4_7, 5_3_9_0_5, 1_9_5_1, 4_2_2_3_8, 4_1_1_7_0, 1_7_7_3_2, 8_3_4, 4_3_6, 1_5, 2_7_5_2_3, 9_8_7_3_3, 2_1_7, 1_4_7, 5_5_4_2, 4_9_8_1, 9_3_0, 1_7_3_4_7, 1_6, 2], [2_0_0_9_1, 6_2_9, 9_4, 8_2_7_8_6, 5_8, 4_9_0, 2_0, 1_5_2_8, 8_4, 5_3_9_0_5, 3_4_4, 8_0_5_9_2, 1_1_0_1_2_8, 1_8_8_2_2, 5_2_6_7, 1_3_0_6, 6_2, 1_5_2_5_3_7, 3_0_8, 7_9_9_7, 4_0_1, 1_2_4_4_2_7, 5_4_9, 3_5_4_4_2, 2_2_5, 1_0_9, 1_5_0_5_5, 2_5_7_4_8, 1_4_7, 7_1_1_9, 4_3_7_1_2, 3_4, 7_6_7, 1_3_5_3_6_6, 1_8, 1_6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_9_2, 6_3_7_8_4, 1_1_9_4_6_6, 1_7, 1_4_7_8_0_8, 8_8_2_1_4, 1_8, 6_5_6, 8_1, 3_2, 3_2_9_6, 1_0_2_8_0, 1_6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
           # fmt: on

           self.tokenizer_integration_test_util(
               expected_encoding=UpperCamelCase_					,				model_name='''microsoft/xprophetnet-large-wiki100-cased'''					,				revision='''1acad1643ddd54a44df6a1b797ada8373685d90e'''					,				)






"""simple docstring"""





import os
import tempfile
import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch


if is_torch_available():
 import torch
 from torch import nn

 from transformers import (
     Adafactor,
     AdamW,
     get_constant_schedule,
     get_constant_schedule_with_warmup,
     get_cosine_schedule_with_warmup,
     get_cosine_with_hard_restarts_schedule_with_warmup,
     get_inverse_sqrt_schedule,
     get_linear_schedule_with_warmup,
     get_polynomial_decay_schedule_with_warmup,
 )


def      _snake_case						(						_snake_case				:			Tuple ,   _snake_case				:			Union[str, Any]=10			):
      lowerCAmelCase					:	Dict       =   []
      for _ in range(_snake_case			):
            lrs.append(scheduler.get_lr()[0]			)
            scheduler.step()
      return lrs


def      _snake_case						(						_snake_case				:			Optional[int] ,   _snake_case				:			int=10			):
      lowerCAmelCase					:	Optional[int]       =   []
      for step in range(_snake_case			):
            lrs.append(scheduler.get_lr()[0]			)
            scheduler.step()
            if step == num_steps // 2:
                  with tempfile.TemporaryDirectory() as tmpdirname:
                        lowerCAmelCase					:	List[str]       =   os.path.join(_snake_case ,   '''schedule.bin'''			)
                        torch.save(scheduler.state_dict() ,   _snake_case			)

                        lowerCAmelCase					:	List[Any]       =   torch.load(_snake_case			)
                        scheduler.load_state_dict(_snake_case			)
      return lrs



@require_torch
class 		snake_case_(       unittest.TestCase			):






     def      lowerCamelCase__     (	self  :					Optional[int]					,				UpperCamelCase_  :					Optional[int]					,				UpperCamelCase_  :					str					,				UpperCamelCase_  :					Any				):
           self.assertEqual(len(UpperCamelCase_				)					,				len(UpperCamelCase_				)				)
           for a, b in zip(UpperCamelCase_					,				UpperCamelCase_				):
                 self.assertAlmostEqual(UpperCamelCase_					,				UpperCamelCase_					,				delta=UpperCamelCase_				)






     def      lowerCamelCase__     (	self  :					Tuple				):
           lowerCAmelCase					:	Any       =   torch.tensor([0.1, -0.2, -0.1]					,				requires_grad=UpperCamelCase_				)
           lowerCAmelCase					:	List[str]       =   torch.tensor([0.4, 0.2, -0.5]				)
           lowerCAmelCase					:	List[Any]       =   nn.MSELoss()
           # No warmup, constant schedule, no gradient clipping
           lowerCAmelCase					:	Union[str, Any]       =   AdamW(params=[w]					,				lr=2E-1					,				weight_decay=0.0				)
           for _ in range(1_0_0				):
                 lowerCAmelCase					:	Union[str, Any]       =   criterion(UpperCamelCase_					,				UpperCamelCase_				)
                 loss.backward()
                 optimizer.step()
                 w.grad.detach_()  # No zero_grad() function on simple tensors. we do it ourselves.
                 w.grad.zero_()
           self.assertListAlmostEqual(w.tolist()					,				[0.4, 0.2, -0.5]					,				tol=1E-2				)






     def      lowerCamelCase__     (	self  :					Union[str, Any]				):
           lowerCAmelCase					:	Tuple       =   torch.tensor([0.1, -0.2, -0.1]					,				requires_grad=UpperCamelCase_				)
           lowerCAmelCase					:	Union[str, Any]       =   torch.tensor([0.4, 0.2, -0.5]				)
           lowerCAmelCase					:	Optional[int]       =   nn.MSELoss()
           # No warmup, constant schedule, no gradient clipping
           lowerCAmelCase					:	Any       =   Adafactor(
               params=[w]					,				lr=1E-2					,				eps=(1E-30, 1E-3)					,				clip_threshold=1.0					,				decay_rate=-0.8					,				betaa=UpperCamelCase_					,				weight_decay=0.0					,				relative_step=UpperCamelCase_					,				scale_parameter=UpperCamelCase_					,				warmup_init=UpperCamelCase_					,				)
           for _ in range(1_0_0_0				):
                 lowerCAmelCase					:	List[Any]       =   criterion(UpperCamelCase_					,				UpperCamelCase_				)
                 loss.backward()
                 optimizer.step()
                 w.grad.detach_()  # No zero_grad() function on simple tensors. we do it ourselves.
                 w.grad.zero_()
           self.assertListAlmostEqual(w.tolist()					,				[0.4, 0.2, -0.5]					,				tol=1E-2				)



@require_torch
class 		snake_case_(       unittest.TestCase			):
     __UpperCamelCase  =					nn.Linear(50							,					50			) if is_torch_available() else None
     __UpperCamelCase  =					AdamW(m.parameters()							,					lr=10.0			) if is_torch_available() else None
     __UpperCamelCase  =					10






     def      lowerCamelCase__     (	self  :					Union[str, Any]					,				UpperCamelCase_  :					str					,				UpperCamelCase_  :					str					,				UpperCamelCase_  :					Optional[Any]					,				UpperCamelCase_  :					Union[str, Any]=None				):
           self.assertEqual(len(UpperCamelCase_				)					,				len(UpperCamelCase_				)				)
           for a, b in zip(UpperCamelCase_					,				UpperCamelCase_				):
                 self.assertAlmostEqual(UpperCamelCase_					,				UpperCamelCase_					,				delta=UpperCamelCase_					,				msg=UpperCamelCase_				)






     def      lowerCamelCase__     (	self  :					Union[str, Any]				):
           lowerCAmelCase					:	Tuple       =   {'''num_warmup_steps''': 2, '''num_training_steps''': 1_0}
           # schedulers doct format
           # function: (sched_args_dict, expected_learning_rates)
           lowerCAmelCase					:	Optional[Any]       =   {
               get_constant_schedule: ({}, [10.0] * self.num_steps),
               get_constant_schedule_with_warmup: (
                   {'''num_warmup_steps''': 4},
                   [0.0, 2.5, 5.0, 7.5, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0],
               ),
               get_linear_schedule_with_warmup: (
                   {**common_kwargs},
                   [0.0, 5.0, 10.0, 8.75, 7.5, 6.25, 5.0, 3.75, 2.5, 1.25],
               ),
               get_cosine_schedule_with_warmup: (
                   {**common_kwargs},
                   [0.0, 5.0, 10.0, 9.61, 8.53, 6.91, 5.0, 3.08, 1.46, 0.38],
               ),
               get_cosine_with_hard_restarts_schedule_with_warmup: (
                   {**common_kwargs, '''num_cycles''': 2},
                   [0.0, 5.0, 10.0, 8.53, 5.0, 1.46, 10.0, 8.53, 5.0, 1.46],
               ),
               get_polynomial_decay_schedule_with_warmup: (
                   {**common_kwargs, '''power''': 2.0, '''lr_end''': 1E-7},
                   [0.0, 5.0, 10.0, 7.656, 5.625, 3.906, 2.5, 1.406, 0.625, 0.156],
               ),
               get_inverse_sqrt_schedule: (
                   {'''num_warmup_steps''': 2},
                   [0.0, 5.0, 10.0, 8.165, 7.071, 6.325, 5.774, 5.345, 5.0, 4.714],
               ),
           }

           for scheduler_func, data in scheds.items():
                 lowerCAmelCase,      lowerCAmelCase					:	Union[str, Any]       =   data

                 lowerCAmelCase					:	List[Any]       =   scheduler_func(self.optimizer					,				**UpperCamelCase_				)
                 self.assertEqual(len([scheduler.get_lr()[0]]				)					,				1				)
                 lowerCAmelCase					:	str       =   unwrap_schedule(UpperCamelCase_					,				self.num_steps				)
                 self.assertListAlmostEqual(
                     UpperCamelCase_					,				UpperCamelCase_					,				tol=1E-2					,				msg=F'''failed for {scheduler_func} in normal scheduler'''					,				)

                 lowerCAmelCase					:	Optional[int]       =   scheduler_func(self.optimizer					,				**UpperCamelCase_				)
                 if scheduler_func.__name__ != "get_constant_schedule":
                       LambdaScheduleWrapper.wrap_scheduler(UpperCamelCase_				)  # wrap to test picklability of the schedule
                 lowerCAmelCase					:	List[Any]       =   unwrap_and_save_reload_schedule(UpperCamelCase_					,				self.num_steps				)
                 self.assertListEqual(UpperCamelCase_					,				UpperCamelCase_					,				msg=F'''failed for {scheduler_func} in save and reload'''				)



class 		snake_case_:






     def __init__(	self  :					List[Any]					,				UpperCamelCase_  :					Any				):
           lowerCAmelCase					:	Tuple       =   fn






     def __call__(	self  :					Union[str, Any]					,				*UpperCamelCase_  :					Optional[Any]					,				**UpperCamelCase_  :					List[Any]				):
           return self.fn(*UpperCamelCase_					,				**UpperCamelCase_				)






     @classmethod
     def      lowerCamelCase__     (	self  :					Any					,				UpperCamelCase_  :					Optional[int]				):
           lowerCAmelCase					:	Union[str, Any]       =   list(map(self					,				scheduler.lr_lambdas				)				)


