






'''simple docstring'''







import os
import tempfile
import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch


if is_torch_available():
			import torch
			from torch import nn

			from transformers import (
			    Adafactor,
			    AdamW,
			    get_constant_schedule,
			    get_constant_schedule_with_warmup,
			    get_cosine_schedule_with_warmup,
			    get_cosine_with_hard_restarts_schedule_with_warmup,
			    get_inverse_sqrt_schedule,
			    get_linear_schedule_with_warmup,
			    get_polynomial_decay_schedule_with_warmup,
			)





def 				SCREAMING_SNAKE_CASE_ (UpperCamelCase       ,    UpperCamelCase=10   )   -> List[str]:
							lowerCamelCase__      :			int 						= []
							for _ in range(UpperCAmelCase__   ):
														lrs.append(scheduler.get_lr()[0]   )
														scheduler.step()
							return lrs





def 				SCREAMING_SNAKE_CASE_ (UpperCamelCase       ,    UpperCamelCase=10   )   -> Tuple:
							lowerCamelCase__      :			Any 						= []
							for step in range(UpperCAmelCase__   ):
														lrs.append(scheduler.get_lr()[0]   )
														scheduler.step()
														if step == num_steps // 2:
																					with tempfile.TemporaryDirectory() as tmpdirname:
																												lowerCamelCase__      :			List[Any] 						= os.path.join(UpperCAmelCase__       ,    """schedule.bin"""   )
																												torch.save(scheduler.state_dict()       ,    UpperCAmelCase__   )

																												lowerCamelCase__      :			Tuple 						= torch.load(UpperCAmelCase__   )
																												scheduler.load_state_dict(UpperCAmelCase__   )
							return lrs

@require_torch
class _lowercase			(       unittest.TestCase	):

							'''simple docstring'''





							def 	lowerCamelCase_       (							self:       Tuple    ,							UpperCamelCase__:       Optional[int]    ,							UpperCamelCase__:       Optional[int]    ,							UpperCamelCase__:       List[Any]  ):
														self.assertEqual(len(_A  )    ,							len(_A  )  )
														for a, b in zip(_A    ,							_A  ):
																					self.assertAlmostEqual(_A    ,							_A    ,							delta=_A  )




							def 	lowerCamelCase_       (							self:       Optional[int]  ):
														lowerCamelCase__      :			str 						= torch.tensor([0.1, -0.2, -0.1]    ,							requires_grad=_A  )
														lowerCamelCase__      :			Optional[int] 						= torch.tensor([0.4, 0.2, -0.5]  )
														lowerCamelCase__      :			Optional[Any] 						= nn.MSELoss()
														# No warmup, constant schedule, no gradient clipping
														lowerCamelCase__      :			int 						= AdamW(params=[w]    ,							lr=2e-1    ,							weight_decay=0.0  )
														for _ in range(100  ):
																					lowerCamelCase__      :			Tuple 						= criterion(_A    ,							_A  )
																					loss.backward()
																					optimizer.step()
																					w.grad.detach_()  # No zero_grad() function on simple tensors. we do it ourselves.
																					w.grad.zero_()
														self.assertListAlmostEqual(w.tolist()    ,							[0.4, 0.2, -0.5]    ,							tol=1e-2  )




							def 	lowerCamelCase_       (							self:       Tuple  ):
														lowerCamelCase__      :			Optional[int] 						= torch.tensor([0.1, -0.2, -0.1]    ,							requires_grad=_A  )
														lowerCamelCase__      :			Dict 						= torch.tensor([0.4, 0.2, -0.5]  )
														lowerCamelCase__      :			Union[str, Any] 						= nn.MSELoss()
														# No warmup, constant schedule, no gradient clipping
														lowerCamelCase__      :			Optional[int] 						= Adafactor(
														    params=[w]    ,							lr=1e-2    ,							eps=(1e-30, 1e-3)    ,							clip_threshold=1.0    ,							decay_rate=-0.8    ,							betaa=_A    ,							weight_decay=0.0    ,							relative_step=_A    ,							scale_parameter=_A    ,							warmup_init=_A    ,							)
														for _ in range(1_000  ):
																					lowerCamelCase__      :			Optional[int] 						= criterion(_A    ,							_A  )
																					loss.backward()
																					optimizer.step()
																					w.grad.detach_()  # No zero_grad() function on simple tensors. we do it ourselves.
																					w.grad.zero_()
														self.assertListAlmostEqual(w.tolist()    ,							[0.4, 0.2, -0.5]    ,							tol=1e-2  )



@require_torch
class _lowercase			(       unittest.TestCase	):

							'''simple docstring'''





							a          = nn.Linear(50       ,   50	) if is_torch_available() else None
							a          = AdamW(m.parameters()       ,   lr=10.0	) if is_torch_available() else None
							a          = 10




							def 	lowerCamelCase_       (							self:       List[str]    ,							UpperCamelCase__:       Dict    ,							UpperCamelCase__:       Optional[Any]    ,							UpperCamelCase__:       Any    ,							UpperCamelCase__:       Dict=None  ):
														self.assertEqual(len(_A  )    ,							len(_A  )  )
														for a, b in zip(_A    ,							_A  ):
																					self.assertAlmostEqual(_A    ,							_A    ,							delta=_A    ,							msg=_A  )




							def 	lowerCamelCase_       (							self:       Optional[int]  ):
														lowerCamelCase__      :			Tuple 						= {"""num_warmup_steps""": 2, """num_training_steps""": 10}
														# schedulers doct format
														# function: (sched_args_dict, expected_learning_rates)
														lowerCamelCase__      :			Union[str, Any] 						= {
														    get_constant_schedule: ({}, [10.0] * self.num_steps),
														    get_constant_schedule_with_warmup: (
														        {"""num_warmup_steps""": 4},
														        [0.0, 2.5, 5.0, 7.5, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0],
														    ),
														    get_linear_schedule_with_warmup: (
														        {**common_kwargs},
														        [0.0, 5.0, 10.0, 8.75, 7.5, 6.25, 5.0, 3.75, 2.5, 1.25],
														    ),
														    get_cosine_schedule_with_warmup: (
														        {**common_kwargs},
														        [0.0, 5.0, 10.0, 9.61, 8.53, 6.91, 5.0, 3.08, 1.46, 0.38],
														    ),
														    get_cosine_with_hard_restarts_schedule_with_warmup: (
														        {**common_kwargs, """num_cycles""": 2},
														        [0.0, 5.0, 10.0, 8.53, 5.0, 1.46, 10.0, 8.53, 5.0, 1.46],
														    ),
														    get_polynomial_decay_schedule_with_warmup: (
														        {**common_kwargs, """power""": 2.0, """lr_end""": 1e-7},
														        [0.0, 5.0, 10.0, 7.656, 5.625, 3.906, 2.5, 1.406, 0.625, 0.156],
														    ),
														    get_inverse_sqrt_schedule: (
														        {"""num_warmup_steps""": 2},
														        [0.0, 5.0, 10.0, 8.165, 7.071, 6.325, 5.774, 5.345, 5.0, 4.714],
														    ),
														}

														for scheduler_func, data in scheds.items():
																					lowerCamelCase__						,	lowerCamelCase__      :			List[Any] 						= data

																					lowerCamelCase__      :			Any 						= scheduler_func(self.optimizer    ,							**_A  )
																					self.assertEqual(len([scheduler.get_lr()[0]]  )    ,							1  )
																					lowerCamelCase__      :			Dict 						= unwrap_schedule(_A    ,							self.num_steps  )
																					self.assertListAlmostEqual(
																					    _A    ,							_A    ,							tol=1e-2    ,							msg=F'''failed for {scheduler_func} in normal scheduler'''    ,							)

																					lowerCamelCase__      :			Any 						= scheduler_func(self.optimizer    ,							**_A  )
																					if scheduler_func.__name__ != "get_constant_schedule":
																												LambdaScheduleWrapper.wrap_scheduler(_A  )  # wrap to test picklability of the schedule
																					lowerCamelCase__      :			Tuple 						= unwrap_and_save_reload_schedule(_A    ,							self.num_steps  )
																					self.assertListEqual(_A    ,							_A    ,							msg=F'''failed for {scheduler_func} in save and reload'''  )



class _lowercase			:

							'''simple docstring'''





							def __init__(							self:       Any    ,							UpperCamelCase__:       Optional[int]  ):
														lowerCamelCase__      :			Optional[Any] 						= fn




							def __call__(							self:       Any    ,							*UpperCamelCase__:       List[str]    ,							**UpperCamelCase__:       Any  ):
														return self.fn(*_A    ,							**_A  )




							@classmethod
							def 	lowerCamelCase_       (							self:       List[str]    ,							UpperCamelCase__:       Optional[Any]  ):
														lowerCamelCase__      :			Optional[int] 						= list(map(self    ,							scheduler.lr_lambdas  )  )









'''simple docstring'''







from ...configuration_utils import PretrainedConfig
from ...utils import logging


_A			:    Optional[Any]					  =logging.get_logger(__name__)

_A			:    Optional[int]					  ={
    '''RWKV/rwkv-4-169m-pile''': '''https://huggingface.co/RWKV/rwkv-4-169m-pile/resolve/main/config.json''',
    '''RWKV/rwkv-4-430m-pile''': '''https://huggingface.co/RWKV/rwkv-4-430m-pile/resolve/main/config.json''',
    '''RWKV/rwkv-4-1b5-pile''': '''https://huggingface.co/RWKV/rwkv-4-1b5-pile/resolve/main/config.json''',
    '''RWKV/rwkv-4-3b-pile''': '''https://huggingface.co/RWKV/rwkv-4-3b-pile/resolve/main/config.json''',
    '''RWKV/rwkv-4-7b-pile''': '''https://huggingface.co/RWKV/rwkv-4-7b-pile/resolve/main/config.json''',
    '''RWKV/rwkv-4-14b-pile''': '''https://huggingface.co/RWKV/rwkv-4-14b-pile/resolve/main/config.json''',
    '''RWKV/rwkv-raven-1b5''': '''https://huggingface.co/RWKV/rwkv-raven-1b5/resolve/main/config.json''',
    '''RWKV/rwkv-raven-3b''': '''https://huggingface.co/RWKV/rwkv-raven-3b/resolve/main/config.json''',
    '''RWKV/rwkv-raven-7b''': '''https://huggingface.co/RWKV/rwkv-raven-7b/resolve/main/config.json''',
    '''RWKV/rwkv-raven-14b''': '''https://huggingface.co/RWKV/rwkv-raven-14b/resolve/main/config.json''',
}

class _lowercase			(       _lowercase	):
							a          = """rwkv"""
							a          = {"""max_position_embeddings""": """context_length"""}




							def __init__(							self:       Tuple    ,							UpperCamelCase__:       Optional[Any]=50_277    ,							UpperCamelCase__:       Union[str, Any]=1_024    ,							UpperCamelCase__:       Tuple=4_096    ,							UpperCamelCase__:       List[Any]=32    ,							UpperCamelCase__:       Dict=None    ,							UpperCamelCase__:       Dict=None    ,							UpperCamelCase__:       int=1e-5    ,							UpperCamelCase__:       Any=0    ,							UpperCamelCase__:       str=0    ,							UpperCamelCase__:       Union[str, Any]=6    ,							UpperCamelCase__:       Optional[int]=False    ,							UpperCamelCase__:       Dict=True    ,							**UpperCamelCase__:       Dict    ,							):
														lowerCamelCase__      :			Dict 						= vocab_size
														lowerCamelCase__      :			Optional[Any] 						= context_length
														lowerCamelCase__      :			Optional[Any] 						= hidden_size
														lowerCamelCase__      :			Any 						= num_hidden_layers
														lowerCamelCase__      :			int 						= attention_hidden_size if attention_hidden_size is not None else hidden_size
														lowerCamelCase__      :			Union[str, Any] 						= intermediate_size if intermediate_size is not None else 4 * hidden_size
														lowerCamelCase__      :			List[str] 						= layer_norm_epsilon
														lowerCamelCase__      :			int 						= rescale_every
														lowerCamelCase__      :			Optional[int] 						= use_cache

														lowerCamelCase__      :			Dict 						= bos_token_id
														lowerCamelCase__      :			Any 						= eos_token_id

														super().__init__(
														    tie_word_embeddings=UpperCamelCase__    ,							bos_token_id=UpperCamelCase__    ,							eos_token_id=UpperCamelCase__    ,							**UpperCamelCase__  )

