





'''simple docstring'''




import math
from enum import Enum
from typing import Optional, Union

from torch.optim import Optimizer
from torch.optim.lr_scheduler import LambdaLR

from .utils import logging


_A:					Optional[int]							=				logging.get_logger(__name__)






class UpperCAmelCase							(		__lowerCamelCase							):
    _A :			int						   =       """linear"""
    _A :			int						   =       """cosine"""
    _A :			str						   =       """cosine_with_restarts"""
    _A :			Dict						   =       """polynomial"""
    _A :			List[str]						   =       """constant"""
    _A :			List[str]						   =       """constant_with_warmup"""
    _A :			Optional[int]						   =       """piecewise_constant"""







def _lowerCAmelCase     (      _lowerCAmelCase       ,    _lowerCAmelCase = -1    )->     int:
 return LambdaLR(lowerCamelCase_       ,    lambda _lowerCAmelCase    : 1       ,    last_epoch=lowerCamelCase_    )







def _lowerCAmelCase     (      _lowerCAmelCase       ,    _lowerCAmelCase       ,    _lowerCAmelCase = -1    )->     int:

 def lr_lambda(_lowerCAmelCase    ):
  if current_step < num_warmup_steps:
   return float(lowerCamelCase_    ) / float(max(1.0       ,    lowerCamelCase_    )    )
  return 1.0

 return LambdaLR(lowerCamelCase_       ,    lowerCamelCase_       ,    last_epoch=lowerCamelCase_    )







def _lowerCAmelCase     (      _lowerCAmelCase       ,    _lowerCAmelCase       ,    _lowerCAmelCase = -1    )->     Union[str, Any]:
 __UpperCAmelCase					=   {}
 __UpperCAmelCase					=   step_rules.split(','    )
 for rule_str in rule_list[:-1]:
  __UpperCAmelCase					=   rule_str.split(':'    )
  __UpperCAmelCase					=   int(lowerCamelCase_    )
  __UpperCAmelCase					=   float(lowerCamelCase_    )
  __UpperCAmelCase					=   value
 __UpperCAmelCase					=   float(rule_list[-1]    )

 def create_rules_function(_lowerCAmelCase       ,    _lowerCAmelCase    ):
  def rule_func(_lowerCAmelCase    ) -> float:
   __UpperCAmelCase					=   sorted(rules_dict.keys()    )
   for i, sorted_step in enumerate(lowerCamelCase_    ):
    if steps < sorted_step:
     return rules_dict[sorted_steps[i]]
   return last_lr_multiple

  return rule_func

 __UpperCAmelCase					=   create_rules_function(lowerCamelCase_       ,    lowerCamelCase_    )

 return LambdaLR(lowerCamelCase_       ,    lowerCamelCase_       ,    last_epoch=lowerCamelCase_    )







def _lowerCAmelCase     (      _lowerCAmelCase       ,    _lowerCAmelCase       ,    _lowerCAmelCase       ,    _lowerCAmelCase=-1    )->     Optional[Any]:

 def lr_lambda(_lowerCAmelCase    ):
  if current_step < num_warmup_steps:
   return float(lowerCamelCase_    ) / float(max(1       ,    lowerCamelCase_    )    )
  return max(
      0.0       ,    float(num_training_steps - current_step    ) / float(max(1       ,    num_training_steps - num_warmup_steps    )    )    )

 return LambdaLR(lowerCamelCase_       ,    lowerCamelCase_       ,    lowerCamelCase_    )







def _lowerCAmelCase     (      _lowerCAmelCase       ,    _lowerCAmelCase       ,    _lowerCAmelCase       ,    _lowerCAmelCase = 0.5       ,    _lowerCAmelCase = -1    )->     Any:

 def lr_lambda(_lowerCAmelCase    ):
  if current_step < num_warmup_steps:
   return float(lowerCamelCase_    ) / float(max(1       ,    lowerCamelCase_    )    )
  __UpperCAmelCase					=   float(current_step - num_warmup_steps    ) / float(max(1       ,    num_training_steps - num_warmup_steps    )    )
  return max(0.0       ,    0.5 * (1.0 + math.cos(math.pi * float(lowerCamelCase_    ) * 2.0 * progress    ))    )

 return LambdaLR(lowerCamelCase_       ,    lowerCamelCase_       ,    lowerCamelCase_    )







def _lowerCAmelCase     (      _lowerCAmelCase       ,    _lowerCAmelCase       ,    _lowerCAmelCase       ,    _lowerCAmelCase = 1       ,    _lowerCAmelCase = -1    )->     List[Any]:

 def lr_lambda(_lowerCAmelCase    ):
  if current_step < num_warmup_steps:
   return float(lowerCamelCase_    ) / float(max(1       ,    lowerCamelCase_    )    )
  __UpperCAmelCase					=   float(current_step - num_warmup_steps    ) / float(max(1       ,    num_training_steps - num_warmup_steps    )    )
  if progress >= 1.0:
   return 0.0
  return max(0.0       ,    0.5 * (1.0 + math.cos(math.pi * ((float(lowerCamelCase_    ) * progress) % 1.0)    ))    )

 return LambdaLR(lowerCamelCase_       ,    lowerCamelCase_       ,    lowerCamelCase_    )







def _lowerCAmelCase     (      _lowerCAmelCase       ,    _lowerCAmelCase       ,    _lowerCAmelCase       ,    _lowerCAmelCase=1e-7       ,    _lowerCAmelCase=1.0       ,    _lowerCAmelCase=-1    )->     Optional[int]:
 __UpperCAmelCase					=   optimizer.defaults['lr']
 if not (lr_init > lr_end):
  raise ValueError(F'lr_end ({lr_end}) must be be smaller than initial lr ({lr_init})'    )

 def lr_lambda(_lowerCAmelCase    ):
  if current_step < num_warmup_steps:
   return float(lowerCamelCase_    ) / float(max(1       ,    lowerCamelCase_    )    )
  elif current_step > num_training_steps:
   return lr_end / lr_init  # as LambdaLR multiplies by lr_init
  else:
   __UpperCAmelCase					=   lr_init - lr_end
   __UpperCAmelCase					=   num_training_steps - num_warmup_steps
   __UpperCAmelCase					=   1 - (current_step - num_warmup_steps) / decay_steps
   __UpperCAmelCase					=   lr_range * pct_remaining**power + lr_end
   return decay / lr_init  # as LambdaLR multiplies by lr_init

 return LambdaLR(lowerCamelCase_       ,    lowerCamelCase_       ,    lowerCamelCase_    )


_A:					Optional[int]							=				{
    SchedulerType.LINEAR: get_linear_schedule_with_warmup,
    SchedulerType.COSINE: get_cosine_schedule_with_warmup,
    SchedulerType.COSINE_WITH_RESTARTS: get_cosine_with_hard_restarts_schedule_with_warmup,
    SchedulerType.POLYNOMIAL: get_polynomial_decay_schedule_with_warmup,
    SchedulerType.CONSTANT: get_constant_schedule,
    SchedulerType.CONSTANT_WITH_WARMUP: get_constant_schedule_with_warmup,
    SchedulerType.PIECEWISE_CONSTANT: get_piecewise_constant_schedule,
}







def _lowerCAmelCase     (      _lowerCAmelCase       ,    _lowerCAmelCase       ,    _lowerCAmelCase = None       ,    _lowerCAmelCase = None       ,    _lowerCAmelCase = None       ,    _lowerCAmelCase = 1       ,    _lowerCAmelCase = 1.0       ,    _lowerCAmelCase = -1       ,    )->     Optional[Any]:
 __UpperCAmelCase					=   SchedulerType(lowerCamelCase_    )
 __UpperCAmelCase					=   TYPE_TO_SCHEDULER_FUNCTION[name]
 if name == SchedulerType.CONSTANT:
  return schedule_func(lowerCamelCase_       ,    last_epoch=lowerCamelCase_    )

 if name == SchedulerType.PIECEWISE_CONSTANT:
  return schedule_func(lowerCamelCase_       ,    step_rules=lowerCamelCase_       ,    last_epoch=lowerCamelCase_    )

 # All other schedulers require `num_warmup_steps`
 if num_warmup_steps is None:
  raise ValueError(F'{name} requires `num_warmup_steps`, please provide that argument.'    )

 if name == SchedulerType.CONSTANT_WITH_WARMUP:
  return schedule_func(lowerCamelCase_       ,    num_warmup_steps=lowerCamelCase_       ,    last_epoch=lowerCamelCase_    )

 # All other schedulers require `num_training_steps`
 if num_training_steps is None:
  raise ValueError(F'{name} requires `num_training_steps`, please provide that argument.'    )

 if name == SchedulerType.COSINE_WITH_RESTARTS:
  return schedule_func(
      lowerCamelCase_       ,    num_warmup_steps=lowerCamelCase_       ,    num_training_steps=lowerCamelCase_       ,    num_cycles=lowerCamelCase_       ,    last_epoch=lowerCamelCase_       ,    )

 if name == SchedulerType.POLYNOMIAL:
  return schedule_func(
      lowerCamelCase_       ,    num_warmup_steps=lowerCamelCase_       ,    num_training_steps=lowerCamelCase_       ,    power=lowerCamelCase_       ,    last_epoch=lowerCamelCase_       ,    )

 return schedule_func(
     lowerCamelCase_       ,    num_warmup_steps=lowerCamelCase_       ,    num_training_steps=lowerCamelCase_       ,    last_epoch=lowerCamelCase_    )












'''simple docstring'''




from ...configuration_utils import PretrainedConfig
from ...utils import logging


_A:					Union[str, Any]							=				logging.get_logger(__name__)

_A:					List[str]							=				{
    """weiweishi/roc-bert-base-zh""": """https://huggingface.co/weiweishi/roc-bert-base-zh/resolve/main/config.json""",
}






class UpperCAmelCase							(		UpperCAmelCase_							):
    _A :			int						   =       """roc_bert"""



    def __init__(  self     ,			__A=30_522     ,			__A=768     ,			__A=12     ,			__A=12     ,			__A=3_072     ,			__A="gelu"     ,			__A=0.1     ,			__A=0.1     ,			__A=512     ,			__A=2     ,			__A=0.0_2     ,			__A=1E-12     ,			__A=True     ,			__A=0     ,			__A="absolute"     ,			__A=None     ,			__A=True     ,			__A=True     ,			__A=768     ,			__A=910     ,			__A=512     ,			__A=24_858     ,			__A=True     ,			**__A     ,			):
     __UpperCAmelCase					=   vocab_size
     __UpperCAmelCase					=   max_position_embeddings
     __UpperCAmelCase					=   hidden_size
     __UpperCAmelCase					=   num_hidden_layers
     __UpperCAmelCase					=   num_attention_heads
     __UpperCAmelCase					=   intermediate_size
     __UpperCAmelCase					=   hidden_act
     __UpperCAmelCase					=   hidden_dropout_prob
     __UpperCAmelCase					=   attention_probs_dropout_prob
     __UpperCAmelCase					=   initializer_range
     __UpperCAmelCase					=   type_vocab_size
     __UpperCAmelCase					=   layer_norm_eps
     __UpperCAmelCase					=   use_cache
     __UpperCAmelCase					=   enable_pronunciation
     __UpperCAmelCase					=   enable_shape
     __UpperCAmelCase					=   pronunciation_embed_dim
     __UpperCAmelCase					=   pronunciation_vocab_size
     __UpperCAmelCase					=   shape_embed_dim
     __UpperCAmelCase					=   shape_vocab_size
     __UpperCAmelCase					=   concat_input
     __UpperCAmelCase					=   position_embedding_type
     __UpperCAmelCase					=   classifier_dropout
     super().__init__(pad_token_id=__A     ,			**__A					)





