| accumulative_counts = 2 |
| batch_size = 1 |
| betas = ( |
| 0.9, |
| 0.95, |
| ) |
| custom_hooks = [ |
| dict(type='xtuner.engine.hooks.VarlenAttnArgsToMessageHubHook'), |
| ] |
| data_num = 150221 |
| data_path = '/cpfs01/shared/llm_ddd/zouyicheng/rm_pretrain/data/preference/single_source_prompt_sft/mixed/HH_puyu' |
| dataloader_num_workers = 0 |
| default_hooks = dict( |
| checkpoint=dict( |
| by_epoch=False, |
| interval=100, |
| max_keep_ckpts=-1, |
| type='mmengine.hooks.CheckpointHook'), |
| logger=dict( |
| interval=10, |
| log_metric_by_epoch=False, |
| type='mmengine.hooks.LoggerHook'), |
| param_scheduler=dict(type='mmengine.hooks.ParamSchedulerHook'), |
| sampler_seed=dict(type='mmengine.hooks.DistSamplerSeedHook'), |
| timer=dict(type='mmengine.hooks.IterTimerHook')) |
| env_cfg = dict( |
| cudnn_benchmark=False, |
| dist_cfg=dict(backend='nccl'), |
| mp_cfg=dict(mp_start_method='fork', opencv_num_threads=0)) |
| launcher = 'pytorch' |
| load_from = None |
| log_level = 'INFO' |
| log_processor = dict(by_epoch=False) |
| loss_type = 'ranking' |
| lr = 1e-05 |
| max_epochs = 1 |
| max_length = 16384 |
| max_norm = 1 |
| max_packed_length = 32768 |
| max_response_length = 4096 |
| model = dict( |
| llm=dict( |
| pretrained_model_name_or_path= |
| '/cpfs01/shared/llm_ddd/zouyicheng/rm_pretrain/rm/RM_PT_internlm2_5_1_8b_DATA_140m_single_mix_Node_40_LR_1e_5_STEP_89122_hf', |
| trust_remote_code=True, |
| type='transformers.AutoModel.from_pretrained'), |
| loss_type='ranking', |
| penalty_type='none', |
| type='xtuner.model.reward.RewardModel', |
| use_varlen_attn=True) |
| optim_type = 'torch.optim.AdamW' |
| optim_wrapper = dict( |
| optimizer=dict( |
| betas=( |
| 0.9, |
| 0.95, |
| ), |
| lr=1e-05, |
| type='torch.optim.AdamW', |
| weight_decay=0), |
| type='DeepSpeedOptimWrapper') |
| param_scheduler = [ |
| dict( |
| begin=0, |
| by_epoch=True, |
| convert_to_iter_based=True, |
| end=0.03, |
| start_factor=1.0000000000000002e-06, |
| type='mmengine.optim.LinearLR'), |
| dict( |
| begin=0.03, |
| by_epoch=True, |
| convert_to_iter_based=True, |
| end=1, |
| eta_min=1.0000000000000002e-06, |
| type='mmengine.optim.CosineAnnealingLR'), |
| ] |
| penalty_type = 'none' |
| pretrained_model_name_or_path = '/cpfs01/shared/llm_ddd/zouyicheng/rm_pretrain/rm/RM_PT_internlm2_5_1_8b_DATA_140m_single_mix_Node_40_LR_1e_5_STEP_89122_hf' |
| randomness = dict(deterministic=False, seed=None) |
| resume = False |
| reward_token_id = 92527 |
| runner_type = 'FlexibleRunner' |
| sampler = 'mmengine.dataset.DefaultSampler' |
| save_steps = 100 |
| save_total_limit = -1 |
| sequence_parallel_size = 1 |
| strategy = dict( |
| config=dict( |
| bf16=dict(enabled=True), |
| fp16=dict(enabled=False, initial_scale_power=16), |
| gradient_accumulation_steps='auto', |
| gradient_clipping='auto', |
| train_micro_batch_size_per_gpu='auto', |
| zero_allow_untested_optimizer=True, |
| zero_force_ds_cpu_optimizer=False, |
| zero_optimization=dict(overlap_comm=True, stage=1)), |
| exclude_frozen_parameters=True, |
| gradient_accumulation_steps=2, |
| gradient_clipping=1, |
| sequence_parallel_size=1, |
| train_micro_batch_size_per_gpu=1, |
| type='xtuner.engine.DeepSpeedStrategy') |
| tokenizer = dict( |
| padding_side='left', |
| pretrained_model_name_or_path= |
| '/cpfs01/shared/llm_ddd/zouyicheng/rm_pretrain/rm/RM_PT_internlm2_5_1_8b_DATA_140m_single_mix_Node_40_LR_1e_5_STEP_89122_hf', |
| trust_remote_code=True, |
| type='transformers.AutoTokenizer.from_pretrained') |
| train_cfg = dict(max_epochs=1, type='xtuner.engine.runner.TrainLoop') |
| train_dataloader = dict( |
| batch_size=1, |
| collate_fn=dict( |
| type= |
| 'xtuner.dataset.collate_fns.preference_collate_fn.preference_collate_fn', |
| use_varlen_attn=True), |
| dataset=dict( |
| dataset=dict( |
| path= |
| '/cpfs01/shared/llm_ddd/zouyicheng/rm_pretrain/data/preference/single_source_prompt_sft/mixed/HH_puyu', |
| type='datasets.load_dataset'), |
| dataset_map_fn=None, |
| is_dpo=False, |
| is_reward=True, |
| max_length=16384, |
| max_packed_length=32768, |
| max_response_length=4096, |
| num_proc=32, |
| reward_token_id=92527, |
| shuffle_before_pack=True, |
| tokenizer=dict( |
| padding_side='left', |
| pretrained_model_name_or_path= |
| '/cpfs01/shared/llm_ddd/zouyicheng/rm_pretrain/rm/RM_PT_internlm2_5_1_8b_DATA_140m_single_mix_Node_40_LR_1e_5_STEP_89122_hf', |
| trust_remote_code=True, |
| type='transformers.AutoTokenizer.from_pretrained'), |
| type='xtuner.dataset.preference_dataset.build_preference_dataset', |
| use_varlen_attn=True), |
| num_workers=0, |
| sampler=dict(shuffle=True, type='mmengine.dataset.DefaultSampler')) |
| train_dataset = dict( |
| dataset=dict( |
| path= |
| '/cpfs01/shared/llm_ddd/zouyicheng/rm_pretrain/data/preference/single_source_prompt_sft/mixed/HH_puyu', |
| type='datasets.load_dataset'), |
| dataset_map_fn=None, |
| is_dpo=False, |
| is_reward=True, |
| max_length=16384, |
| max_packed_length=32768, |
| max_response_length=4096, |
| num_proc=32, |
| reward_token_id=92527, |
| shuffle_before_pack=True, |
| tokenizer=dict( |
| padding_side='left', |
| pretrained_model_name_or_path= |
| '/cpfs01/shared/llm_ddd/zouyicheng/rm_pretrain/rm/RM_PT_internlm2_5_1_8b_DATA_140m_single_mix_Node_40_LR_1e_5_STEP_89122_hf', |
| trust_remote_code=True, |
| type='transformers.AutoTokenizer.from_pretrained'), |
| type='xtuner.dataset.preference_dataset.build_preference_dataset', |
| use_varlen_attn=True) |
| use_varlen_attn = True |
| visualizer = dict( |
| type='mmengine.visualization.Visualizer', |
| vis_backends=[ |
| dict(type='mmengine.visualization.TensorboardVisBackend'), |
| ]) |
| warmup_ratio = 0.03 |
| weight_decay = 0 |
| work_dir = './work_dirs/RM_SFT_reward_pt_1_8b_final_DATA_HH_puyu_mixed_Node_2_LR_1e-5' |
|
|
