qlora-train/ds_config_zero2.json
2025-04-28 08:50:31 +00:00

52 lines
945 B
JSON

{
"train_batch_size": 32,
"gradient_accumulation_steps": 2,
"zero_optimization": {
"stage": 2,
"offload_param": {
"device": "cpu",
"pin_memory": true
},
"offload_optimizer": {
"device": "cpu",
"pin_memory": true
},
"overlap_comm": true,
"allgather_partitions": true,
"reduce_scatter": true
},
"fp16": {
"enabled": true,
"loss_scale": 0
},
"activation_checkpointing": {
"partition_activations": true,
"contiguous_memory_optimization": true,
"num_checkpointed_layers": 8
},
"optimizer": {
"type": "AdamW",
"params": {
"lr": 1e-5,
"betas": [0.9, 0.999],
"eps": 1e-8
}
},
"scheduler": {
"type": "WarmupLR",
"params": {
"warmup_min_lr": 0,
"warmup_max_lr": 1e-5,
"warmup_num_steps": 500
}
},
"logging": {
"steps": 200
},
"checkpoint": {
"steps": 1000,
"save": true
}
}