{ "train_batch_size": 32, "gradient_accumulation_steps": 2, "zero_optimization": { "stage": 2, "offload_param": { "device": "cpu", "pin_memory": true }, "offload_optimizer": { "device": "cpu", "pin_memory": true }, "overlap_comm": true, "allgather_partitions": true, "reduce_scatter": true }, "fp16": { "enabled": true, "loss_scale": 0 }, "activation_checkpointing": { "partition_activations": true, "contiguous_memory_optimization": true, "num_checkpointed_layers": 8 }, "optimizer": { "type": "AdamW", "params": { "lr": 1e-5, "betas": [0.9, 0.999], "eps": 1e-8 } }, "scheduler": { "type": "WarmupLR", "params": { "warmup_min_lr": 0, "warmup_max_lr": 1e-5, "warmup_num_steps": 500 } }, "logging": { "steps": 200 }, "checkpoint": { "steps": 1000, "save": true } }