准备训练环境
硬件要求
| 配置项 | 推荐配置 | 最低配置 |
|---|
| GPU | 8× xxxxx × 2‑4 节点 | 8× xxxxxxxxxxxxxxx |
| CPU | 双路 AMD EPYC / Intel Xeon，64+ 核心 | 32 核心以上 |
| 内存 | 1‑2 TB DDR4/DDR5 ECC | 512 GB |
| 存储 | NVMe SSD 4TB+ | NVMe SSD 2TB |
| 网络 | InfiniBand HDR 200Gb/s | 10GbE |
# 创建 conda 环境
conda create -n llama2-70b python=3.10 -y
conda activate llama2-70b
# 安装 PyTorch (CUDA 12.1)
pip install torch==2.1.2 torchvision==0.16.2 torchaudio==2.1.2 --index-url https://download.pytorch.org/whl/cu121
# 训练核心依赖
pip install transformers==4.36.2 accelerate==0.25.0 deepspeed==0.12.6
pip install peft==0.7.1 datasets==2.16.1 sentencepiece==0.1.99
pip install flash-attn==2.5.0 --no-build-isolation
# 监控工具
pip install wandb tensorboard
{
"train_batch_size": "auto",
"train_micro_batch_size_per_gpu": "auto",
"gradient_accumulation_steps": "auto",
"zero_optimization": {
"stage": 3,
"offload_optimizer": { "device": "cpu", "pin_memory": true },
"offload_param": { "device": "cpu", "pin_memory": true },
"overlap_comm": true,
"contiguous_gradients": true,
"sub_group_size": 1e9,
"reduce_bucket_size": "auto",
"stage3_prefetch_bucket_size": "auto",
"stage3_param_persistence_threshold": "auto",
"stage3_max_live_parameters": 1e9,
"stage3_max_reuse_distance": 1e9,
"stage3_gather_16bit_weights_on_model_save": true
},
"bf16": { "enabled": true },
"gradient_clipping": 1.0,
"wall_clock_breakdown": false
}
