From 5e100f12442701905103c5040af58198b76824ee Mon Sep 17 00:00:00 2001
From: 游雁 <zhifu.gzf@alibaba-inc.com>
Date: 星期五, 08 十一月 2024 13:51:50 +0800
Subject: [PATCH] ds stage0
---
examples/deepspeed_conf/ds_stage0.json | 30 ++++++++++++++++++++++++++++++
1 files changed, 30 insertions(+), 0 deletions(-)
diff --git a/examples/deepspeed_conf/ds_stage0.json b/examples/deepspeed_conf/ds_stage0.json
new file mode 100644
index 0000000..661a6a0
--- /dev/null
+++ b/examples/deepspeed_conf/ds_stage0.json
@@ -0,0 +1,30 @@
+{
+ "train_micro_batch_size_per_gpu": 1,
+ "gradient_accumulation_steps": 1,
+ "steps_per_print": 100,
+ "gradient_clipping": 5,
+ "fp16": {
+ "enabled": false,
+ "auto_cast": false,
+ "loss_scale": 0,
+ "initial_scale_power": 16,
+ "loss_scale_window": 1000,
+ "hysteresis": 2,
+ "consecutive_hysteresis": false,
+ "min_loss_scale": 1
+ },
+ "bf16": {
+ "enabled": true
+ },
+ "zero_force_ds_cpu_optimizer": false,
+ "zero_optimization": {
+ "stage": 0,
+ "allgather_partitions": true,
+ "allgather_bucket_size": 5e8,
+ "overlap_comm": true,
+ "reduce_scatter": true,
+ "reduce_bucket_size": 5e8,
+ "contiguous_gradients": true,
+ "round_robin_gradients": true
+ }
+}
--
Gitblit v1.9.1