From d6fedbfa77abee8b31340dc1a7aa344b84c8134c Mon Sep 17 00:00:00 2001
From: 游雁 <zhifu.gzf@alibaba-inc.com>
Date: 星期四, 06 六月 2024 19:22:11 +0800
Subject: [PATCH] auto frontend
---
examples/industrial_data_pretraining/deepspeed/ds_stage2.json | 33 ++++++
examples/industrial_data_pretraining/deepspeed/ds_z2_config.json | 28 +++++
examples/industrial_data_pretraining/deepspeed/ds_z2_offload_config.json | 32 ++++++
funasr/models/llm_asr/model.py | 12 -
examples/industrial_data_pretraining/deepspeed/ds_z3_config.json | 30 ++++++
examples/industrial_data_pretraining/deepspeed/ds_stage1.json | 33 ++++++
examples/industrial_data_pretraining/deepspeed/ds_z3_offload_config.json | 38 +++++++
examples/industrial_data_pretraining/deepspeed/ds_z0_config.json | 28 +++++
examples/industrial_data_pretraining/deepspeed/ds_stage3.json | 41 ++++++++
9 files changed, 266 insertions(+), 9 deletions(-)
diff --git a/examples/industrial_data_pretraining/deepspeed/ds_stage1.json b/examples/industrial_data_pretraining/deepspeed/ds_stage1.json
new file mode 100644
index 0000000..51804c1
--- /dev/null
+++ b/examples/industrial_data_pretraining/deepspeed/ds_stage1.json
@@ -0,0 +1,33 @@
+{
+ "train_micro_batch_size_per_gpu": 1,
+ "gradient_accumulation_steps": 1,
+ "steps_per_print": 100,
+ "gradient_clipping": 5,
+ "fp16": {
+ "enabled": false,
+ "auto_cast": false,
+ "loss_scale": 0,
+ "initial_scale_power": 16,
+ "loss_scale_window": 1000,
+ "hysteresis": 2,
+ "consecutive_hysteresis": false,
+ "min_loss_scale": 1
+ },
+ "bf16": {
+ "enabled": true
+ },
+ "zero_force_ds_cpu_optimizer": false,
+ "zero_optimization": {
+ "stage": 1,
+ "offload_optimizer": {
+ "device": "none",
+ "pin_memory": true
+ },
+ "allgather_partitions": true,
+ "allgather_bucket_size": 5e8,
+ "overlap_comm": true,
+ "reduce_scatter": true,
+ "reduce_bucket_size": 5e8,
+ "contiguous_gradients" : true
+ }
+}
diff --git a/examples/industrial_data_pretraining/deepspeed/ds_stage2.json b/examples/industrial_data_pretraining/deepspeed/ds_stage2.json
new file mode 100644
index 0000000..c11b7d6
--- /dev/null
+++ b/examples/industrial_data_pretraining/deepspeed/ds_stage2.json
@@ -0,0 +1,33 @@
+{
+ "train_micro_batch_size_per_gpu": 1,
+ "gradient_accumulation_steps": 1,
+ "steps_per_print": 100,
+ "gradient_clipping": 5,
+ "fp16": {
+ "enabled": false,
+ "auto_cast": false,
+ "loss_scale": 0,
+ "initial_scale_power": 16,
+ "loss_scale_window": 1000,
+ "hysteresis": 2,
+ "consecutive_hysteresis": false,
+ "min_loss_scale": 1
+ },
+ "bf16": {
+ "enabled": true
+ },
+ "zero_force_ds_cpu_optimizer": false,
+ "zero_optimization": {
+ "stage": 2,
+ "offload_optimizer": {
+ "device": "none",
+ "pin_memory": true
+ },
+ "allgather_partitions": true,
+ "allgather_bucket_size": 5e8,
+ "overlap_comm": false,
+ "reduce_scatter": true,
+ "reduce_bucket_size": 5e8,
+ "contiguous_gradients" : true
+ }
+}
diff --git a/examples/industrial_data_pretraining/deepspeed/ds_stage3.json b/examples/industrial_data_pretraining/deepspeed/ds_stage3.json
new file mode 100644
index 0000000..ba38293
--- /dev/null
+++ b/examples/industrial_data_pretraining/deepspeed/ds_stage3.json
@@ -0,0 +1,41 @@
+{
+ "train_micro_batch_size_per_gpu": 1,
+ "gradient_accumulation_steps": 1,
+ "steps_per_print": 100,
+ "gradient_clipping": 5,
+ "fp16": {
+ "enabled": false,
+ "auto_cast": false,
+ "loss_scale": 0,
+ "initial_scale_power": 16,
+ "loss_scale_window": 1000,
+ "hysteresis": 2,
+ "consecutive_hysteresis": false,
+ "min_loss_scale": 1
+ },
+ "bf16": {
+ "enabled": true
+ },
+ "zero_force_ds_cpu_optimizer": false,
+ "zero_optimization": {
+ "stage": 3,
+ "offload_optimizer": {
+ "device": "none",
+ "pin_memory": true
+ },
+ "offload_param": {
+ "device": "none",
+ "pin_memory": true
+ },
+ "allgather_partitions": true,
+ "allgather_bucket_size": 5e8,
+ "overlap_comm": true,
+ "reduce_scatter": true,
+ "reduce_bucket_size": 5e8,
+ "contiguous_gradients" : true,
+ "stage3_max_live_parameters": 1e9,
+ "stage3_max_reuse_distance": 1e9,
+ "stage3_prefetch_bucket_size": 5e8,
+ "stage3_param_persistence_threshold": 1e5
+ }
+}
diff --git a/examples/industrial_data_pretraining/deepspeed/ds_z0_config.json b/examples/industrial_data_pretraining/deepspeed/ds_z0_config.json
new file mode 100644
index 0000000..ed32667
--- /dev/null
+++ b/examples/industrial_data_pretraining/deepspeed/ds_z0_config.json
@@ -0,0 +1,28 @@
+{
+ "train_batch_size": "auto",
+ "train_micro_batch_size_per_gpu": "auto",
+ "gradient_accumulation_steps": "auto",
+ "gradient_clipping": "auto",
+ "zero_allow_untested_optimizer": true,
+ "fp16": {
+ "enabled": "auto",
+ "loss_scale": 0,
+ "loss_scale_window": 1000,
+ "initial_scale_power": 16,
+ "hysteresis": 2,
+ "min_loss_scale": 1
+ },
+ "bf16": {
+ "enabled": "auto"
+ },
+ "zero_optimization": {
+ "stage": 0,
+ "allgather_partitions": true,
+ "allgather_bucket_size": 5e8,
+ "overlap_comm": true,
+ "reduce_scatter": true,
+ "reduce_bucket_size": 5e8,
+ "contiguous_gradients": true,
+ "round_robin_gradients": true
+ }
+}
\ No newline at end of file
diff --git a/examples/industrial_data_pretraining/deepspeed/ds_z2_config.json b/examples/industrial_data_pretraining/deepspeed/ds_z2_config.json
new file mode 100644
index 0000000..0a1bd1d
--- /dev/null
+++ b/examples/industrial_data_pretraining/deepspeed/ds_z2_config.json
@@ -0,0 +1,28 @@
+{
+ "train_batch_size": "auto",
+ "train_micro_batch_size_per_gpu": "auto",
+ "gradient_accumulation_steps": "auto",
+ "gradient_clipping": "auto",
+ "zero_allow_untested_optimizer": true,
+ "fp16": {
+ "enabled": "auto",
+ "loss_scale": 0,
+ "loss_scale_window": 1000,
+ "initial_scale_power": 16,
+ "hysteresis": 2,
+ "min_loss_scale": 1
+ },
+ "bf16": {
+ "enabled": "auto"
+ },
+ "zero_optimization": {
+ "stage": 2,
+ "allgather_partitions": true,
+ "allgather_bucket_size": 5e8,
+ "overlap_comm": true,
+ "reduce_scatter": true,
+ "reduce_bucket_size": 5e8,
+ "contiguous_gradients": true,
+ "round_robin_gradients": true
+ }
+}
\ No newline at end of file
diff --git a/examples/industrial_data_pretraining/deepspeed/ds_z2_offload_config.json b/examples/industrial_data_pretraining/deepspeed/ds_z2_offload_config.json
new file mode 100644
index 0000000..7a39836
--- /dev/null
+++ b/examples/industrial_data_pretraining/deepspeed/ds_z2_offload_config.json
@@ -0,0 +1,32 @@
+{
+ "train_batch_size": "auto",
+ "train_micro_batch_size_per_gpu": "auto",
+ "gradient_accumulation_steps": "auto",
+ "gradient_clipping": "auto",
+ "zero_allow_untested_optimizer": true,
+ "fp16": {
+ "enabled": "auto",
+ "loss_scale": 0,
+ "loss_scale_window": 1000,
+ "initial_scale_power": 16,
+ "hysteresis": 2,
+ "min_loss_scale": 1
+ },
+ "bf16": {
+ "enabled": "auto"
+ },
+ "zero_optimization": {
+ "stage": 2,
+ "offload_optimizer": {
+ "device": "cpu",
+ "pin_memory": true
+ },
+ "allgather_partitions": true,
+ "allgather_bucket_size": 5e8,
+ "overlap_comm": true,
+ "reduce_scatter": true,
+ "reduce_bucket_size": 5e8,
+ "contiguous_gradients": true,
+ "round_robin_gradients": true
+ }
+}
\ No newline at end of file
diff --git a/examples/industrial_data_pretraining/deepspeed/ds_z3_config.json b/examples/industrial_data_pretraining/deepspeed/ds_z3_config.json
new file mode 100644
index 0000000..ccf9560
--- /dev/null
+++ b/examples/industrial_data_pretraining/deepspeed/ds_z3_config.json
@@ -0,0 +1,30 @@
+{
+ "train_batch_size": "auto",
+ "train_micro_batch_size_per_gpu": "auto",
+ "gradient_accumulation_steps": "auto",
+ "gradient_clipping": "auto",
+ "zero_allow_untested_optimizer": true,
+ "fp16": {
+ "enabled": "auto",
+ "loss_scale": 0,
+ "loss_scale_window": 1000,
+ "initial_scale_power": 16,
+ "hysteresis": 2,
+ "min_loss_scale": 1
+ },
+ "bf16": {
+ "enabled": "auto"
+ },
+ "zero_optimization": {
+ "stage": 3,
+ "overlap_comm": true,
+ "contiguous_gradients": true,
+ "sub_group_size": 1e9,
+ "reduce_bucket_size": "auto",
+ "stage3_prefetch_bucket_size": "auto",
+ "stage3_param_persistence_threshold": "auto",
+ "stage3_max_live_parameters": 1e9,
+ "stage3_max_reuse_distance": 1e9,
+ "stage3_gather_16bit_weights_on_model_save": true
+ }
+}
\ No newline at end of file
diff --git a/examples/industrial_data_pretraining/deepspeed/ds_z3_offload_config.json b/examples/industrial_data_pretraining/deepspeed/ds_z3_offload_config.json
new file mode 100644
index 0000000..026aabb
--- /dev/null
+++ b/examples/industrial_data_pretraining/deepspeed/ds_z3_offload_config.json
@@ -0,0 +1,38 @@
+{
+ "train_batch_size": "auto",
+ "train_micro_batch_size_per_gpu": "auto",
+ "gradient_accumulation_steps": "auto",
+ "gradient_clipping": "auto",
+ "zero_allow_untested_optimizer": true,
+ "fp16": {
+ "enabled": "auto",
+ "loss_scale": 0,
+ "loss_scale_window": 1000,
+ "initial_scale_power": 16,
+ "hysteresis": 2,
+ "min_loss_scale": 1
+ },
+ "bf16": {
+ "enabled": "auto"
+ },
+ "zero_optimization": {
+ "stage": 3,
+ "offload_optimizer": {
+ "device": "cpu",
+ "pin_memory": true
+ },
+ "offload_param": {
+ "device": "cpu",
+ "pin_memory": true
+ },
+ "overlap_comm": true,
+ "contiguous_gradients": true,
+ "sub_group_size": 1e9,
+ "reduce_bucket_size": "auto",
+ "stage3_prefetch_bucket_size": "auto",
+ "stage3_param_persistence_threshold": "auto",
+ "stage3_max_live_parameters": 1e9,
+ "stage3_max_reuse_distance": 1e9,
+ "stage3_gather_16bit_weights_on_model_save": true
+ }
+}
\ No newline at end of file
diff --git a/funasr/models/llm_asr/model.py b/funasr/models/llm_asr/model.py
index 411b59d..0d9bf7f 100644
--- a/funasr/models/llm_asr/model.py
+++ b/funasr/models/llm_asr/model.py
@@ -476,14 +476,8 @@
# audio_adaptor
encoder_out, encoder_out_lens = self.audio_adaptor(encoder_out, encoder_out_lens)
- input_ids[input_ids == -1] = 0
- input_ids[input_ids == -100] = 0
- if hasattr(self.llm.model, "embed_tokens"):
- inputs_embeds = self.llm.model.embed_tokens(input_ids)
- elif hasattr(self.llm.model.model, "embed_tokens"):
- inputs_embeds = self.llm.model.model.embed_tokens(input_ids)
- else:
- inputs_embeds = self.llm.model.model.model.embed_tokens(input_ids)
+ input_ids[input_ids < 0] = 0
+ inputs_embeds = self.llm.model.get_input_embeddings()(input_ids)
batch_size, token_num, dims = inputs_embeds.shape
_, l, _ = encoder_out.shape
@@ -492,7 +486,7 @@
inputs_embeds[batch_idx, fbank_beg_idx : fbank_beg_idx + l, :] = encoder_out[
batch_idx, :l, :
]
-
+ labels_ids[labels_ids == -1] = -100
model_outputs = self.llm(
inputs_embeds=inputs_embeds, attention_mask=attention_mask, labels=labels_ids
)
--
Gitblit v1.9.1