From d80ac2fd2df4e7fb8a28acfa512bb11472b5cc99 Mon Sep 17 00:00:00 2001
From: liugz18 <57401541+liugz18@users.noreply.github.com>
Date: 星期四, 18 七月 2024 21:34:55 +0800
Subject: [PATCH] Rename 'res' in line 514 to avoid with naming conflict with line 365
---
funasr/models/lora/layers.py | 89 +++++++++++++++++++++++++++-----------------
1 files changed, 54 insertions(+), 35 deletions(-)
diff --git a/funasr/models/lora/layers.py b/funasr/models/lora/layers.py
index 76f046c..b91253d 100644
--- a/funasr/models/lora/layers.py
+++ b/funasr/models/lora/layers.py
@@ -9,7 +9,8 @@
import math
from typing import Optional, List
-class LoRALayer():
+
+class LoRALayer:
def __init__(
self,
r: int,
@@ -20,7 +21,7 @@
self.r = r
self.lora_alpha = lora_alpha
# Optional dropout
- if lora_dropout > 0.:
+ if lora_dropout > 0.0:
self.lora_dropout = nn.Dropout(p=lora_dropout)
else:
self.lora_dropout = lambda x: x
@@ -41,8 +42,9 @@
**kwargs
):
nn.Embedding.__init__(self, num_embeddings, embedding_dim, **kwargs)
- LoRALayer.__init__(self, r=r, lora_alpha=lora_alpha, lora_dropout=0,
- merge_weights=merge_weights)
+ LoRALayer.__init__(
+ self, r=r, lora_alpha=lora_alpha, lora_dropout=0, merge_weights=merge_weights
+ )
# Actual trainable parameters
if r > 0:
self.lora_A = nn.Parameter(self.weight.new_zeros((r, num_embeddings)))
@@ -54,7 +56,7 @@
def reset_parameters(self):
nn.Embedding.reset_parameters(self)
- if hasattr(self, 'lora_A'):
+ if hasattr(self, "lora_A"):
# initialize A the same way as the default for nn.Linear and B to zero
nn.init.zeros_(self.lora_A)
nn.init.normal_(self.lora_B)
@@ -80,8 +82,13 @@
result = nn.Embedding.forward(self, x)
if self.r > 0:
after_A = F.embedding(
- x, self.lora_A.T, self.padding_idx, self.max_norm,
- self.norm_type, self.scale_grad_by_freq, self.sparse
+ x,
+ self.lora_A.T,
+ self.padding_idx,
+ self.max_norm,
+ self.norm_type,
+ self.scale_grad_by_freq,
+ self.sparse,
)
result += (after_A @ self.lora_B.T) * self.scaling
return result
@@ -97,14 +104,15 @@
out_features: int,
r: int = 0,
lora_alpha: int = 1,
- lora_dropout: float = 0.,
- fan_in_fan_out: bool = False, # Set this to True if the layer to replace stores weight like (fan_in, fan_out)
+ lora_dropout: float = 0.0,
+ fan_in_fan_out: bool = False, # Set this to True if the layer to replace stores weight like (fan_in, fan_out)
merge_weights: bool = True,
**kwargs
):
nn.Linear.__init__(self, in_features, out_features, **kwargs)
- LoRALayer.__init__(self, r=r, lora_alpha=lora_alpha, lora_dropout=lora_dropout,
- merge_weights=merge_weights)
+ LoRALayer.__init__(
+ self, r=r, lora_alpha=lora_alpha, lora_dropout=lora_dropout, merge_weights=merge_weights
+ )
self.fan_in_fan_out = fan_in_fan_out
# Actual trainable parameters
@@ -120,7 +128,7 @@
def reset_parameters(self):
nn.Linear.reset_parameters(self)
- if hasattr(self, 'lora_A'):
+ if hasattr(self, "lora_A"):
# initialize A the same way as the default for nn.Linear and B to zero
nn.init.kaiming_uniform_(self.lora_A, a=math.sqrt(5))
nn.init.zeros_(self.lora_B)
@@ -128,6 +136,7 @@
def train(self, mode: bool = True):
def T(w):
return w.T if self.fan_in_fan_out else w
+
nn.Linear.train(self, mode)
if self.merge_weights and self.merged:
# Make sure that the weights are not merged
@@ -138,6 +147,7 @@
def eval(self):
def T(w):
return w.T if self.fan_in_fan_out else w
+
nn.Linear.eval(self)
if self.merge_weights and not self.merged:
# Merge the weights and mark it
@@ -148,6 +158,7 @@
def forward(self, x: torch.Tensor):
def T(w):
return w.T if self.fan_in_fan_out else w
+
if self.r > 0 and not self.merged:
result = F.linear(x, T(self.weight), bias=self.bias)
if self.r > 0:
@@ -165,33 +176,34 @@
out_features: int,
r: int = 0,
lora_alpha: int = 1,
- lora_dropout: float = 0.,
+ lora_dropout: float = 0.0,
enable_lora: List[bool] = [False],
fan_in_fan_out: bool = False,
merge_weights: bool = True,
**kwargs
):
nn.Linear.__init__(self, in_features, out_features, **kwargs)
- LoRALayer.__init__(self, r=r, lora_alpha=lora_alpha, lora_dropout=lora_dropout,
- merge_weights=merge_weights)
- assert out_features % len(enable_lora) == 0, \
- 'The length of enable_lora must divide out_features'
+ LoRALayer.__init__(
+ self, r=r, lora_alpha=lora_alpha, lora_dropout=lora_dropout, merge_weights=merge_weights
+ )
+ assert (
+ out_features % len(enable_lora) == 0
+ ), "The length of enable_lora must divide out_features"
self.enable_lora = enable_lora
self.fan_in_fan_out = fan_in_fan_out
# Actual trainable parameters
if r > 0 and any(enable_lora):
- self.lora_A = nn.Parameter(
- self.weight.new_zeros((r * sum(enable_lora), in_features)))
+ self.lora_A = nn.Parameter(self.weight.new_zeros((r * sum(enable_lora), in_features)))
self.lora_B = nn.Parameter(
self.weight.new_zeros((out_features // len(enable_lora) * sum(enable_lora), r))
- ) # weights for Conv1D with groups=sum(enable_lora)
+ ) # weights for Conv1D with groups=sum(enable_lora)
self.scaling = self.lora_alpha / self.r
# Freezing the pre-trained weight matrix
self.weight.requires_grad = False
# Compute the indices
- self.lora_ind = self.weight.new_zeros(
- (out_features, ), dtype=torch.bool
- ).view(len(enable_lora), -1)
+ self.lora_ind = self.weight.new_zeros((out_features,), dtype=torch.bool).view(
+ len(enable_lora), -1
+ )
self.lora_ind[enable_lora, :] = True
self.lora_ind = self.lora_ind.view(-1)
self.reset_parameters()
@@ -200,7 +212,7 @@
def reset_parameters(self):
nn.Linear.reset_parameters(self)
- if hasattr(self, 'lora_A'):
+ if hasattr(self, "lora_A"):
# initialize A the same way as the default for nn.Linear and B to zero
nn.init.kaiming_uniform_(self.lora_A, a=math.sqrt(5))
nn.init.zeros_(self.lora_B)
@@ -216,6 +228,7 @@
def train(self, mode: bool = True):
def T(w):
return w.T if self.fan_in_fan_out else w
+
nn.Linear.train(self, mode)
if self.merge_weights and self.merged:
# Make sure that the weights are not merged
@@ -223,7 +236,7 @@
delta_w = F.conv1d(
self.lora_A.data.unsqueeze(0),
self.lora_B.data.unsqueeze(-1),
- groups=sum(self.enable_lora)
+ groups=sum(self.enable_lora),
).squeeze(0)
self.weight.data -= self.zero_pad(T(delta_w * self.scaling))
self.merged = False
@@ -231,6 +244,7 @@
def eval(self):
def T(w):
return w.T if self.fan_in_fan_out else w
+
nn.Linear.eval(self)
if self.merge_weights and not self.merged:
# Merge the weights and mark it
@@ -238,7 +252,7 @@
delta_w = F.conv1d(
self.lora_A.data.unsqueeze(0),
self.lora_B.data.unsqueeze(-1),
- groups=sum(self.enable_lora)
+ groups=sum(self.enable_lora),
).squeeze(0)
self.weight.data += self.zero_pad(T(delta_w * self.scaling))
self.merged = True
@@ -246,6 +260,7 @@
def forward(self, x: torch.Tensor):
def T(w):
return w.T if self.fan_in_fan_out else w
+
if self.merged:
return F.linear(x, T(self.weight), bias=self.bias)
else:
@@ -255,7 +270,7 @@
after_B = F.conv1d(
after_A.transpose(-2, -1),
self.lora_B.unsqueeze(-1),
- groups=sum(self.enable_lora)
+ groups=sum(self.enable_lora),
).transpose(-2, -1)
result += self.zero_pad(after_B) * self.scaling
return result
@@ -270,21 +285,22 @@
kernel_size: int,
r: int = 0,
lora_alpha: int = 1,
- lora_dropout: float = 0.,
+ lora_dropout: float = 0.0,
merge_weights: bool = True,
**kwargs
):
nn.Conv2d.__init__(self, in_channels, out_channels, kernel_size, **kwargs)
- LoRALayer.__init__(self, r=r, lora_alpha=lora_alpha, lora_dropout=lora_dropout,
- merge_weights=merge_weights)
+ LoRALayer.__init__(
+ self, r=r, lora_alpha=lora_alpha, lora_dropout=lora_dropout, merge_weights=merge_weights
+ )
assert type(kernel_size) is int
# Actual trainable parameters
if r > 0:
self.lora_A = nn.Parameter(
- self.weight.new_zeros((r*kernel_size, in_channels*kernel_size))
+ self.weight.new_zeros((r * kernel_size, in_channels * kernel_size))
)
self.lora_B = nn.Parameter(
- self.weight.new_zeros((out_channels*kernel_size, r*kernel_size))
+ self.weight.new_zeros((out_channels * kernel_size, r * kernel_size))
)
self.scaling = self.lora_alpha / self.r
# Freezing the pre-trained weight matrix
@@ -293,7 +309,7 @@
def reset_parameters(self):
nn.Conv2d.reset_parameters(self)
- if hasattr(self, 'lora_A'):
+ if hasattr(self, "lora_A"):
# initialize A the same way as the default for nn.Linear and B to zero
nn.init.kaiming_uniform_(self.lora_A, a=math.sqrt(5))
nn.init.zeros_(self.lora_B)
@@ -317,7 +333,10 @@
return F.conv2d(
x,
self.weight + (self.lora_B @ self.lora_A).view(self.weight.shape) * self.scaling,
- self.bias, self.stride, self.padding, self.dilation, self.groups
+ self.bias,
+ self.stride,
+ self.padding,
+ self.dilation,
+ self.groups,
)
return nn.Conv2d.forward(self, x)
-
--
Gitblit v1.9.1