From 8f596af4be1c2e5c4e4b4a7008ba96f412d40fca Mon Sep 17 00:00:00 2001
From: 游雁 <zhifu.gzf@alibaba-inc.com>
Date: 星期一, 29 四月 2024 14:32:43 +0800
Subject: [PATCH] batch
---
funasr/models/rwkv_bat/rwkv_feed_forward.py | 22 +++++++---------------
1 files changed, 7 insertions(+), 15 deletions(-)
diff --git a/funasr/models/rwkv_bat/rwkv_feed_forward.py b/funasr/models/rwkv_bat/rwkv_feed_forward.py
index ddb4285..a8056bf 100644
--- a/funasr/models/rwkv_bat/rwkv_feed_forward.py
+++ b/funasr/models/rwkv_bat/rwkv_feed_forward.py
@@ -1,14 +1,10 @@
-"""Feed-forward (channel mixing) module for RWKV block.
-
-Based/Modified from https://github.com/BlinkDL/RWKV-LM/blob/main/RWKV-v4/src/model.py
-
-Some variables are renamed according to https://github.com/huggingface/transformers/blob/main/src/transformers/models/rwkv/modeling_rwkv.py.
-
-""" # noqa
-
-from typing import List, Optional, Tuple
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+# MIT License (https://opensource.org/licenses/MIT)
import torch
+from typing import List, Optional, Tuple
class FeedForward(torch.nn.Module):
@@ -76,14 +72,10 @@
state: Decoder hidden state. [5 x (B, 1, size, N)]
"""
- shifted_x = (
- self.time_shift(x) if state is None else state[0][..., self.block_id]
- )
+ shifted_x = self.time_shift(x) if state is None else state[0][..., self.block_id]
key = x * self.time_mix_key + shifted_x * (1 - self.time_mix_key)
- receptance = x * self.time_mix_receptance + shifted_x * (
- 1 - self.time_mix_receptance
- )
+ receptance = x * self.time_mix_receptance + shifted_x * (1 - self.time_mix_receptance)
key = torch.square(torch.relu(self.proj_key(key)))
value = self.proj_value(self.dropout(key))
--
Gitblit v1.9.1