From 86f2905ae8062f3a9bb684c35fbd41f228cb4c66 Mon Sep 17 00:00:00 2001
From: 游雁 <zhifu.gzf@alibaba-inc.com>
Date: 星期二, 30 四月 2024 12:53:40 +0800
Subject: [PATCH] Merge branch 'dev_gzf_exp' of github.com:alibaba-damo-academy/FunASR into dev_gzf_exp merge
---
runtime/java/java_http2ws_src/http/src/main/java/com/example/funasr_java_client/Servcvice/impl/RecognitionServiceImpl.java | 100 ++++++++++++++
runtime/java/java_http2ws_src/http/src/main/java/com/example/funasr_java_client/Servcvice/RecognitionService.java | 19 ++
runtime/java/java_http2ws_src/http/src/main/java/com/example/funasr_java_client/FunasrJavaClientApplication.java | 20 ++
runtime/java/java_http2ws_src/http/src/main/java/com/example/funasr_java_client/Servcvice/impl/RecognitionServiceImpl2.java | 112 ++++++++++++++++
funasr/models/ct_transformer/model.py | 4
runtime/java/java_http2ws_src/http/src/Readme.md | 14 ++
runtime/java/java_http2ws_src/http/src/main/java/com/example/funasr_java_client/RecognitionController.java | 36 +++++
runtime/java/java_http2ws_src/http/src/main/java/com/example/funasr_java_client/Servcvice/RecognitionService2.java | 18 ++
runtime/java/java_http2ws_src/http/src/main/java/com/example/funasr_java_client/WebSocketClient.java | 63 +++++++++
runtime/java/java_http2ws_src/http/src/main/resources/application.yml | 21 +++
runtime/java/java_http2ws_src/http/src/main/resources/application.properties | 2
11 files changed, 406 insertions(+), 3 deletions(-)
diff --git a/funasr/models/ct_transformer/model.py b/funasr/models/ct_transformer/model.py
index 534c7ab..abc5dfd 100644
--- a/funasr/models/ct_transformer/model.py
+++ b/funasr/models/ct_transformer/model.py
@@ -287,9 +287,7 @@
# y, _ = self.wrapped_model(**data)
y, _ = self.punc_forward(**data)
_, indices = y.view(-1, y.shape[-1]).topk(1, dim=1)
- punctuations = indices
- if indices.size()[0] != 1:
- punctuations = torch.squeeze(indices)
+ punctuations = torch.squeeze(indices, dim=1)
assert punctuations.size()[0] == len(mini_sentence)
# Search for the last Period/QuestionMark as cache
diff --git a/runtime/java/java_http2ws_src/http/src/Readme.md b/runtime/java/java_http2ws_src/http/src/Readme.md
new file mode 100644
index 0000000..ef03d1d
--- /dev/null
+++ b/runtime/java/java_http2ws_src/http/src/Readme.md
@@ -0,0 +1,14 @@
+dependencies {
+ implementation("org.springframework.boot:spring-boot-starter-web")
+ implementation("org.json:json:20240303")
+ implementation("org.springframework.boot:spring-boot-starter-websocket")
+}
+
+
+浣跨敤鎺ュ彛娴嬭瘯宸ュ叿 form-data鏍煎紡浼犲叆鏂囦欢 杩斿洖娴嬭瘯鎴愬姛鍗宠繍琛屾垚鍔�
+
+榛樿璁块棶璺緞:
+ io璺緞: http://localhost:8081/recognition/testIO
+ nio璺緞: http://localhost:8081/recognition/testNIO
+
+application.yml涓彲鏍规嵁鑷韩闇�瑕佷慨鏀瑰搴旀ā鍨嬪弬鏁�
\ No newline at end of file
diff --git a/runtime/java/java_http2ws_src/http/src/main/java/com/example/funasr_java_client/FunasrJavaClientApplication.java b/runtime/java/java_http2ws_src/http/src/main/java/com/example/funasr_java_client/FunasrJavaClientApplication.java
new file mode 100644
index 0000000..5d41892
--- /dev/null
+++ b/runtime/java/java_http2ws_src/http/src/main/java/com/example/funasr_java_client/FunasrJavaClientApplication.java
@@ -0,0 +1,20 @@
+package com.example.funasr_java_client;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+/**
+ *
+ * @author Virgil Qiu
+ * @since 2024/04/24
+ *
+ */
+
+@SpringBootApplication
+public class FunasrJavaClientApplication {
+
+ public static void main(String[] args) {
+ SpringApplication.run(FunasrJavaClientApplication.class, args);
+ }
+
+}
diff --git a/runtime/java/java_http2ws_src/http/src/main/java/com/example/funasr_java_client/RecognitionController.java b/runtime/java/java_http2ws_src/http/src/main/java/com/example/funasr_java_client/RecognitionController.java
new file mode 100644
index 0000000..df60bea
--- /dev/null
+++ b/runtime/java/java_http2ws_src/http/src/main/java/com/example/funasr_java_client/RecognitionController.java
@@ -0,0 +1,36 @@
+package com.example.funasr_java_client.Servcvice;
+
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.*;
+import org.springframework.web.multipart.MultipartFile;
+
+import java.io.IOException;
+import java.util.concurrent.ExecutionException;
+
+/**
+ *
+ * @author Virgil Qiu
+ * @since 2024/04/24
+ *
+ */
+@RestController
+@RequestMapping("/recognition")
+public class RecognitionController {
+
+ private final RecognitionService recognitionService;
+
+ public RecognitionController(RecognitionService recognitionService) {
+ this.recognitionService = recognitionService;
+ }
+ @PostMapping("/testNIO")
+ public String testIO(@RequestParam MultipartFile file) throws IOException, ExecutionException, InterruptedException {
+ recognitionService.recognition(file);
+ return "娴嬭瘯鎴愬姛";
+ }
+
+ @PostMapping("/testIO")
+ public String testNIO(@RequestParam MultipartFile file) throws IOException, ExecutionException, InterruptedException {
+ recognitionService.recognition(file);
+ return "娴嬭瘯鎴愬姛";
+ }
+}
diff --git a/runtime/java/java_http2ws_src/http/src/main/java/com/example/funasr_java_client/Servcvice/RecognitionService.java b/runtime/java/java_http2ws_src/http/src/main/java/com/example/funasr_java_client/Servcvice/RecognitionService.java
new file mode 100644
index 0000000..44d9142
--- /dev/null
+++ b/runtime/java/java_http2ws_src/http/src/main/java/com/example/funasr_java_client/Servcvice/RecognitionService.java
@@ -0,0 +1,19 @@
+package com.example.funasr_java_client.Servcvice;
+
+import org.springframework.web.multipart.MultipartFile;
+
+import java.io.IOException;
+import java.util.concurrent.ExecutionException;
+
+
+/**
+ *
+ * @author Virgil Qiu
+ * @since 2024/04/24
+ *
+ */
+
+public interface RecognitionService {
+
+ Object recognition(MultipartFile file) throws IOException, ExecutionException, InterruptedException;
+}
diff --git a/runtime/java/java_http2ws_src/http/src/main/java/com/example/funasr_java_client/Servcvice/RecognitionService2.java b/runtime/java/java_http2ws_src/http/src/main/java/com/example/funasr_java_client/Servcvice/RecognitionService2.java
new file mode 100644
index 0000000..f08245c
--- /dev/null
+++ b/runtime/java/java_http2ws_src/http/src/main/java/com/example/funasr_java_client/Servcvice/RecognitionService2.java
@@ -0,0 +1,18 @@
+package com.example.funasr_java_client.Servcvice;
+
+import org.springframework.web.multipart.MultipartFile;
+
+import java.io.IOException;
+import java.util.concurrent.ExecutionException;
+
+/**
+ *
+ * @author Virgil Qiu
+ * @since 2024/04/24
+ *
+ */
+
+public interface RecognitionService2 {
+
+ Object recognition(MultipartFile file) throws IOException, ExecutionException, InterruptedException;
+}
diff --git a/runtime/java/java_http2ws_src/http/src/main/java/com/example/funasr_java_client/Servcvice/impl/RecognitionServiceImpl.java b/runtime/java/java_http2ws_src/http/src/main/java/com/example/funasr_java_client/Servcvice/impl/RecognitionServiceImpl.java
new file mode 100644
index 0000000..af589a6
--- /dev/null
+++ b/runtime/java/java_http2ws_src/http/src/main/java/com/example/funasr_java_client/Servcvice/impl/RecognitionServiceImpl.java
@@ -0,0 +1,100 @@
+package com.example.funasr_java_client.Servcvice.impl;
+
+import com.example.funasr_java_client.Servcvice.RecognitionService;
+import com.example.funasr_java_client.WebSocketClient;
+import org.json.JSONObject;
+import org.springframework.stereotype.Service;
+import org.springframework.web.multipart.MultipartFile;
+import org.springframework.web.socket.BinaryMessage;
+import org.springframework.web.socket.TextMessage;
+import org.springframework.web.socket.WebSocketSession;
+import org.springframework.web.socket.client.standard.StandardWebSocketClient;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.concurrent.ExecutionException;
+
+/**
+ *
+ * @author Virgil Qiu
+ * @since 2024/04/24
+ *
+ */
+@Service
+public class RecognitionServiceImpl implements RecognitionService {
+ @Value("${parameters.fileUrl}")
+ private String fileUrl;
+ @Value("${parameters.model}")
+ private String model;
+ @Value("${parameters.hotWords}")
+ private String hotWords;
+ @Value("${parameters.serverIpPort}")
+ private String serverIpPort;
+ @Override
+ public Object recognition(MultipartFile file) throws IOException, ExecutionException, InterruptedException {
+ if (file.isEmpty()) {
+ return "0"; // 鏂囦欢涓虹┖锛岃繑鍥炵壒娈婂��
+ }
+
+
+ String originalFilename = file.getOriginalFilename();
+ String[] parts = originalFilename.split("\\.");
+ String prefix = (parts.length > 0) ? parts[0] : originalFilename;
+ System.out.println(prefix);
+ String localFilePath = fileUrl + prefix + ".pcm";
+
+ File localFile = new File(localFilePath);
+
+ File destDir = localFile.getParentFile();
+ if (!destDir.exists() && !destDir.mkdirs()) {
+ throw new IOException("Unable to create destination directory: " + localFilePath);
+ }
+
+ file.transferTo(localFile);
+
+ WebSocketClient client = new WebSocketClient();
+ URI uri = URI.create(serverIpPort);
+ StandardWebSocketClient standardWebSocketClient = new StandardWebSocketClient();
+ WebSocketSession webSocketSession = standardWebSocketClient.execute(client, null, uri).get();
+
+
+ JSONObject configJson = new JSONObject();
+ configJson.put("mode", model);
+ configJson.put("wav_name", prefix);
+ configJson.put("wav_format", "pcm"); // 鏂囦欢鏍煎紡涓簆cm
+ configJson.put("is_speaking", true);
+ configJson.put("hotwords", hotWords");
+ configJson.put("itn", true);
+
+ // 鍙戦�侀厤缃弬鏁颁笌meta淇℃伅
+ webSocketSession.sendMessage(new TextMessage(configJson.toString()));
+
+ byte[] audioData;
+ try {
+ audioData = Files.readAllBytes(Paths.get(localFilePath));
+ } catch (IOException e) {
+ System.err.println("Error reading file: " + e.getMessage());
+ e.printStackTrace();
+ return "Error reading audio file"; // Return an appropriate error message or throw an exception
+ }
+
+ ByteBuffer audioByteBuffer = ByteBuffer.wrap(audioData);
+
+ BinaryMessage binaryMessage = new BinaryMessage(audioByteBuffer);
+ webSocketSession.sendMessage(binaryMessage);
+
+ // 鍙戦�侀煶棰戠粨鏉熸爣蹇�
+ JSONObject endMarkerJson = new JSONObject();
+ endMarkerJson.put("is_speaking", false);
+ webSocketSession.sendMessage(new TextMessage(endMarkerJson.toString()));
+
+ // TODO: 瀹炵幇鎺ユ敹骞跺鐞嗘湇鍔＄杩斿洖鐨勮瘑鍒粨鏋�
+
+ return "test";
+
+ }
+}
diff --git a/runtime/java/java_http2ws_src/http/src/main/java/com/example/funasr_java_client/Servcvice/impl/RecognitionServiceImpl2.java b/runtime/java/java_http2ws_src/http/src/main/java/com/example/funasr_java_client/Servcvice/impl/RecognitionServiceImpl2.java
new file mode 100644
index 0000000..4cbdafa
--- /dev/null
+++ b/runtime/java/java_http2ws_src/http/src/main/java/com/example/funasr_java_client/Servcvice/impl/RecognitionServiceImpl2.java
@@ -0,0 +1,112 @@
+package com.example.funasr_java_client.Servcvice.impl;
+
+import com.example.funasr_java_client.Servcvice.RecognitionService;
+import com.example.funasr_java_client.Servcvice.RecognitionService2;
+import com.example.funasr_java_client.WebSocketClient;
+import org.json.JSONObject;
+import org.springframework.stereotype.Service;
+import org.springframework.web.multipart.MultipartFile;
+import org.springframework.web.socket.BinaryMessage;
+import org.springframework.web.socket.TextMessage;
+import org.springframework.web.socket.WebSocketSession;
+import org.springframework.web.socket.client.standard.StandardWebSocketClient;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.concurrent.ExecutionException;
+
+/**
+ *
+ * @author Virgil Qiu
+ * @since 2024/04/24
+ *
+ */
+
+@Service
+public class RecognitionServiceImpl2 implements RecognitionService2 {
+ @Value("${parameters.fileUrl}")
+ private String fileUrl;
+ @Value("${parameters.model}")
+ private String model;
+ @Value("${parameters.hotWords}")
+ private String hotWords;
+ @Value("${parameters.serverIpPort}")
+ private String serverIpPort;
+ @Override
+ public Object recognition(MultipartFile file) throws IOException, ExecutionException, InterruptedException {
+ if (file.isEmpty()) {
+ return "0"; // 鏂囦欢涓虹┖锛岃繑鍥炵壒娈婂��
+ }
+
+
+ String originalFilename = file.getOriginalFilename();
+ String[] parts = originalFilename.split("\\.");
+ String prefix = (parts.length > 0) ? parts[0] : originalFilename;
+ System.out.println(prefix);
+ String localFilePath = fileUrl + prefix + ".pcm";
+
+ File localFile = new File(localFilePath);
+
+ File destDir = localFile.getParentFile();
+ if (!destDir.exists() && !destDir.mkdirs()) {
+ throw new IOException("Unable to create destination directory: " + localFilePath);
+ }
+
+ file.transferTo(localFile);
+
+ WebSocketClient client = new WebSocketClient();
+ URI uri = URI.create(serverIpPort);
+ StandardWebSocketClient standardWebSocketClient = new StandardWebSocketClient();
+ WebSocketSession webSocketSession = standardWebSocketClient.execute(client, null, uri).get();
+
+
+ JSONObject configJson = new JSONObject();
+ configJson.put("mode", model);
+ configJson.put("wav_name", prefix);
+ configJson.put("wav_format", "pcm"); // 鏂囦欢鏍煎紡涓簆cm
+ configJson.put("is_speaking", true);
+ configJson.put("hotwords", hotWords);
+ configJson.put("itn", true);
+
+ // 鍙戦�侀厤缃弬鏁颁笌meta淇℃伅
+ webSocketSession.sendMessage(new TextMessage(configJson.toString()));
+
+
+ try (FileInputStream fis = new FileInputStream(localFilePath)) {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ byte[] buffer = new byte[1024];
+ int bytesRead;
+ while ((bytesRead = fis.read(buffer)) != -1) {
+ baos.write(buffer, 0, bytesRead);
+ }
+
+ // 灏嗘墍鏈夎鍙栫殑瀛楄妭鍚堝苟鍒颁竴涓瓧鑺傛暟缁勪腑
+ byte[] completeData = baos.toByteArray();
+
+ // 浣跨敤瀛楄妭鏁扮粍鍒涘缓BinaryMessage瀹炰緥
+ BinaryMessage binaryMessage = new BinaryMessage(completeData);
+ webSocketSession.sendMessage(binaryMessage);
+ // 浣跨敤鎴栧彂閫乥inaryMessage...
+ } catch (IOException e) {
+ System.err.println("Error reading file: " + e.getMessage());
+ e.printStackTrace();
+ }
+
+
+ // 鍙戦�侀煶棰戠粨鏉熸爣蹇�
+ JSONObject endMarkerJson = new JSONObject();
+ endMarkerJson.put("is_speaking", false);
+ webSocketSession.sendMessage(new TextMessage(endMarkerJson.toString()));
+
+ // TODO: 瀹炵幇鎺ユ敹骞跺鐞嗘湇鍔＄杩斿洖鐨勮瘑鍒粨鏋�
+
+ return "test";
+
+ }
+}
diff --git a/runtime/java/java_http2ws_src/http/src/main/java/com/example/funasr_java_client/WebSocketClient.java b/runtime/java/java_http2ws_src/http/src/main/java/com/example/funasr_java_client/WebSocketClient.java
new file mode 100644
index 0000000..1857374
--- /dev/null
+++ b/runtime/java/java_http2ws_src/http/src/main/java/com/example/funasr_java_client/WebSocketClient.java
@@ -0,0 +1,63 @@
+package com.example.funasr_java_client;
+
+import org.springframework.stereotype.Component;
+import org.springframework.web.socket.*;
+
+
+/**
+ *
+ * @author Virgil Qiu
+ * @since 2024/04/24
+ *
+ */
+
+
+@Component
+public class WebSocketClient implements WebSocketHandler {
+
+ private WebSocketSession session;
+
+ @Override
+ public void afterConnectionEstablished(WebSocketSession session) throws Exception {
+ this.session = session;
+ System.out.println("WebSocket connection established.");
+ }
+
+ @Override
+ public void handleMessage(WebSocketSession session, WebSocketMessage<?> message) throws Exception {
+ if (message instanceof TextMessage) {
+ String receivedMessage = ((TextMessage) message).getPayload();
+ System.out.println("Received message from server: " + receivedMessage);
+ // 鍦ㄨ繖閲屽鐞嗘帴鏀跺埌鐨勬秷鎭�
+ }
+ }
+
+ @Override
+ public void handleTransportError(WebSocketSession session, Throwable exception) throws Exception {
+ System.err.println("WebSocket transport error: " + exception.getMessage());
+ session.close(CloseStatus.SERVER_ERROR);
+ }
+
+ @Override
+ public void afterConnectionClosed(WebSocketSession session, CloseStatus status) throws Exception {
+ System.out.println("WebSocket connection closed with status: " + status);
+ }
+
+ @Override
+ public boolean supportsPartialMessages() {
+ return false;
+ }
+
+ public void sendMessage(String message) {
+ if (session != null && session.isOpen()) {
+ try {
+ session.sendMessage(new TextMessage(message));
+ System.out.println("Sent message to server: " + message);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ } else {
+ System.err.println("WebSocket session is not open. Cannot send message.");
+ }
+ }
+}
diff --git a/runtime/java/java_http2ws_src/http/src/main/resources/application.properties b/runtime/java/java_http2ws_src/http/src/main/resources/application.properties
new file mode 100644
index 0000000..b21adeb
--- /dev/null
+++ b/runtime/java/java_http2ws_src/http/src/main/resources/application.properties
@@ -0,0 +1,2 @@
+spring.application.name=funasr_java_client
+server.port=8081
diff --git a/runtime/java/java_http2ws_src/http/src/main/resources/application.yml b/runtime/java/java_http2ws_src/http/src/main/resources/application.yml
new file mode 100644
index 0000000..5f179bf
--- /dev/null
+++ b/runtime/java/java_http2ws_src/http/src/main/resources/application.yml
@@ -0,0 +1,21 @@
+#/**
+# *
+# * @author Virgil Qiu
+# * @since 2024/04/24
+# *
+# */
+
+spring:
+ application:
+ name: java_http_client
+server:
+ port: 8081
+
+
+parameters:
+ model: "offline" #绂荤嚎妯″瀷涓轰緥
+ hotWords: "{\"鑷畾涔塡":20,\"鐑瘝\":20,\"璁剧疆\":30}"
+ fileUrl: "E:/EI/Audio"
+ serverIpPort: "ws://your_funasr_ip:port"
+
+
--
Gitblit v1.9.1