package com.custom.webui.service.impl;
|
|
import com.custom.webui.config.props.LlmProperties;
|
import com.custom.webui.exception.BusinessException;
|
import com.custom.webui.service.LlmService;
|
import com.custom.webui.util.ReasoningLeakFilter;
|
import com.custom.webui.util.ThinkingTagStripper;
|
import com.fasterxml.jackson.databind.JsonNode;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.node.ArrayNode;
|
import com.fasterxml.jackson.databind.node.ObjectNode;
|
import lombok.extern.slf4j.Slf4j;
|
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
import org.springframework.stereotype.Service;
|
|
import java.io.BufferedReader;
|
import java.io.IOException;
|
import java.io.InputStream;
|
import java.io.InputStreamReader;
|
import java.net.HttpURLConnection;
|
import java.net.URI;
|
import java.nio.charset.StandardCharsets;
|
import java.util.List;
|
import java.util.Map;
|
|
/**
|
* OpenAI 兼容协议实现。
|
*
|
* <p><b>为什么这一个类能覆盖几乎全部推理服务</b>:
|
* {@code POST {baseUrl}/chat/completions} 已经成为事实标准,
|
* vLLM、LM Studio、SGLang、Xinference、Ollama(/v1 端点)、
|
* one-api/new-api 网关,以及 OpenAI / DeepSeek / Kimi / 通义 / GLM
|
* 云端 API 都实现了它。换服务只需改 {@code baseUrl} 与 {@code model}。
|
*
|
* <p>与 {@link OllamaLlmService} 的关键差异:
|
* <table border="1">
|
* <caption>协议差异</caption>
|
* <tr><th></th><th>Ollama 原生</th><th>OpenAI 兼容</th></tr>
|
* <tr><td>流式格式</td><td>NDJSON,每行一个 JSON</td><td>SSE,{@code data: {...}},以 {@code data: [DONE]} 结束</td></tr>
|
* <tr><td>增量位置</td><td>{@code message.content}</td><td>{@code choices[0].delta.content}</td></tr>
|
* <tr><td>上下文窗口参数</td><td>{@code options.num_ctx}</td><td>无对应项,只有 {@code max_tokens}(输出上限)</td></tr>
|
* <tr><td>思考内容</td><td>{@code message.thinking}</td><td>{@code delta.reasoning_content},或正文里的 {@code <think>} 标签</td></tr>
|
* </table>
|
*
|
* <p><b>思考内容的两种形态都要处理</b>:实测 vLLM 上同一模型,
|
* 流式时走 {@code delta.reasoning_content},非流式却把 {@code <think>}
|
* 标签写进 {@code content}。因此这里两条路都做了剥离,
|
* 否则整段推理会灌进聊天气泡并永久写进数据库。
|
*/
|
@Slf4j
|
@Service
|
@ConditionalOnProperty(name = "webui.llm.provider", havingValue = "openai-compat")
|
public class OpenAiCompatLlmService implements LlmService {
|
|
private static final String DONE_MARKER = "[DONE]";
|
|
private final LlmProperties properties;
|
private final ObjectMapper objectMapper;
|
|
public OpenAiCompatLlmService(LlmProperties properties, ObjectMapper objectMapper) {
|
this.properties = properties;
|
this.objectMapper = objectMapper;
|
log.info("LLM 实现: OpenAI 兼容, baseUrl={}, model={}",
|
properties.getOpenaiCompat().getBaseUrl(),
|
properties.getOpenaiCompat().getModel());
|
}
|
|
// ============================================================
|
// 流式
|
// ============================================================
|
|
@Override
|
public void streamChat(List<LlmMessage> messages, LlmOptions options, TokenConsumer consumer) {
|
HttpURLConnection conn = null;
|
// 统计变量声明在 try 之外:流结束后要打日志,作用域需覆盖整个方法
|
int totalLines = 0;
|
int dataLines = 0;
|
int parsedFrames = 0;
|
int contentFrames = 0;
|
// 用于「推理泄漏」检测:累积全部正文,结束时统一判定。
|
// 为什么不在流中实时判断:启发式规则需要看完整文本才准
|
//(单个分片可能只是 "Wait" 这样的普通词),过早拦截会误伤。
|
// 代价是开启该功能时正文要等流结束才推给前端,因此默认关闭。
|
StringBuilder leakProbe = new StringBuilder();
|
boolean stripLeak = properties.getOpenaiCompat().isStripReasoningLeak();
|
ThinkingTagStripper stripper = new ThinkingTagStripper(
|
properties.getOpenaiCompat().isIncludeThinking()
|
? (thinking) -> consumer.accept(thinking, true)
|
: null);
|
|
try {
|
conn = openConnection("/chat/completions",
|
properties.getOpenaiCompat().getTimeoutMillis());
|
writeBody(conn, buildRequestBody(messages, options, true));
|
|
int status = conn.getResponseCode();
|
if (status != 200) {
|
String err = readErrorBody(conn);
|
log.error("OpenAI 兼容服务返回非 200: status={}, body={}", status, err);
|
throw BusinessException.serverError(
|
"LLM 服务返回错误 (HTTP " + status + ")。地址: "
|
+ properties.getOpenaiCompat().getBaseUrl()
|
+ ",模型: " + resolveModel(options) + "。详情: " + err);
|
}
|
|
try (BufferedReader reader = new BufferedReader(
|
new InputStreamReader(conn.getInputStream(), StandardCharsets.UTF_8))) {
|
String line;
|
while ((line = reader.readLine()) != null) {
|
totalLines++;
|
// SSE 帧格式:以 "data: " 开头,空行分隔
|
if (line.isEmpty() || !line.startsWith("data:")) {
|
continue;
|
}
|
dataLines++;
|
String payload = line.substring("data:".length()).trim();
|
if (payload.isEmpty()) {
|
continue;
|
}
|
if (DONE_MARKER.equals(payload)) {
|
log.debug("收到 [DONE],结束流");
|
break;
|
}
|
|
JsonNode node;
|
try {
|
node = objectMapper.readTree(payload);
|
} catch (IOException parseError) {
|
log.warn("无法解析流式分片: {}", abbreviate(payload));
|
continue;
|
}
|
|
parsedFrames++;
|
if (node.hasNonNull("error")) {
|
throw BusinessException.serverError(
|
"LLM 服务错误: " + node.get("error").toString());
|
}
|
|
JsonNode delta = node.path("choices").path(0).path("delta");
|
if (delta.isMissingNode()) {
|
continue;
|
}
|
|
// 形态一:独立的推理字段(vLLM / DeepSeek 等)
|
for (String field : new String[]{"reasoning_content", "reasoning"}) {
|
JsonNode reasoning = delta.get(field);
|
if (reasoning != null && !reasoning.asText().isEmpty()
|
&& properties.getOpenaiCompat().isIncludeThinking()) {
|
consumer.accept(reasoning.asText(), true);
|
}
|
}
|
|
// 形态二:正文里夹带 <think> 标签(可能跨 chunk 被切开)
|
JsonNode content = delta.get("content");
|
if (content != null && !content.asText().isEmpty()) {
|
contentFrames++;
|
String visible = stripper.accept(content.asText());
|
if (!visible.isEmpty()) {
|
if (stripLeak) {
|
leakProbe.append(visible);
|
} else {
|
consumer.accept(visible, false);
|
}
|
}
|
}
|
}
|
}
|
|
log.info("流式读取完成: 总行数={} data行={} 可解析帧={} 含content帧={}",
|
totalLines, dataLines, parsedFrames, contentFrames);
|
|
// 流结束:把缓冲区剩余内容吐出来,避免标签前缀恰好结尾时丢字
|
String tail = stripper.flush();
|
if (!tail.isEmpty()) {
|
if (stripLeak) {
|
leakProbe.append(tail);
|
} else {
|
consumer.accept(tail, false);
|
}
|
}
|
|
// 开启推理泄漏过滤时,在这里统一判定并输出
|
if (stripLeak) {
|
String full = leakProbe.toString();
|
if (ReasoningLeakFilter.looksLikeReasoning(full)) {
|
log.warn("正文被判定为推理泄漏(模型把思考写进了 content),已丢弃 {} 字。"
|
+ "如需保留请关闭 webui.llm.openaiCompat.strip-reasoning-leak", full.length());
|
} else if (!full.isEmpty()) {
|
consumer.accept(full, false);
|
}
|
}
|
|
} catch (BusinessException e) {
|
throw e;
|
} catch (IOException e) {
|
log.error("调用 OpenAI 兼容服务失败", e);
|
throw BusinessException.serverError(
|
"无法连接 LLM 服务 " + properties.getOpenaiCompat().getBaseUrl()
|
+ "。请确认该服务已启动。原因: " + e.getMessage());
|
} finally {
|
if (conn != null) {
|
conn.disconnect();
|
}
|
}
|
}
|
|
// ============================================================
|
// 非流式(会话标题等短任务)
|
// ============================================================
|
|
@Override
|
public String complete(List<LlmMessage> messages, LlmOptions options) {
|
HttpURLConnection conn = null;
|
try {
|
conn = openConnection("/chat/completions", 60000);
|
writeBody(conn, buildRequestBody(messages, options, false));
|
|
if (conn.getResponseCode() != 200) {
|
log.warn("非流式调用失败, status={}", conn.getResponseCode());
|
return null;
|
}
|
try (InputStream in = conn.getInputStream()) {
|
JsonNode node = objectMapper.readTree(in);
|
JsonNode content = node.path("choices").path(0).path("message").path("content");
|
if (content.isMissingNode()) {
|
return null;
|
}
|
// 非流式响应里 reasoning_content 是独立字段,但 content 里
|
// **可能仍夹带 <think> 标签**,先做标签剥离
|
String cleaned = ThinkingTagStripper.stripAll(content.asText());
|
// 再按配置决定是否过滤「模型把推理写进正文」的泄漏
|
if (properties.getOpenaiCompat().isStripReasoningLeak()
|
&& ReasoningLeakFilter.looksLikeReasoning(cleaned)) {
|
log.warn("非流式正文被判定为推理泄漏,已丢弃 {} 字", cleaned.length());
|
return "";
|
}
|
return cleaned;
|
}
|
} catch (Exception e) {
|
// 标题生成属于锦上添花,失败不抛出,由调用方回退
|
log.warn("非流式调用异常: {}", e.getMessage());
|
return null;
|
} finally {
|
if (conn != null) {
|
conn.disconnect();
|
}
|
}
|
}
|
|
@Override
|
public String defaultModel() {
|
return properties.getOpenaiCompat().getModel();
|
}
|
|
// ============================================================
|
// 内部辅助
|
// ============================================================
|
|
private HttpURLConnection openConnection(String path, int timeoutMillis) throws IOException {
|
String baseUrl = properties.getOpenaiCompat().getBaseUrl();
|
if (baseUrl.endsWith("/")) {
|
baseUrl = baseUrl.substring(0, baseUrl.length() - 1);
|
}
|
URI uri = URI.create(baseUrl + path);
|
HttpURLConnection conn = (HttpURLConnection) uri.toURL().openConnection();
|
conn.setRequestMethod("POST");
|
conn.setDoOutput(true);
|
conn.setConnectTimeout(5000);
|
conn.setReadTimeout(timeoutMillis);
|
conn.setRequestProperty("Content-Type", "application/json; charset=UTF-8");
|
conn.setRequestProperty("Accept", "text/event-stream, application/json");
|
|
String apiKey = properties.getOpenaiCompat().getApiKey();
|
if (apiKey != null && !apiKey.isBlank()) {
|
conn.setRequestProperty("Authorization", "Bearer " + apiKey);
|
}
|
// 部分网关需要额外头(如 X-Api-Key),从配置透传
|
for (Map.Entry<String, String> header : properties.getOpenaiCompat().getExtraHeaders().entrySet()) {
|
conn.setRequestProperty(header.getKey(), header.getValue());
|
}
|
return conn;
|
}
|
|
private String buildRequestBody(List<LlmMessage> messages, LlmOptions options, boolean stream)
|
throws IOException {
|
ObjectNode root = objectMapper.createObjectNode();
|
root.put("model", resolveModel(options));
|
root.put("stream", stream);
|
|
ArrayNode messageArray = root.putArray("messages");
|
for (LlmMessage m : messages) {
|
ObjectNode item = messageArray.addObject();
|
item.put("role", m.role());
|
item.put("content", m.content());
|
}
|
|
// 采样参数名与 Ollama 一致(temperature / top_p),可直接复用
|
Double temperature = options.temperature() != null
|
? options.temperature() : properties.getDefaults().getTemperature();
|
Double topP = options.topP() != null
|
? options.topP() : properties.getDefaults().getTopP();
|
if (temperature != null) {
|
root.put("temperature", temperature);
|
}
|
if (topP != null) {
|
root.put("top_p", topP);
|
}
|
|
// 输出长度上限。注意这与 Ollama 的 num_ctx 语义不同:
|
// num_ctx 是「输入+输出」总窗口,max_tokens 只是输出上限。
|
// OpenAI 协议没有暴露上下文窗口的字段,所以这里不能用 numCtx 顶替。
|
Integer maxTokens = properties.getOpenaiCompat().getMaxTokens();
|
if (maxTokens != null && maxTokens > 0) {
|
root.put("max_tokens", maxTokens);
|
}
|
|
return objectMapper.writeValueAsString(root);
|
}
|
|
private String resolveModel(LlmOptions options) {
|
if (options != null && options.modelName() != null && !options.modelName().isBlank()) {
|
return options.modelName().trim();
|
}
|
return properties.getOpenaiCompat().getModel();
|
}
|
|
private void writeBody(HttpURLConnection conn, String body) throws IOException {
|
try (var out = conn.getOutputStream()) {
|
out.write(body.getBytes(StandardCharsets.UTF_8));
|
out.flush();
|
}
|
}
|
|
private String readErrorBody(HttpURLConnection conn) {
|
try (InputStream err = conn.getErrorStream()) {
|
if (err == null) {
|
return "(无响应体)";
|
}
|
return new String(err.readAllBytes(), StandardCharsets.UTF_8);
|
} catch (IOException e) {
|
return "(读取错误响应失败)";
|
}
|
}
|
|
private String abbreviate(String s) {
|
return s.length() > 200 ? s.substring(0, 200) + "..." : s;
|
}
|
}
|