Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-23 23:53:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into tts-response

Browse Source
This commit is contained in:
hrz
2025-05-27 15:36:42 +08:00
committed by GitHub
parent 6a6aceff1d 9ca98f6391
commit d242a9e5c0
30 changed files with 302 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/manager-api/pom.xml
+1 -1
View File
@@ -258,4 +258,4 @@
</plugin>
</plugins>
</build>
</project>
</project>
main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentChatHistoryReportDTO.java
+2
View File
@@ -28,4 +28,6 @@ public class AgentChatHistoryReportDTO {
private String content;
@Schema(description = "base64编码的opus音频数据", example = "")
private String audioBase64;
@Schema(description = "上报时间，十位时间戳，空时默认使用当前时间", example = "1745657732")
private Long reportTime;
}
main/manager-api/src/main/java/xiaozhi/modules/agent/service/AgentChatAudioService.java
+1 -1
View File
@@ -27,4 +27,4 @@ public interface AgentChatAudioService extends IService<AgentChatAudioEntity> {
* @return 音频数据
*/
byte[] getAudio(String audioId);
}
}
main/manager-api/src/main/java/xiaozhi/modules/agent/service/biz/impl/AgentChatHistoryBizServiceImpl.java
+7 -5
View File
@@ -47,7 +47,8 @@ public class AgentChatHistoryBizServiceImpl implements AgentChatHistoryBizServic
public Boolean report(AgentChatHistoryReportDTO report) {
String macAddress = report.getMacAddress();
Byte chatType = report.getChatType();
log.info("小智设备聊天上报请求: macAddress={}, type={}", macAddress, chatType);
Long reportTimeMillis = null != report.getReportTime() ? report.getReportTime() * 1000 : System.currentTimeMillis();
log.info("小智设备聊天上报请求: macAddress={}, type={} reportTime={}", macAddress, chatType, reportTimeMillis);
// 根据设备MAC地址查询对应的默认智能体，判断是否需要上报
AgentEntity agentEntity = agentService.getDefaultAgentByMacAddress(macAddress);
@@ -59,10 +60,10 @@ public class AgentChatHistoryBizServiceImpl implements AgentChatHistoryBizServic
String agentId = agentEntity.getId();
if (Objects.equals(chatHistoryConf, Constant.ChatHistoryConfEnum.RECORD_TEXT.getCode())) {
saveChatText(report, agentId, macAddress, null);
saveChatText(report, agentId, macAddress, null, reportTimeMillis);
} else if (Objects.equals(chatHistoryConf, Constant.ChatHistoryConfEnum.RECORD_TEXT_AUDIO.getCode())) {
String audioId = saveChatAudio(report);
saveChatText(report, agentId, macAddress, audioId);
saveChatText(report, agentId, macAddress, audioId, reportTimeMillis);
}
// 更新设备最后对话时间
@@ -92,8 +93,7 @@ public class AgentChatHistoryBizServiceImpl implements AgentChatHistoryBizServic
/**
* 组装上报数据
*/
private void saveChatText(AgentChatHistoryReportDTO report, String agentId, String macAddress, String audioId) {
private void saveChatText(AgentChatHistoryReportDTO report, String agentId, String macAddress, String audioId, Long reportTime) {
// 构建聊天记录实体
AgentChatHistoryEntity entity = AgentChatHistoryEntity.builder()
.macAddress(macAddress)
@@ -102,6 +102,8 @@ public class AgentChatHistoryBizServiceImpl implements AgentChatHistoryBizServic
.chatType(report.getChatType())
.content(report.getContent())
.audioId(audioId)
.createdAt(new Date(reportTime))
// NOTE(haotian): 2025/5/26 updateAt可以不设置，重点是createAt，而且这样可以看到上报延迟
.build();
// 保存数据
main/manager-api/src/main/java/xiaozhi/modules/agent/service/impl/AgentChatAudioServiceImpl.java
+1 -1
View File
@@ -31,4 +31,4 @@ public class AgentChatAudioServiceImpl extends ServiceImpl<AiAgentChatAudioDao,
AgentChatAudioEntity entity = getById(audioId);
return entity != null ? entity.getAudio() : null;
}
}
}
main/manager-api/src/main/java/xiaozhi/modules/config/service/impl/ConfigServiceImpl.java
+26 -4
View File
@@ -251,6 +251,7 @@ public class ConfigServiceImpl implements ConfigService {
String[] modelTypes = { "VAD", "ASR", "TTS", "Memory", "Intent", "LLM" };
String[] modelIds = { vadModelId, asrModelId, ttsModelId, memModelId, intentModelId, llmModelId };
String intentLLMModelId = null;
String memLocalShortLLMModelId = null;
for (int i = 0; i < modelIds.length; i++) {
if (modelIds[i] == null) {
@@ -269,7 +270,7 @@ public class ConfigServiceImpl implements ConfigService {
Map<String, Object> map = (Map<String, Object>) model.getConfigJson();
if ("intent_llm".equals(map.get("type"))) {
intentLLMModelId = (String) map.get("llm");
if (intentLLMModelId != null && intentLLMModelId.equals(llmModelId)) {
if (StringUtils.isNotBlank(intentLLMModelId) && intentLLMModelId.equals(llmModelId)) {
intentLLMModelId = null;
}
}
@@ -281,10 +282,31 @@ public class ConfigServiceImpl implements ConfigService {
}
}
}
if ("Memory".equals(modelTypes[i])) {
Map<String, Object> map = (Map<String, Object>) model.getConfigJson();
if ("mem_local_short".equals(map.get("type"))) {
memLocalShortLLMModelId = (String) map.get("llm");
if (StringUtils.isNotBlank(memLocalShortLLMModelId)
&& memLocalShortLLMModelId.equals(llmModelId)) {
memLocalShortLLMModelId = null;
}
}
}
// 如果是LLM类型，且intentLLMModelId不为空，则添加附加模型
if ("LLM".equals(modelTypes[i]) && intentLLMModelId != null) {
ModelConfigEntity intentLLM = modelConfigService.getModelById(intentLLMModelId, isCache);
typeConfig.put(intentLLM.getId(), intentLLM.getConfigJson());
if ("LLM".equals(modelTypes[i])) {
if (StringUtils.isNotBlank(intentLLMModelId)) {
if (!typeConfig.containsKey(intentLLMModelId)) {
ModelConfigEntity intentLLM = modelConfigService.getModelById(intentLLMModelId, isCache);
typeConfig.put(intentLLM.getId(), intentLLM.getConfigJson());
}
}
if (StringUtils.isNotBlank(memLocalShortLLMModelId)) {
if (!typeConfig.containsKey(memLocalShortLLMModelId)) {
ModelConfigEntity memLocalShortLLM = modelConfigService
.getModelById(memLocalShortLLMModelId, isCache);
typeConfig.put(memLocalShortLLM.getId(), memLocalShortLLM.getConfigJson());
}
}
}
}
result.put(modelTypes[i], typeConfig);
main/manager-api/src/main/resources/application-dev.yml
+1 -1
View File
@@ -48,4 +48,4 @@ spring:
max-active: 8 # 连接池最大连接数（使用负值表示没有限制）
max-idle: 8 # 连接池中的最大空闲连接
min-idle: 0 # 连接池中的最小空闲连接
shutdown-timeout: 100ms # 客户端优雅关闭的等待时间
shutdown-timeout: 100ms # 客户端优雅关闭的等待时间
main/manager-api/src/main/resources/db/changelog/202505271412.sql
+4
View File
@@ -0,0 +1,4 @@
-- 本地短期记忆配置可以设置独立的LLM
update `ai_model_provider` set fields = '[{"key":"llm","label":"LLM模型","type":"string"}]' where id = 'SYSTEM_Memory_mem_local_short';
update `ai_model_config` set config_json = '{\"type\": \"mem_local_short\", \"llm\": \"LLM_ChatGLMLLM\"}' where id = 'Memory_mem_local_short';
main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml
+8 -1
View File
@@ -162,4 +162,11 @@ databaseChangeLog:
changes:
- sqlFile:
encoding: utf8
path: classpath:db/changelog/202505151451.sql
path: classpath:db/changelog/202505151451.sql
- changeSet:
id: 202505271412
author: hrz
changes:
- sqlFile:
encoding: utf8
path: classpath:db/changelog/202505271412.sql
main/manager-web/src/components/HeaderBar.vue
+1 -1
View File
@@ -51,7 +51,7 @@
字典管理
</el-dropdown-item>
<el-dropdown-item @click.native="goProviderManagement">
供应器管理
字段管理
</el-dropdown-item>
<el-dropdown-item @click.native="goServerSideManagement">
服务端管理
main/manager-web/src/views/ModelConfig.vue
+1 -1
View File
@@ -467,7 +467,7 @@ export default {
.main-wrapper {
margin: 5px 22px;
border-radius: 15px;
min-height: calc(100vh - 24vh);
min-height: calc(100vh - 26vh);
height: auto;
max-height: 80vh;
box-shadow: 0 2px 12px rgba(0, 0, 0, 0.1);
main/manager-web/src/views/ProviderManagement.vue
+1 -1
View File
@@ -3,7 +3,7 @@
<HeaderBar />
<div class="operation-bar">
<h2 class="page-title">供应器管理</h2>
<h2 class="page-title">字段管理</h2>
<div class="right-operations">
<el-dropdown trigger="click" @command="handleSelectModelType" @visible-change="handleDropdownVisibleChange">
<el-button class="category-btn">
main/xiaozhi-server/config.yaml
+5 -1
View File
@@ -218,8 +218,12 @@ Memory:
# 不想使用记忆功能，可以使用nomem
type: nomem
mem_local_short:
# 本地记忆功能，通过selected_module的llm总结，数据保存在本地，不会上传到服务器
# 本地记忆功能，通过selected_module的llm总结，数据保存在本地服务器，不会上传到外部服务器
type: mem_local_short
# 配备记忆存储独立的思考模型
# 如果这里不填，则会默认使用selected_module.LLM的模型作为意图识别的思考模型
# 如果你的不想使用selected_module.LLM记忆存储，这里最好使用独立的LLM作为意图识别，例如使用免费的ChatGLMLLM
llm: ChatGLMLLM
ASR:
FunASR:
main/xiaozhi-server/config/manage_api_client.py
+2 -1
View File
@@ -160,7 +160,7 @@ def save_mem_local_short(mac_address: str, short_momery: str) -> Optional[Dict]:
def report(
mac_address: str, session_id: str, chat_type: int, content: str, audio
mac_address: str, session_id: str, chat_type: int, content: str, audio, report_time
) -> Optional[Dict]:
"""带熔断的业务方法示例"""
if not content or not ManageApiClient._instance:
@@ -174,6 +174,7 @@ def report(
"sessionId": session_id,
"chatType": chat_type,
"content": content,
"reportTime": report_time,
"audioBase64": (
base64.b64encode(audio).decode("utf-8") if audio else None
),
main/xiaozhi-server/core/connection.py
+52 -9
View File
@@ -13,6 +13,8 @@ import websockets
from typing import Dict, Any
from plugins_func.loadplugins import auto_import_modules
from config.logger import setup_logging
from config.config_loader import get_project_dir
from core.utils import p3
from core.utils.dialogue import Message, Dialogue
from core.providers.tts.dto.dto import ContentType, TTSMessageDTO, SentenceType
from core.providers.tts.default import DefaultTTS
@@ -84,12 +86,12 @@ class ConnectionHandler:
# 线程任务相关
self.loop = asyncio.get_event_loop()
self.stop_event = threading.Event()
self.executor = ThreadPoolExecutor(max_workers=10)
self.executor = ThreadPoolExecutor(max_workers=5)
# 上报线程
# 添加上报线程池
self.report_queue = queue.Queue()
self.report_thread = None
# TODO(haotian): 2025/5/12 可以通过修改此处，调节asr的上报和tts的上报
# 未来可以通过修改此处，调节asr的上报和tts的上报，目前默认都开启
self.report_asr_enable = self.read_config_from_api
self.report_tts_enable = self.read_config_from_api
@@ -453,6 +455,37 @@ class ConnectionHandler:
save_to_file=not self.read_config_from_api,
)
# 获取记忆总结配置
memory_config = self.config["Memory"]
memory_type = self.config["Memory"][self.config["selected_module"]["Memory"]][
"type"
]
# 如果使用 nomen，直接返回
if memory_type == "nomem":
return
# 使用 mem_local_short 模式
elif memory_type == "mem_local_short":
memory_llm_name = memory_config[self.config["selected_module"]["Memory"]][
"llm"
]
if memory_llm_name and memory_llm_name in self.config["LLM"]:
# 如果配置了专用LLM，则创建独立的LLM实例
from core.utils import llm as llm_utils
memory_llm_config = self.config["LLM"][memory_llm_name]
memory_llm_type = memory_llm_config.get("type", memory_llm_name)
memory_llm = llm_utils.create_instance(
memory_llm_type, memory_llm_config
)
self.logger.bind(tag=TAG).info(
f"为记忆总结创建了专用LLM: {memory_llm_name}, 类型: {memory_llm_type}"
)
self.memory.set_llm(memory_llm)
else:
# 否则使用主LLM
self.memory.set_llm(self.llm)
self.logger.bind(tag=TAG).info("使用主LLM作为意图识别模型")
def _initialize_intent(self):
self.intent_type = self.config["Intent"][
self.config["selected_module"]["Intent"]
@@ -762,16 +795,15 @@ class ConnectionHandler:
if item is None: # 检测毒丸对象
break
type, text, audio_data = item
type, text, audio_data, report_time = item
try:
# 执行上报（传入二进制数据）
report(self, type, text, audio_data)
# 提交任务到线程池
self.executor.submit(
self._process_report, type, text, audio_data, report_time
)
except Exception as e:
self.logger.bind(tag=TAG).error(f"聊天记录上报线程异常: {e}")
finally:
# 标记任务完成
self.report_queue.task_done()
except queue.Empty:
continue
except Exception as e:
@@ -779,6 +811,17 @@ class ConnectionHandler:
self.logger.bind(tag=TAG).info("聊天记录上报线程已退出")
def _process_report(self, type, text, audio_data, report_time):
"""处理上报任务"""
try:
# 执行上报（传入二进制数据）
report(self, type, text, audio_data, report_time)
except Exception as e:
self.logger.bind(tag=TAG).error(f"上报处理异常: {e}")
finally:
# 标记任务完成
self.report_queue.task_done()
def clearSpeakStatus(self):
self.logger.bind(tag=TAG).debug(f"清除服务端讲话状态")
self.asr_server_receive = True
main/xiaozhi-server/core/handle/intentHandler.py
+2
View File
@@ -145,3 +145,5 @@ async def process_intent_result(conn, intent_result, original_text):
def speak_txt(conn, text):
conn.tts.tts_one_sentence(conn, ContentType.TEXT, content_detail=text)
conn.dialogue.put(Message(role="assistant", content=text))
main/xiaozhi-server/core/handle/reportHandle.py
+8 -5
View File
@@ -8,6 +8,7 @@ TTS上报功能已集成到ConnectionHandler类中。
具体实现请参考core/connection.py中的相关代码。
"""
import time
import opuslib_next
@@ -16,7 +17,7 @@ from config.manage_api_client import report as manage_report
TAG = __name__
def report(conn, type, text, opus_data):
def report(conn, type, text, opus_data, report_time):
"""执行聊天记录上报操作
Args:
@@ -24,6 +25,7 @@ def report(conn, type, text, opus_data):
type: 上报类型，1为用户，2为智能体
text: 合成文本
opus_data: opus音频数据
report_time: 上报时间
"""
try:
if opus_data:
@@ -37,6 +39,7 @@ def report(conn, type, text, opus_data):
chat_type=type,
content=text,
audio=audio_data,
report_time=report_time,
)
except Exception as e:
conn.logger.bind(tag=TAG).error(f"聊天记录上报失败: {e}")
@@ -104,12 +107,12 @@ def enqueue_tts_report(conn, text, opus_data):
try:
# 使用连接对象的队列，传入文本和二进制数据而非文件路径
if conn.chat_history_conf == 2:
conn.report_queue.put((2, text, opus_data))
conn.report_queue.put((2, text, opus_data, int(time.time())))
conn.logger.bind(tag=TAG).debug(
f"TTS数据已加入上报队列: {conn.device_id}, 音频大小: {len(opus_data)} "
)
else:
conn.report_queue.put((2, text, None))
conn.report_queue.put((2, text, None, int(time.time())))
conn.logger.bind(tag=TAG).debug(
f"TTS数据已加入上报队列: {conn.device_id}, 不上报音频"
)
@@ -132,12 +135,12 @@ def enqueue_asr_report(conn, text, opus_data):
try:
# 使用连接对象的队列，传入文本和二进制数据而非文件路径
if conn.chat_history_conf == 2:
conn.report_queue.put((1, text, opus_data))
conn.report_queue.put((1, text, opus_data, int(time.time())))
conn.logger.bind(tag=TAG).debug(
f"ASR数据已加入上报队列: {conn.device_id}, 音频大小: {len(opus_data)} "
)
else:
conn.report_queue.put((1, text, None))
conn.report_queue.put((1, text, None, int(time.time())))
conn.logger.bind(tag=TAG).debug(
f"ASR数据已加入上报队列: {conn.device_id}, 不上报音频"
)
main/xiaozhi-server/core/providers/llm/base.py
+3 -3
View File
@@ -10,7 +10,7 @@ class LLMProviderBase(ABC):
"""LLM response generator"""
pass
def response_no_stream(self, system_prompt, user_prompt):
def response_no_stream(self, system_prompt, user_prompt, **kwargs):
try:
# 构造对话格式
dialogue = [
@@ -18,7 +18,7 @@ class LLMProviderBase(ABC):
{"role": "user", "content": user_prompt}
]
result = ""
for part in self.response("", dialogue):
for part in self.response("", dialogue, **kwargs):
result += part
return result
@@ -30,7 +30,7 @@ class LLMProviderBase(ABC):
"""
Default implementation for function calling (streaming)
This should be overridden by providers that support function calls
Returns: generator that yields either text tokens or a special function call token
"""
# For providers that don't support functions, just return regular response
main/xiaozhi-server/core/providers/llm/coze/coze.py
+1 -1
View File
@@ -25,7 +25,7 @@ class LLMProvider(LLMProviderBase):
self.session_conversation_map = {} # 存储session_id和conversation_id的映射
check_model_key("CozeLLM", self.personal_access_token)
def response(self, session_id, dialogue):
def response(self, session_id, dialogue, **kwargs):
coze_api_token = self.personal_access_token
coze_api_base = COZE_CN_BASE_URL
main/xiaozhi-server/core/providers/llm/dify/dify.py
+1 -1
View File
@@ -17,7 +17,7 @@ class LLMProvider(LLMProviderBase):
self.session_conversation_map = {} # 存储session_id和conversation_id的映射
check_model_key("DifyLLM", self.api_key)
def response(self, session_id, dialogue):
def response(self, session_id, dialogue, **kwargs):
try:
# 取最后一条用户消息
last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
main/xiaozhi-server/core/providers/llm/fastgpt/fastgpt.py
+1 -1
View File
@@ -16,7 +16,7 @@ class LLMProvider(LLMProviderBase):
self.variables = config.get("variables", {})
check_model_key("FastGPTLLM", self.api_key)
def response(self, session_id, dialogue):
def response(self, session_id, dialogue, **kwargs):
try:
# 取最后一条用户消息
last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
main/xiaozhi-server/core/providers/llm/gemini/gemini.py
+1 -1
View File
@@ -112,7 +112,7 @@ class LLMProvider(LLMProviderBase):
]
# Gemini文档提到，无需维护session-id，直接用dialogue拼接而成
def response(self, session_id, dialogue):
def response(self, session_id, dialogue, **kwargs):
yield from self._generate(dialogue, None)
def response_with_functions(self, session_id, dialogue, functions=None):
main/xiaozhi-server/core/providers/llm/homeassistant/homeassistant.py
+1 -1
View File
@@ -14,7 +14,7 @@ class LLMProvider(LLMProviderBase):
self.base_url = config.get("base_url", config.get("url")) # 默认使用 base_url
self.api_url = f"{self.base_url}/api/conversation/process" # 拼接完整的 API URL
def response(self, session_id, dialogue):
def response(self, session_id, dialogue, **kwargs):
try:
# home assistant语音助手自带意图，无需使用xiaozhi ai自带的，只需要把用户说的话传递给home assistant即可
main/xiaozhi-server/core/providers/llm/ollama/ollama.py
+38 -26
View File
@@ -18,13 +18,13 @@ class LLMProvider(LLMProviderBase):
self.client = OpenAI(
base_url=self.base_url,
api_key="ollama" # Ollama doesn't need an API key but OpenAI client requires one
api_key="ollama", # Ollama doesn't need an API key but OpenAI client requires one
)
# 检查是否是qwen3模型
self.is_qwen3 = self.model_name and self.model_name.lower().startswith("qwen3")
def response(self, session_id, dialogue):
def response(self, session_id, dialogue, **kwargs):
try:
# 如果是qwen3模型，在用户最后一条消息中添加/no_think指令
if self.is_qwen3:
@@ -35,7 +35,9 @@ class LLMProvider(LLMProviderBase):
for i in range(len(dialogue_copy) - 1, -1, -1):
if dialogue_copy[i]["role"] == "user":
# 在用户消息前添加/no_think指令
dialogue_copy[i]["content"] = "/no_think " + dialogue_copy[i]["content"]
dialogue_copy[i]["content"] = (
"/no_think " + dialogue_copy[i]["content"]
)
logger.bind(tag=TAG).debug(f"为qwen3模型添加/no_think指令")
break
@@ -43,9 +45,7 @@ class LLMProvider(LLMProviderBase):
dialogue = dialogue_copy
responses = self.client.chat.completions.create(
model=self.model_name,
messages=dialogue,
stream=True
model=self.model_name, messages=dialogue, stream=True
)
is_active = True
# 用于处理跨chunk的标签
@@ -53,29 +53,33 @@ class LLMProvider(LLMProviderBase):
for chunk in responses:
try:
delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None
content = delta.content if hasattr(delta, 'content') else ''
delta = (
chunk.choices[0].delta
if getattr(chunk, "choices", None)
else None
)
content = delta.content if hasattr(delta, "content") else ""
if content:
# 将内容添加到缓冲区
buffer += content
# 处理缓冲区中的标签
while '<think>' in buffer and '</think>' in buffer:
while "<think>" in buffer and "</think>" in buffer:
# 找到完整的<think></think>标签并移除
pre = buffer.split('<think>', 1)[0]
post = buffer.split('</think>', 1)[1]
pre = buffer.split("<think>", 1)[0]
post = buffer.split("</think>", 1)[1]
buffer = pre + post
# 处理只有开始标签的情况
if '<think>' in buffer:
if "<think>" in buffer:
is_active = False
buffer = buffer.split('<think>', 1)[0]
buffer = buffer.split("<think>", 1)[0]
# 处理只有结束标签的情况
if '</think>' in buffer:
if "</think>" in buffer:
is_active = True
buffer = buffer.split('</think>', 1)[1]
buffer = buffer.split("</think>", 1)[1]
# 如果当前处于活动状态且缓冲区有内容，则输出
if is_active and buffer:
@@ -100,7 +104,9 @@ class LLMProvider(LLMProviderBase):
for i in range(len(dialogue_copy) - 1, -1, -1):
if dialogue_copy[i]["role"] == "user":
# 在用户消息前添加/no_think指令
dialogue_copy[i]["content"] = "/no_think " + dialogue_copy[i]["content"]
dialogue_copy[i]["content"] = (
"/no_think " + dialogue_copy[i]["content"]
)
logger.bind(tag=TAG).debug(f"为qwen3模型添加/no_think指令")
break
@@ -119,9 +125,15 @@ class LLMProvider(LLMProviderBase):
for chunk in stream:
try:
delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None
content = delta.content if hasattr(delta, 'content') else None
tool_calls = delta.tool_calls if hasattr(delta, 'tool_calls') else None
delta = (
chunk.choices[0].delta
if getattr(chunk, "choices", None)
else None
)
content = delta.content if hasattr(delta, "content") else None
tool_calls = (
delta.tool_calls if hasattr(delta, "tool_calls") else None
)
# 如果是工具调用，直接传递
if tool_calls:
@@ -134,21 +146,21 @@ class LLMProvider(LLMProviderBase):
buffer += content
# 处理缓冲区中的标签
while '<think>' in buffer and '</think>' in buffer:
while "<think>" in buffer and "</think>" in buffer:
# 找到完整的<think></think>标签并移除
pre = buffer.split('<think>', 1)[0]
post = buffer.split('</think>', 1)[1]
pre = buffer.split("<think>", 1)[0]
post = buffer.split("</think>", 1)[1]
buffer = pre + post
# 处理只有开始标签的情况
if '<think>' in buffer:
if "<think>" in buffer:
is_active = False
buffer = buffer.split('<think>', 1)[0]
buffer = buffer.split("<think>", 1)[0]
# 处理只有结束标签的情况
if '</think>' in buffer:
if "</think>" in buffer:
is_active = True
buffer = buffer.split('</think>', 1)[1]
buffer = buffer.split("</think>", 1)[1]
# 如果当前处于活动状态且缓冲区有内容，则输出
if is_active and buffer:
main/xiaozhi-server/core/providers/llm/openai/openai.py
+21 -10
View File
@@ -16,26 +16,37 @@ class LLMProvider(LLMProviderBase):
self.base_url = config.get("base_url")
else:
self.base_url = config.get("url")
max_tokens = config.get("max_tokens")
if max_tokens is None or max_tokens == "":
max_tokens = 500
try:
max_tokens = int(max_tokens)
except (ValueError, TypeError):
max_tokens = 500
self.max_tokens = max_tokens
param_defaults = {
"max_tokens": (500, int),
"temperature": (0.7, lambda x: round(float(x), 1)),
"top_p": (1.0, lambda x: round(float(x), 1)),
"frequency_penalty": (0, lambda x: round(float(x), 1))
}
for param, (default, converter) in param_defaults.items():
value = config.get(param)
try:
setattr(self, param, converter(value) if value not in (None, "") else default)
except (ValueError, TypeError):
setattr(self, param, default)
logger.debug(
f"意图识别参数初始化: {self.temperature}, {self.max_tokens}, {self.top_p}, {self.frequency_penalty}")
check_model_key("LLM", self.api_key)
self.client = openai.OpenAI(api_key=self.api_key, base_url=self.base_url)
def response(self, session_id, dialogue):
def response(self, session_id, dialogue, **kwargs):
try:
responses = self.client.chat.completions.create(
model=self.model_name,
messages=dialogue,
stream=True,
max_tokens=self.max_tokens,
max_tokens=kwargs.get("max_tokens", self.max_tokens),
temperature=kwargs.get("temperature", self.temperature),
top_p=kwargs.get("top_p", self.top_p),
frequency_penalty=kwargs.get("frequency_penalty", self.frequency_penalty),
)
is_active = True
main/xiaozhi-server/core/providers/llm/xinference/xinference.py
+34 -21
View File
@@ -15,39 +15,45 @@ class LLMProvider(LLMProviderBase):
# 如果没有v1，增加v1
if not self.base_url.endswith("/v1"):
self.base_url = f"{self.base_url}/v1"
logger.bind(tag=TAG).info(f"Initializing Xinference LLM provider with model: {self.model_name}, base_url: {self.base_url}")
logger.bind(tag=TAG).info(
f"Initializing Xinference LLM provider with model: {self.model_name}, base_url: {self.base_url}"
)
try:
self.client = OpenAI(
base_url=self.base_url,
api_key="xinference" # Xinference has a similar setup to Ollama where it doesn't need an actual key
api_key="xinference", # Xinference has a similar setup to Ollama where it doesn't need an actual key
)
logger.bind(tag=TAG).info("Xinference client initialized successfully")
except Exception as e:
logger.bind(tag=TAG).error(f"Error initializing Xinference client: {e}")
raise
def response(self, session_id, dialogue):
def response(self, session_id, dialogue, **kwargs):
try:
logger.bind(tag=TAG).debug(f"Sending request to Xinference with model: {self.model_name}, dialogue length: {len(dialogue)}")
responses = self.client.chat.completions.create(
model=self.model_name,
messages=dialogue,
stream=True
logger.bind(tag=TAG).debug(
f"Sending request to Xinference with model: {self.model_name}, dialogue length: {len(dialogue)}"
)
is_active=True
responses = self.client.chat.completions.create(
model=self.model_name, messages=dialogue, stream=True
)
is_active = True
for chunk in responses:
try:
delta = chunk.choices[0].delta if getattr(chunk, 'choices', None) else None
content = delta.content if hasattr(delta, 'content') else ''
delta = (
chunk.choices[0].delta
if getattr(chunk, "choices", None)
else None
)
content = delta.content if hasattr(delta, "content") else ""
if content:
if '<think>' in content:
if "<think>" in content:
is_active = False
content = content.split('<think>')[0]
if '</think>' in content:
content = content.split("<think>")[0]
if "</think>" in content:
is_active = True
content = content.split('</think>')[-1]
content = content.split("</think>")[-1]
if is_active:
yield content
except Exception as e:
@@ -59,10 +65,14 @@ class LLMProvider(LLMProviderBase):
def response_with_functions(self, session_id, dialogue, functions=None):
try:
logger.bind(tag=TAG).debug(f"Sending function call request to Xinference with model: {self.model_name}, dialogue length: {len(dialogue)}")
logger.bind(tag=TAG).debug(
f"Sending function call request to Xinference with model: {self.model_name}, dialogue length: {len(dialogue)}"
)
if functions:
logger.bind(tag=TAG).debug(f"Function calls enabled with: {[f.get('function', {}).get('name') for f in functions]}")
logger.bind(tag=TAG).debug(
f"Function calls enabled with: {[f.get('function', {}).get('name') for f in functions]}"
)
stream = self.client.chat.completions.create(
model=self.model_name,
messages=dialogue,
@@ -74,7 +84,7 @@ class LLMProvider(LLMProviderBase):
delta = chunk.choices[0].delta
content = delta.content
tool_calls = delta.tool_calls
if content:
yield content, tool_calls
elif tool_calls:
@@ -82,4 +92,7 @@ class LLMProvider(LLMProviderBase):
except Exception as e:
logger.bind(tag=TAG).error(f"Error in Xinference function call: {e}")
yield {"type": "content", "content": f"【Xinference服务响应异常: {str(e)}】"}
yield {
"type": "content",
"content": f"【Xinference服务响应异常: {str(e)}】",
}
main/xiaozhi-server/core/providers/memory/base.py
+7 -1
View File
@@ -9,7 +9,13 @@ class MemoryProviderBase(ABC):
def __init__(self, config):
self.config = config
self.role_id = None
self.llm = None
def set_llm(self, llm):
self.llm = llm
# 获取模型名称和类型信息
model_name = getattr(llm, "model_name", str(llm.__class__.__name__))
# 记录更详细的日志
logger.bind(tag=TAG).info(f"记忆总结设置LLM: {model_name}")
@abstractmethod
async def save_memory(self, msgs):
main/xiaozhi-server/core/providers/memory/mem_local_short/mem_local_short.py
+22 -11
View File
@@ -107,7 +107,7 @@ TAG = __name__
class MemoryProvider(MemoryProviderBase):
def __init__(self, config, summary_memory):
super().__init__(config)
self.short_momery = ""
self.short_memory = ""
self.save_to_file = True
self.memory_path = get_project_dir() + "data/.memory.yaml"
self.load_memory(summary_memory)
@@ -122,7 +122,7 @@ class MemoryProvider(MemoryProviderBase):
def load_memory(self, summary_memory):
# api获取到总结记忆后直接返回
if summary_memory or not self.save_to_file:
self.short_momery = summary_memory
self.short_memory = summary_memory
return
all_memory = {}
@@ -130,18 +130,21 @@ class MemoryProvider(MemoryProviderBase):
with open(self.memory_path, "r", encoding="utf-8") as f:
all_memory = yaml.safe_load(f) or {}
if self.role_id in all_memory:
self.short_momery = all_memory[self.role_id]
self.short_memory = all_memory[self.role_id]
def save_memory_to_file(self):
all_memory = {}
if os.path.exists(self.memory_path):
with open(self.memory_path, "r", encoding="utf-8") as f:
all_memory = yaml.safe_load(f) or {}
all_memory[self.role_id] = self.short_momery
all_memory[self.role_id] = self.short_memory
with open(self.memory_path, "w", encoding="utf-8") as f:
yaml.dump(all_memory, f, allow_unicode=True)
async def save_memory(self, msgs):
# 打印使用的模型信息
model_info = getattr(self.llm, "model_name", str(self.llm.__class__.__name__))
logger.bind(tag=TAG).debug(f"使用记忆保存模型: {model_info}")
if self.llm is None:
logger.bind(tag=TAG).error("LLM is not set for memory provider")
return None
@@ -155,31 +158,39 @@ class MemoryProvider(MemoryProviderBase):
msgStr += f"User: {msg.content}\n"
elif msg.role == "assistant":
msgStr += f"Assistant: {msg.content}\n"
if self.short_momery and len(self.short_momery) > 0:
if self.short_memory and len(self.short_memory) > 0:
msgStr += "历史记忆：\n"
msgStr += self.short_momery
msgStr += self.short_memory
# 当前时间
time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
msgStr += f"当前时间：{time_str}"
if self.save_to_file:
result = self.llm.response_no_stream(short_term_memory_prompt, msgStr)
result = self.llm.response_no_stream(
short_term_memory_prompt,
msgStr,
max_tokens=2000,
temperature=0.2,
)
json_str = extract_json_data(result)
try:
json.loads(json_str) # 检查json格式是否正确
self.short_momery = json_str
self.short_memory = json_str
self.save_memory_to_file()
except Exception as e:
print("Error:", e)
else:
result = self.llm.response_no_stream(
short_term_memory_prompt_only_content, msgStr
short_term_memory_prompt_only_content,
msgStr,
max_tokens=2000,
temperature=0.2,
)
save_mem_local_short(self.role_id, result)
logger.bind(tag=TAG).info(f"Save memory successful - Role: {self.role_id}")
return self.short_momery
return self.short_memory
async def query_memory(self, query: str) -> str:
return self.short_momery
return self.short_memory
main/xiaozhi-server/core/providers/tts/aliyun.py
+18 -5
View File
@@ -91,11 +91,24 @@ class TTSProvider(TTSProviderBase):
self.appkey = config.get("appkey")
self.format = config.get("format", "wav")
self.sample_rate = config.get("sample_rate", 16000)
self.voice = config.get("voice", "xiaoyun")
self.volume = config.get("volume", 50)
self.speech_rate = config.get("speech_rate", 0)
self.pitch_rate = config.get("pitch_rate", 0)
sample_rate = config.get("sample_rate", "16000")
self.sample_rate = int(sample_rate) if sample_rate else 16000
if config.get("private_voice"):
self.voice = config.get("private_voice")
else:
self.voice = config.get("voice", "xiaoyun")
volume = config.get("volume", "50")
self.volume = int(volume) if volume else 50
speech_rate = config.get("speech_rate", "0")
self.speech_rate = int(speech_rate) if speech_rate else 0
pitch_rate = config.get("pitch_rate", "0")
self.pitch_rate = int(pitch_rate) if pitch_rate else 0
self.host = config.get("host", "nls-gateway-cn-shanghai.aliyuncs.com")
self.api_url = f"https://{self.host}/stream/v1/tts"
self.header = {"Content-Type": "application/json"}
main/xiaozhi-server/test/test_page.html
+31 -14
View File
@@ -14,7 +14,7 @@
}
.container {
max-width: 800px;
max-width: 1000px;
margin: 0 auto;
background-color: white;
border-radius: 10px;
@@ -482,9 +482,10 @@
</span>
</h2>
<div class="connection-controls">
<input type="text" id="otaUrl" value="http://127.0.0.1:8002/xiaozhi/ota/" placeholder="OTA服务器地址" />
<input type="text" id="otaUrl" value="http://127.0.0.1:8002/xiaozhi/ota/"
placeholder="OTA服务器地址，如：http://127.0.0.1:8002/xiaozhi/ota/" />
<input type="text" id="serverUrl" value="ws://127.0.0.1:8000/xiaozhi/v1/"
placeholder="WebSocket服务器地址" />
placeholder="WebSocket服务器地址，如：ws://127.0.0.1:8000/xiaozhi/v1/" />
<button id="connectButton">连接</button>
<button id="authTestButton">测试认证</button>
</div>
@@ -879,6 +880,7 @@
// 流已结束且没有更多数据
log("音频播放完成", 'info');
isAudioPlaying = false;
this.endOfStream = false;
streamingContext = null;
} else {
// 等待更多数据
@@ -1061,7 +1063,7 @@
}
}
// 初始化音频录制和处理
// 初始化音频录制和处理
async function initAudio() {
try {
// 请求麦克风权限
@@ -1306,7 +1308,8 @@
// 先检查OTA状态
log('正在检查OTA状态...', 'info');
const otaUrl = document.getElementById('otaUrl').value.trim();
localStorage.setItem('otaUrl', otaUrl);
localStorage.setItem('wsUrl', url);
try {
const otaResponse = await fetch(otaUrl, {
method: 'POST',
@@ -1563,6 +1566,10 @@
const message = messageInput.value.trim();
if (message === '' || !websocket || websocket.readyState !== WebSocket.OPEN) return;
audioBufferQueue = [];
isAudioBuffering = false;
isAudioPlaying = false;
try {
// 直接发送listen消息，不需要重复发送hello
const listenMessage = {
@@ -1633,6 +1640,16 @@
// 初始更新显示值
updateDisplayValues();
const savedOtaUrl = localStorage.getItem('otaUrl');
if (savedOtaUrl) {
document.getElementById('otaUrl').value = savedOtaUrl;
}
const savedWsUrl = localStorage.getItem('wsUrl');
if (savedWsUrl) {
document.getElementById('serverUrl').value = savedWsUrl;
}
// 切换面板显示
toggleButton.addEventListener('click', () => {
const isExpanded = configPanel.classList.contains('expanded');
@@ -1954,7 +1971,7 @@
this.buffer = new Int16Array(this.frameSize);
this.bufferIndex = 0;
this.isRecording = false;
// 监听来自主线程的消息
this.port.onmessage = (event) => {
if (event.data.command === 'start') {
@@ -1962,7 +1979,7 @@
this.port.postMessage({ type: 'status', status: 'started' });
} else if (event.data.command === 'stop') {
this.isRecording = false;
// 发送剩余的缓冲区
if (this.bufferIndex > 0) {
const finalBuffer = this.buffer.slice(0, this.bufferIndex);
@@ -1972,18 +1989,18 @@
});
this.bufferIndex = 0;
}
this.port.postMessage({ type: 'status', status: 'stopped' });
}
};
}
process(inputs, outputs, parameters) {
if (!this.isRecording) return true;
const input = inputs[0][0]; // 获取第一个输入通道
if (!input) return true;
// 将浮点采样转换为16位整数并存储
for (let i = 0; i < input.length; i++) {
if (this.bufferIndex >= this.frameSize) {
@@ -1994,15 +2011,15 @@
});
this.bufferIndex = 0;
}
// 转换为16位整数 (-32768到32767)
this.buffer[this.bufferIndex++] = Math.max(-32768, Math.min(32767, Math.floor(input[i] * 32767)));
}
return true;
}
}
registerProcessor('audio-recorder-processor', AudioRecorderProcessor);
`;