Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-25 00:23:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into function-call-v2

Browse Source
This commit is contained in:
hrz
2025-04-17 11:53:25 +08:00
committed by GitHub
parent 652f5a3247 de5eaf4467
commit 77ff4599ea
70 changed files with 5671 additions and 695 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/xiaozhi-server/core/connection.py
+43 -8
View File
@@ -27,7 +27,11 @@ from core.handle.functionHandler import FunctionHandler
from plugins_func.register import Action, ActionResponse
from core.auth import AuthMiddleware, AuthenticationError
from core.mcp.manager import MCPManager
from config.config_loader import get_private_config_from_api
from config.config_loader import (
get_private_config_from_api,
DeviceNotFoundException,
DeviceBindException,
)
TAG = __name__
@@ -46,6 +50,9 @@ class ConnectionHandler:
self.logger = setup_logging()
self.auth = AuthMiddleware(config)
self.need_bind = False
self.bind_code = None
self.websocket = None
self.headers = None
self.client_ip = None
@@ -109,6 +116,27 @@ class ConnectionHandler:
try:
# 获取并验证headers
self.headers = dict(ws.request.headers)
if self.headers.get("device-id", None) is None:
# 尝试从 URL 的查询参数中获取 device-id
from urllib.parse import parse_qs, urlparse
# 从 WebSocket 请求中获取路径
request_path = ws.request.path
if not request_path:
self.logger.bind(tag=TAG).error("无法获取请求路径")
return
parsed_url = urlparse(request_path)
query_params = parse_qs(parsed_url.query)
if "device-id" in query_params:
self.headers["device-id"] = query_params["device-id"][0]
self.headers["client-id"] = query_params["client-id"][0]
else:
self.logger.bind(tag=TAG).error(
"无法从请求头和URL查询参数中获取device-id"
)
return
# 获取客户端ip地址
self.client_ip = ws.remote_address[0]
self.logger.bind(tag=TAG).info(
@@ -177,7 +205,6 @@ class ConnectionHandler:
self._initialize_models()
"""加载提示词"""
self.prompt = self.config["prompt"]
self.dialogue.put(Message(role="system", content=self.prompt))
"""加载记忆"""
@@ -206,7 +233,15 @@ class ConnectionHandler:
)
private_config["delete_audio"] = bool(self.config.get("delete_audio", True))
self.logger.bind(tag=TAG).info(f"获取差异化配置成功: {private_config}")
except DeviceNotFoundException as e:
self.need_bind = True
private_config = {}
except DeviceBindException as e:
self.need_bind = True
self.bind_code = e.bind_code
private_config = {}
except Exception as e:
self.need_bind = True
self.logger.bind(tag=TAG).error(f"获取差异化配置失败: {e}")
private_config = {}
@@ -254,8 +289,6 @@ class ConnectionHandler:
self.config["selected_module"]["Intent"] = private_config[
"selected_module"
]["Intent"]
if private_config.get("prompt", None) is not None:
self.config["prompt"] = private_config["prompt"]
try:
modules = initialize_modules(
self.logger,
@@ -270,6 +303,10 @@ class ConnectionHandler:
except Exception as e:
self.logger.bind(tag=TAG).error(f"初始化组件失败: {e}")
modules = {}
if modules.get("vad", None) is not None:
self.vad = modules["vad"]
if modules.get("asr", None) is not None:
self.asr = modules["asr"]
if modules.get("tts", None) is not None:
self.tts = modules["tts"]
if modules.get("llm", None) is not None:
@@ -278,6 +315,8 @@ class ConnectionHandler:
self.intent = modules["intent"]
if modules.get("memory", None) is not None:
self.memory = modules["memory"]
if modules.get("prompt", None) is not None:
self.change_system_prompt(modules["prompt"])
def _initialize_memory(self):
"""初始化记忆模块"""
@@ -345,7 +384,6 @@ class ConnectionHandler:
response_message = []
processed_chars = 0 # 跟踪已处理的字符位置
try:
start_time = time.time()
# 使用带记忆的对话
future = asyncio.run_coroutine_threadsafe(
self.memory.query_memory(query), self.loop
@@ -367,9 +405,6 @@ class ConnectionHandler:
if self.client_abort:
break
end_time = time.time()
# self.logger.bind(tag=TAG).debug(f"大模型返回时间: {end_time - start_time} 秒, 生成token={content}")
# 合并当前全部文本并处理未分割部分
full_text = "".join(response_message)
current_text = full_text[processed_chars:] # 从未处理的位置开始
main/xiaozhi-server/core/handle/helloHandle.py
+2
View File
@@ -81,6 +81,8 @@ async def wakeupWordsResponse(conn):
"""唤醒词响应"""
wakeup_word = random.choice(WAKEUP_CONFIG["words"])
result = conn.llm.response_no_stream(conn.config["prompt"], wakeup_word)
if result is None or result == "":
return
tts_file = await asyncio.to_thread(conn.tts.to_tts, result)
if tts_file is not None and os.path.exists(tts_file):
main/xiaozhi-server/core/handle/receiveAudioHandle.py
+46 -3
View File
@@ -1,5 +1,6 @@
from config.logger import setup_logging
import time
import asyncio
from core.utils.util import remove_punctuation_and_length
from core.handle.sendAudioHandle import send_stt_message
from core.handle.intentHandler import handle_user_intent
@@ -35,9 +36,7 @@ async def handleAudioMessage(conn, audio):
if len(conn.asr_audio) < 15:
conn.asr_server_receive = True
else:
text, file_path = await conn.asr.speech_to_text(
conn.asr_audio, conn.session_id
)
text, _ = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id)
logger.bind(tag=TAG).info(f"识别文本: {text}")
text_len, _ = remove_punctuation_and_length(text)
if text_len > 0:
@@ -49,6 +48,9 @@ async def handleAudioMessage(conn, audio):
async def startToChat(conn, text):
if conn.need_bind:
await check_bind_device(conn)
return
# 首先进行意图分析
intent_handled = await handle_user_intent(conn, text)
@@ -85,3 +87,44 @@ async def no_voice_close_connect(conn):
"请你以“时间过得真快”未来头，用富有感情、依依不舍的话来结束这场对话吧。"
)
await startToChat(conn, prompt)
async def check_bind_device(conn):
if conn.bind_code:
# 确保bind_code是6位数字
if len(conn.bind_code) != 6:
logger.bind(tag=TAG).error(f"无效的绑定码格式: {conn.bind_code}")
text = "绑定码格式错误，请检查配置。"
await send_stt_message(conn, text)
return
text = f"请登录控制面板，输入{conn.bind_code}，绑定设备。"
await send_stt_message(conn, text)
conn.tts_first_text_index = 0
conn.tts_last_text_index = 6
conn.llm_finish_task = True
# 播放提示音
music_path = "config/assets/bind_code.wav"
opus_packets, _ = conn.tts.audio_to_opus_data(music_path)
conn.audio_play_queue.put((opus_packets, text, 0))
# 逐个播放数字
for i in range(6): # 确保只播放6位数字
try:
digit = conn.bind_code[i]
num_path = f"config/assets/bind_code/{digit}.wav"
num_packets, _ = conn.tts.audio_to_opus_data(num_path)
conn.audio_play_queue.put((num_packets, None, i + 1))
except Exception as e:
logger.bind(tag=TAG).error(f"播放数字音频失败: {e}")
continue
else:
text = f"没有找到该设备的版本信息，请正确配置 OTA地址，然后重新编译固件。"
await send_stt_message(conn, text)
conn.tts_first_text_index = 0
conn.tts_last_text_index = 0
conn.llm_finish_task = True
music_path = "config/assets/bind_not_found.wav"
opus_packets, _ = conn.tts.audio_to_opus_data(music_path)
conn.audio_play_queue.put((opus_packets, text, 0))
main/xiaozhi-server/core/handle/sendAudioHandle.py
-1
View File
@@ -3,7 +3,6 @@ import json
import asyncio
import time
from core.utils.util import (
remove_punctuation_and_length,
get_string_no_punctuation_or_emoji,
)
main/xiaozhi-server/core/providers/llm/coze/coze.py
+6 -8
View File
@@ -4,6 +4,7 @@ import json
import re
from core.providers.llm.base import LLMProviderBase
import os
# official coze sdk for Python [cozepy](https://github.com/coze-dev/coze-py)
from cozepy import COZE_CN_BASE_URL
from cozepy import Coze, TokenAuth, Message, ChatStatus, MessageContentType, ChatEventType # noqa
@@ -16,8 +17,8 @@ logger = setup_logging()
class LLMProvider(LLMProviderBase):
def __init__(self, config):
self.personal_access_token = config.get("personal_access_token")
self.bot_id = config.get("bot_id")
self.user_id = config.get("user_id")
self.bot_id = str(config.get("bot_id"))
self.user_id = str(config.get("user_id"))
self.session_conversation_map = {} # 存储session_id和conversation_id的映射
def response(self, session_id, dialogue):
@@ -25,16 +26,13 @@ class LLMProvider(LLMProviderBase):
coze_api_base = COZE_CN_BASE_URL
last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
coze = Coze(auth=TokenAuth(token=coze_api_token), base_url=coze_api_base)
conversation_id = self.session_conversation_map.get(session_id)
# 如果没有找到conversation_id，则创建新的对话
if not conversation_id:
conversation = coze.conversations.create(
messages=[
]
)
conversation = coze.conversations.create(messages=[])
conversation_id = conversation.id
self.session_conversation_map[session_id] = conversation_id # 更新映射
@@ -68,4 +66,4 @@ class LLMProvider(LLMProviderBase):
dialogue.pop()
for token in self.response(session_id, dialogue):
yield token, None
yield token, None
main/xiaozhi-server/core/providers/tts/fishspeech.py
+9 -5
View File
@@ -8,7 +8,7 @@ from pydantic import BaseModel, Field, conint, model_validator
from typing_extensions import Annotated
from datetime import datetime
from typing import Literal
from core.utils.util import check_model_key
from core.utils.util import check_model_key, parse_string_to_list
from core.providers.tts.base import TTSProviderBase
from config.logger import setup_logging
@@ -86,8 +86,8 @@ class TTSProvider(TTSProviderBase):
super().__init__(config, delete_audio_file)
self.reference_id = config.get("reference_id")
self.reference_audio = config.get("reference_audio", [])
self.reference_text = config.get("reference_text", [])
self.reference_audio = parse_string_to_list(config.get("reference_audio"))
self.reference_text = parse_string_to_list(config.get("reference_text"))
self.format = config.get("format", "wav")
self.channels = int(config.get("channels", 1))
self.rate = int(config.get("rate", 44100))
@@ -101,9 +101,13 @@ class TTSProvider(TTSProviderBase):
self.top_p = float(config.get("top_p", 0.7))
self.repetition_penalty = float(config.get("repetition_penalty", 1.2))
self.temperature = float(config.get("temperature", 0.7))
self.streaming = bool(config.get("streaming", False))
self.streaming = str(config.get("streaming", False)).lower() in (
"true",
"1",
"yes",
)
self.use_memory_cache = config.get("use_memory_cache", "on")
self.seed = config.get("seed")
self.seed = config.get("seed") or None
self.api_url = config.get("api_url", "http://127.0.0.1:8080/v1/tts")
def generate_filename(self, extension=".wav"):
main/xiaozhi-server/core/providers/tts/gpt_sovits_v2.py
+25 -5
View File
@@ -6,6 +6,7 @@ import requests
from config.logger import setup_logging
from datetime import datetime
from core.providers.tts.base import TTSProviderBase
from core.utils.util import parse_string_to_list
TAG = __name__
logger = setup_logging()
@@ -25,14 +26,33 @@ class TTSProvider(TTSProviderBase):
self.text_split_method = config.get("text_split_method", "cut0")
self.batch_size = int(config.get("batch_size", 1))
self.batch_threshold = float(config.get("batch_threshold", 0.75))
self.split_bucket = bool(config.get("split_bucket", True))
self.return_fragment = bool(config.get("return_fragment", False))
self.split_bucket = str(config.get("split_bucket", True)).lower() in (
"true",
"1",
"yes",
)
self.return_fragment = str(config.get("return_fragment", False)).lower() in (
"true",
"1",
"yes",
)
self.speed_factor = float(config.get("speed_factor", 1.0))
self.streaming_mode = bool(config.get("streaming_mode", False))
self.streaming_mode = str(config.get("streaming_mode", False)).lower() in (
"true",
"1",
"yes",
)
self.seed = int(config.get("seed", -1))
self.parallel_infer = bool(config.get("parallel_infer", True))
self.parallel_infer = str(config.get("parallel_infer", True)).lower() in (
"true",
"1",
"yes",
)
self.repetition_penalty = float(config.get("repetition_penalty", 1.35))
self.aux_ref_audio_paths = config.get("aux_ref_audio_paths", [])
self.aux_ref_audio_paths = parse_string_to_list(
config.get("aux_ref_audio_paths")
)
def generate_filename(self, extension=".wav"):
return os.path.join(
main/xiaozhi-server/core/providers/tts/gpt_sovits_v3.py
+3 -2
View File
@@ -4,6 +4,7 @@ import requests
from config.logger import setup_logging
from datetime import datetime
from core.providers.tts.base import TTSProviderBase
from core.utils.util import parse_string_to_list
TAG = __name__
logger = setup_logging()
@@ -22,9 +23,9 @@ class TTSProvider(TTSProviderBase):
self.temperature = float(config.get("temperature", 1.0))
self.cut_punc = config.get("cut_punc", "")
self.speed = float(config.get("speed", 1.0))
self.inp_refs = config.get("inp_refs", [])
self.inp_refs = parse_string_to_list(config.get("inp_refs"))
self.sample_steps = int(config.get("sample_steps", 32))
self.if_sr = bool(config.get("if_sr", False))
self.if_sr = str(config.get("if_sr", False)).lower() in ("true", "1", "yes")
def generate_filename(self, extension=".wav"):
return os.path.join(
main/xiaozhi-server/core/providers/tts/minimax.py
+2 -1
View File
@@ -4,6 +4,7 @@ import json
import requests
from datetime import datetime
from core.providers.tts.base import TTSProviderBase
from core.utils.util import parse_string_to_list
class TTSProvider(TTSProviderBase):
@@ -40,7 +41,7 @@ class TTSProvider(TTSProviderBase):
**config.get("pronunciation_dict", {}),
}
self.audio_setting = {**defult_audio_setting, **config.get("audio_setting", {})}
self.timber_weights = config.get("timber_weights", [])
self.timber_weights = parse_string_to_list(config.get("timber_weights"))
if self.voice_id:
self.voice_setting["voice_id"] = self.voice_id
main/xiaozhi-server/core/providers/tts/siliconflow.py
+1 -1
View File
@@ -16,7 +16,7 @@ class TTSProvider(TTSProviderBase):
self.voice = config.get("voice")
self.response_format = config.get("response_format")
self.sample_rate = config.get("sample_rate")
self.speed = float(config.get("speed"))
self.speed = float(config.get("speed", 1.0))
self.gain = config.get("gain")
self.host = "api.siliconflow.cn"
main/xiaozhi-server/core/providers/tts/ttson.py
+1 -1
View File
@@ -22,7 +22,7 @@ class TTSProvider(TTSProviderBase):
self.to_lang = config.get("to_lang")
self.volume_change_dB = int(config.get("volume_change_dB", 0))
self.speed_factor = int(config.get("speed_factor", 1))
self.stream = bool(config.get("stream", False))
self.stream = str(config.get("stream", False)).lower() in ("true", "1", "yes")
self.output_file = config.get("output_dir")
self.pitch_factor = int(config.get("pitch_factor", 0))
self.format = config.get("format", "mp3")
main/xiaozhi-server/core/utils/util.py
+63 -32
View File
@@ -163,6 +163,24 @@ def check_model_key(modelType, modelKey):
return True
def parse_string_to_list(value, separator=";"):
"""
将输入值转换为列表
Args:
value: 输入值，可以是 None、字符串或列表
separator: 分隔符，默认为分号
Returns:
list: 处理后的列表
"""
if value is None or value == "":
return []
elif isinstance(value, str):
return [item.strip() for item in value.split(separator) if item.strip()]
elif isinstance(value, list):
return value
return []
def check_ffmpeg_installed():
ffmpeg_installed = False
try:
@@ -222,80 +240,93 @@ def initialize_modules(
# 初始化TTS模块
if init_tts:
select_tts_module = config["selected_module"]["TTS"]
tts_type = (
config["selected_module"]["TTS"]
if "type" not in config["TTS"][config["selected_module"]["TTS"]]
else config["TTS"][config["selected_module"]["TTS"]]["type"]
select_tts_module
if "type" not in config["TTS"][select_tts_module]
else config["TTS"][select_tts_module]["type"]
)
modules["tts"] = tts.create_instance(
tts_type,
config["TTS"][config["selected_module"]["TTS"]],
bool(config.get("delete_audio", True)),
config["TTS"][select_tts_module],
str(config.get("delete_audio", True)).lower() in ("true", "1", "yes"),
)
logger.bind(tag=TAG).info(f"初始化组件: tts成功")
logger.bind(tag=TAG).info(f"初始化组件: tts成功 {select_tts_module}")
# 初始化LLM模块
if init_llm:
select_llm_module = config["selected_module"]["LLM"]
llm_type = (
config["selected_module"]["LLM"]
if "type" not in config["LLM"][config["selected_module"]["LLM"]]
else config["LLM"][config["selected_module"]["LLM"]]["type"]
select_llm_module
if "type" not in config["LLM"][select_llm_module]
else config["LLM"][select_llm_module]["type"]
)
modules["llm"] = llm.create_instance(
llm_type,
config["LLM"][config["selected_module"]["LLM"]],
config["LLM"][select_llm_module],
)
logger.bind(tag=TAG).info(f"初始化组件: llm成功")
logger.bind(tag=TAG).info(f"初始化组件: llm成功 {select_llm_module}")
# 初始化Intent模块
if init_intent:
select_intent_module = config["selected_module"]["Intent"]
intent_type = (
config["selected_module"]["Intent"]
if "type" not in config["Intent"][config["selected_module"]["Intent"]]
else config["Intent"][config["selected_module"]["Intent"]]["type"]
select_intent_module
if "type" not in config["Intent"][select_intent_module]
else config["Intent"][select_intent_module]["type"]
)
modules["intent"] = intent.create_instance(
intent_type,
config["Intent"][config["selected_module"]["Intent"]],
config["Intent"][select_intent_module],
)
logger.bind(tag=TAG).info(f"初始化组件: intent成功")
logger.bind(tag=TAG).info(f"初始化组件: intent成功 {select_intent_module}")
# 初始化Memory模块
if init_memory:
select_memory_module = config["selected_module"]["Memory"]
memory_type = (
config["selected_module"]["Memory"]
if "type" not in config["Memory"][config["selected_module"]["Memory"]]
else config["Memory"][config["selected_module"]["Memory"]]["type"]
select_memory_module
if "type" not in config["Memory"][select_memory_module]
else config["Memory"][select_memory_module]["type"]
)
modules["memory"] = memory.create_instance(
memory_type,
config["Memory"][config["selected_module"]["Memory"]],
config["Memory"][select_memory_module],
)
logger.bind(tag=TAG).info(f"初始化组件: memory成功")
logger.bind(tag=TAG).info(f"初始化组件: memory成功 {select_memory_module}")
# 初始化VAD模块
if init_vad:
select_vad_module = config["selected_module"]["VAD"]
vad_type = (
config["selected_module"]["VAD"]
if "type" not in config["VAD"][config["selected_module"]["VAD"]]
else config["VAD"][config["selected_module"]["VAD"]]["type"]
select_vad_module
if "type" not in config["VAD"][select_vad_module]
else config["VAD"][select_vad_module]["type"]
)
modules["vad"] = vad.create_instance(
vad_type,
config["VAD"][config["selected_module"]["VAD"]],
config["VAD"][select_vad_module],
)
logger.bind(tag=TAG).info(f"初始化组件: vad成功")
logger.bind(tag=TAG).info(f"初始化组件: vad成功 {select_vad_module}")
# 初始化ASR模块
if init_asr:
select_asr_module = config["selected_module"]["ASR"]
asr_type = (
config["selected_module"]["ASR"]
if "type" not in config["ASR"][config["selected_module"]["ASR"]]
else config["ASR"][config["selected_module"]["ASR"]]["type"]
select_asr_module
if "type" not in config["ASR"][select_asr_module]
else config["ASR"][select_asr_module]["type"]
)
modules["asr"] = asr.create_instance(
asr_type,
config["ASR"][config["selected_module"]["ASR"]],
bool(config.get("delete_audio", True)),
config["ASR"][select_asr_module],
str(config.get("delete_audio", True)).lower() in ("true", "1", "yes"),
)
logger.bind(tag=TAG).info(f"初始化组件: asr成功")
logger.bind(tag=TAG).info(f"初始化组件: asr成功 {select_asr_module}")
# 初始化自定义prompt
if config.get("prompt", None) is not None:
modules["prompt"] = config["prompt"]
logger.bind(tag=TAG).info(f"初始化组件: prompt成功 {modules['prompt'][:50]}...")
return modules