mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-26 00:53:54 +08:00
Merge branch 'main' into mem0ai
This commit is contained in:
玄凤科技
+46
-41
@@ -22,6 +22,10 @@ from core.utils.auth_code_gen import AuthCodeGenerator
|
||||
TAG = __name__
|
||||
|
||||
|
||||
class TTSException(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class ConnectionHandler:
|
||||
def __init__(self, config: Dict[str, Any], _vad, _asr, _llm, _tts, _music, _memory):
|
||||
self.config = config
|
||||
@@ -68,10 +72,8 @@ class ConnectionHandler:
|
||||
self.dialogue = Dialogue()
|
||||
|
||||
# tts相关变量
|
||||
self.tts_first_text = None
|
||||
self.tts_last_text = None
|
||||
self.tts_start_speak_time = None
|
||||
self.tts_duration = 0
|
||||
self.tts_first_text_index = -1
|
||||
self.tts_last_text_index = -1
|
||||
|
||||
# iot相关变量
|
||||
self.iot_descriptors = {}
|
||||
@@ -229,6 +231,7 @@ class ConnectionHandler:
|
||||
return None
|
||||
|
||||
self.llm_finish_task = False
|
||||
text_index = 0
|
||||
for content in llm_responses:
|
||||
response_message.append(content)
|
||||
if self.client_abort:
|
||||
@@ -242,7 +245,7 @@ class ConnectionHandler:
|
||||
current_text = full_text[processed_chars:] # 从未处理的位置开始
|
||||
|
||||
# 查找最后一个有效标点
|
||||
punctuations = ("。", "？", "！", "?", "!", ";", "；", ":", "：", "，")
|
||||
punctuations = ("。", "？", "！", "?", "!", ";", "；", ":", "：")
|
||||
last_punct_pos = -1
|
||||
for punct in punctuations:
|
||||
pos = current_text.rfind(punct)
|
||||
@@ -254,8 +257,12 @@ class ConnectionHandler:
|
||||
segment_text_raw = current_text[:last_punct_pos + 1]
|
||||
segment_text = get_string_no_punctuation_or_emoji(segment_text_raw)
|
||||
if segment_text:
|
||||
self.recode_first_last_text(segment_text)
|
||||
future = self.executor.submit(self.speak_and_play, segment_text)
|
||||
# 强制设置空字符，测试TTS出错返回语音的健壮性
|
||||
# if text_index % 2 == 0:
|
||||
# segment_text = " "
|
||||
text_index += 1
|
||||
self.recode_first_last_text(segment_text, text_index)
|
||||
future = self.executor.submit(self.speak_and_play, segment_text, text_index)
|
||||
self.tts_queue.put(future)
|
||||
processed_chars += len(segment_text_raw) # 更新已处理字符位置
|
||||
|
||||
@@ -265,8 +272,9 @@ class ConnectionHandler:
|
||||
if remaining_text:
|
||||
segment_text = get_string_no_punctuation_or_emoji(remaining_text)
|
||||
if segment_text:
|
||||
self.recode_first_last_text(segment_text)
|
||||
future = self.executor.submit(self.speak_and_play, segment_text)
|
||||
text_index += 1
|
||||
self.recode_first_last_text(segment_text, text_index)
|
||||
future = self.executor.submit(self.speak_and_play, segment_text, text_index)
|
||||
self.tts_queue.put(future)
|
||||
|
||||
self.llm_finish_task = True
|
||||
@@ -282,30 +290,28 @@ class ConnectionHandler:
|
||||
if future is None:
|
||||
continue
|
||||
text = None
|
||||
opus_datas, text_index, tts_file = [], 0, None
|
||||
try:
|
||||
self.logger.bind(tag=TAG).debug("正在处理TTS任务...")
|
||||
tts_file, text = future.result(timeout=10)
|
||||
tts_file, text, text_index = future.result(timeout=10)
|
||||
if text is None or len(text) <= 0:
|
||||
continue
|
||||
if tts_file is None:
|
||||
self.logger.bind(tag=TAG).error(f"TTS文件生成失败: {text}")
|
||||
continue
|
||||
self.logger.bind(tag=TAG).debug(f"TTS文件生成完毕，文件路径: {tts_file}")
|
||||
if os.path.exists(tts_file):
|
||||
opus_datas, duration = self.tts.wav_to_opus_data(tts_file)
|
||||
self.logger.bind(tag=TAG).error(f"TTS出错：{text_index}: tts text is empty")
|
||||
elif tts_file is None:
|
||||
self.logger.bind(tag=TAG).error(f"TTS出错： file is empty: {text_index}: {text}")
|
||||
else:
|
||||
self.logger.bind(tag=TAG).error(f"TTS文件不存在: {tts_file}")
|
||||
opus_datas = []
|
||||
self.logger.bind(tag=TAG).debug(f"TTS生成：文件路径: {tts_file}")
|
||||
if os.path.exists(tts_file):
|
||||
opus_datas, duration = self.tts.wav_to_opus_data(tts_file)
|
||||
else:
|
||||
self.logger.bind(tag=TAG).error(f"TTS出错：文件不存在{tts_file}")
|
||||
except TimeoutError:
|
||||
self.logger.bind(tag=TAG).error("TTS 任务超时")
|
||||
continue
|
||||
self.logger.bind(tag=TAG).error("TTS超时")
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"TTS 任务出错: {e}")
|
||||
continue
|
||||
self.logger.bind(tag=TAG).error(f"TTS出错: {e}")
|
||||
if not self.client_abort:
|
||||
# 如果没有中途打断就发送语音
|
||||
self.audio_play_queue.put((opus_datas, text))
|
||||
if self.tts.delete_audio_file and os.path.exists(tts_file):
|
||||
self.audio_play_queue.put((opus_datas, text, text_index))
|
||||
if self.tts.delete_audio_file and tts_file is not None and os.path.exists(tts_file):
|
||||
os.remove(tts_file)
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"TTS任务处理错误: {e}")
|
||||
@@ -314,42 +320,41 @@ class ConnectionHandler:
|
||||
self.websocket.send(json.dumps({"type": "tts", "state": "stop", "session_id": self.session_id})),
|
||||
self.loop
|
||||
)
|
||||
self.logger.bind(tag=TAG).error(f"tts_priority priority_thread: {text}{e}")
|
||||
self.logger.bind(tag=TAG).error(f"tts_priority priority_thread: {text} {e}")
|
||||
|
||||
def _audio_play_priority_thread(self):
|
||||
while not self.stop_event.is_set():
|
||||
text = None
|
||||
try:
|
||||
opus_datas, text = self.audio_play_queue.get()
|
||||
future = asyncio.run_coroutine_threadsafe(sendAudioMessage(self, opus_datas, text), self.loop)
|
||||
opus_datas, text, text_index = self.audio_play_queue.get()
|
||||
future = asyncio.run_coroutine_threadsafe(sendAudioMessage(self, opus_datas, text, text_index),
|
||||
self.loop)
|
||||
future.result()
|
||||
except Exception as e:
|
||||
self.logger.bind(tag=TAG).error(f"audio_play_priority priority_thread: {text}{e}")
|
||||
self.logger.bind(tag=TAG).error(f"audio_play_priority priority_thread: {text} {e}")
|
||||
|
||||
def speak_and_play(self, text):
|
||||
def speak_and_play(self, text, text_index=0):
|
||||
if text is None or len(text) <= 0:
|
||||
self.logger.bind(tag=TAG).info(f"无需tts转换，query为空，{text}")
|
||||
return None, text
|
||||
return None, text, text_index
|
||||
tts_file = self.tts.to_tts(text)
|
||||
if tts_file is None:
|
||||
self.logger.bind(tag=TAG).error(f"tts转换失败，{text}")
|
||||
return None, text
|
||||
return None, text, text_index
|
||||
self.logger.bind(tag=TAG).debug(f"TTS 文件生成完毕: {tts_file}")
|
||||
return tts_file, text
|
||||
return tts_file, text, text_index
|
||||
|
||||
def clearSpeakStatus(self):
|
||||
self.logger.bind(tag=TAG).debug(f"清除服务端讲话状态")
|
||||
self.asr_server_receive = True
|
||||
self.tts_last_text = None
|
||||
self.tts_first_text = None
|
||||
self.tts_duration = 0
|
||||
self.tts_start_speak_time = None
|
||||
self.tts_last_text_index = -1
|
||||
self.tts_first_text_index = -1
|
||||
|
||||
def recode_first_last_text(self, text):
|
||||
if not self.tts_first_text:
|
||||
def recode_first_last_text(self, text, text_index=0):
|
||||
if self.tts_first_text_index == -1:
|
||||
self.logger.bind(tag=TAG).info(f"大模型说出第一句话: {text}")
|
||||
self.tts_first_text = text
|
||||
self.tts_last_text = text
|
||||
self.tts_first_text_index = text_index
|
||||
self.tts_last_text_index = text_index
|
||||
|
||||
async def close(self):
|
||||
"""资源清理方法"""
|
||||
|
||||
@@ -139,14 +139,14 @@ class MusicHandler:
|
||||
return
|
||||
text = f"正在播放{selected_music}"
|
||||
await send_stt_message(conn, text)
|
||||
conn.tts_first_text = selected_music
|
||||
conn.tts_last_text = selected_music
|
||||
conn.tts_first_text_index = 0
|
||||
conn.tts_last_text_index = 0
|
||||
conn.llm_finish_task = True
|
||||
if music_path.endswith(".p3"):
|
||||
opus_packets, duration = p3.decode_opus_from_file(music_path)
|
||||
else:
|
||||
opus_packets, duration = conn.tts.wav_to_opus_data(music_path)
|
||||
conn.audio_play_queue.put((opus_packets, selected_music))
|
||||
conn.audio_play_queue.put((opus_packets, selected_music, 0))
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"播放音乐失败: {str(e)}")
|
||||
|
||||
@@ -15,11 +15,10 @@ async def isLLMWantToFinish(last_text):
|
||||
return False
|
||||
|
||||
|
||||
async def sendAudioMessage(conn, audios, text):
|
||||
async def sendAudioMessage(conn, audios, text, text_index=0):
|
||||
# 发送句子开始消息
|
||||
if text == conn.tts_first_text:
|
||||
if text_index == conn.tts_first_text_index:
|
||||
logger.bind(tag=TAG).info(f"发送第一段语音: {text}")
|
||||
conn.tts_start_speak_time = time.perf_counter()
|
||||
await send_tts_message(conn, "sentence_start", text)
|
||||
|
||||
# 初始化流控参数
|
||||
@@ -45,7 +44,7 @@ async def sendAudioMessage(conn, audios, text):
|
||||
play_position += frame_duration # 更新播放位置
|
||||
await send_tts_message(conn, "sentence_end", text)
|
||||
# 发送结束消息（如果是最后一个文本）
|
||||
if conn.llm_finish_task and text == conn.tts_last_text:
|
||||
if conn.llm_finish_task and text_index == conn.tts_last_text_index:
|
||||
await send_tts_message(conn, 'stop', None)
|
||||
if await isLLMWantToFinish(text):
|
||||
await conn.close()
|
||||
|
||||
@@ -3,94 +3,35 @@ import requests
|
||||
import json
|
||||
import re
|
||||
from core.providers.llm.base import LLMProviderBase
|
||||
import os
|
||||
# official coze sdk for Python [cozepy](https://github.com/coze-dev/coze-py)
|
||||
from cozepy import COZE_CN_BASE_URL
|
||||
from cozepy import Coze, TokenAuth, Message, ChatStatus, MessageContentType, ChatEventType # noqa
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
# 定义用于匹配中文标点符号的正则表达式（包括句号、感叹号、问号、分号）
|
||||
punctuation_pattern = re.compile(r'([。！？；])')
|
||||
|
||||
class LLMProvider(LLMProviderBase):
|
||||
def __init__(self, config):
|
||||
self.personal_access_token = config.get("personal_access_token")
|
||||
self.bot_id = config.get("bot_id")
|
||||
self.user_id = config.get("user_id") # 默认用户 ID
|
||||
self.base_url = config.get("base_url")
|
||||
self.user_id = config.get("user_id")
|
||||
|
||||
def response(self, session_id, dialogue):
|
||||
try:
|
||||
# 从对话中取出最新的用户消息
|
||||
last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
|
||||
data = {
|
||||
"conversation_id": session_id,
|
||||
"bot_id": self.bot_id,
|
||||
"user": self.user_id,
|
||||
"query": last_msg["content"],
|
||||
"stream": True
|
||||
}
|
||||
logger.bind(tag=TAG).info(f"发送到 Coze API 的请求数据: {json.dumps(data, ensure_ascii=False)}")
|
||||
|
||||
headers = {
|
||||
'Authorization': f'Bearer {self.personal_access_token}',
|
||||
'Content-Type': 'application/json',
|
||||
'Accept': '*/*',
|
||||
'Host': 'api.coze.cn',
|
||||
'Connection': 'keep-alive'
|
||||
}
|
||||
|
||||
response = requests.post(
|
||||
self.base_url,
|
||||
headers=headers,
|
||||
json=data,
|
||||
stream=True
|
||||
)
|
||||
logger.bind(tag=TAG).info(f"请求状态: {response.status_code}")
|
||||
|
||||
if response.status_code == 200:
|
||||
# 对每一行流数据进行处理，不做跨块累积
|
||||
for line_bytes in response.iter_lines(decode_unicode=False):
|
||||
if not line_bytes:
|
||||
continue
|
||||
try:
|
||||
# 使用 utf-8 解码，错误部分用替换符
|
||||
line = line_bytes.decode('utf-8', errors='replace')
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"解码失败: {e}")
|
||||
continue
|
||||
if line.startswith("data:"):
|
||||
data_str = line[len("data:"):].strip()
|
||||
if data_str == "[DONE]":
|
||||
break
|
||||
try:
|
||||
data_chunk = json.loads(data_str)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.bind(tag=TAG).error(f"JSON解析失败: {e} 数据: {line}")
|
||||
continue
|
||||
msg = data_chunk.get("message", {})
|
||||
if msg.get("role") == "assistant" and msg.get("type") == "answer":
|
||||
content = msg.get("content", "")
|
||||
# 如果返回内容中包含标点符号，则按标点拆分，立即返回每个片段
|
||||
if punctuation_pattern.search(content):
|
||||
# 利用 finditer 找到每个标点，并返回以标点结尾的片段
|
||||
start = 0
|
||||
for match in punctuation_pattern.finditer(content):
|
||||
end = match.end()
|
||||
sentence = content[start:end].strip()
|
||||
if sentence:
|
||||
yield sentence
|
||||
start = end
|
||||
# 如果拆分后剩余内容也返回（不含标点），直接返回
|
||||
if start < len(content):
|
||||
remainder = content[start:].strip()
|
||||
if remainder:
|
||||
yield remainder
|
||||
else:
|
||||
# 如果没有标点，则直接返回这块内容
|
||||
if content.strip():
|
||||
yield content.strip()
|
||||
else:
|
||||
logger.bind(tag=TAG).error(f"请求失败，状态码: {response.status_code}")
|
||||
yield f"【Coze服务响应异常：请求失败，状态码 {response.status_code}】"
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"Error in Coze response generation: {e}")
|
||||
yield "【Coze服务响应异常】"
|
||||
coze_api_token = self.personal_access_token
|
||||
coze_api_base = COZE_CN_BASE_URL
|
||||
|
||||
last_msg = next(m for m in reversed(dialogue) if m["role"] == "user")
|
||||
|
||||
coze = Coze(auth=TokenAuth(token=coze_api_token), base_url=coze_api_base)
|
||||
|
||||
for event in coze.chat.stream(
|
||||
bot_id=self.bot_id,
|
||||
user_id=self.user_id,
|
||||
additional_messages=[
|
||||
Message.build_user_question_text(last_msg["content"]),
|
||||
],
|
||||
):
|
||||
if event.event == ChatEventType.CONVERSATION_MESSAGE_DELTA:
|
||||
print(event.message.content, end="", flush=True)
|
||||
yield event.message.content
|
||||
Reference in New Issue
Block a user