Hass/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-29 18:23:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: 流式tts，function聊天时候的支持

Browse Source
This commit is contained in:
lizhongxiang
2025-03-13 17:44:10 +08:00
parent 1dfa619243
commit c271b88c87
3 changed files with 65 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/xiaozhi-server/config.yaml
+7 -7
View File
@@ -72,14 +72,14 @@ selected_module:
# 将根据配置名称对应的type调用实际的LLM适配器 # 将根据配置名称对应的type调用实际的LLM适配器
LLM: ChatGLMLLM LLM: ChatGLMLLM
# TTS将根据配置名称对应的type调用实际的TTS适配器 # TTS将根据配置名称对应的type调用实际的TTS适配器
TTS: EdgeTTS TTS: FishSpeech
# 记忆模块，默认不开启记忆；如果想使用超长记忆，推荐使用mem0ai；如果注重隐私，请使用本地的mem_local_short # 记忆模块，默认不开启记忆；如果想使用超长记忆，推荐使用mem0ai；如果注重隐私，请使用本地的mem_local_short
Memory: nomem Memory: nomem
# 意图识别模块，默认不开启。开启后，可以播放音乐、控制音量、识别退出指令 # 意图识别模块，默认不开启。开启后，可以播放音乐、控制音量、识别退出指令
# 意图识别使用intent_llm，优点：通用性强，缺点：增加串行前置意图识别模块，会增加处理时间 # 意图识别使用intent_llm，优点：通用性强，缺点：增加串行前置意图识别模块，会增加处理时间
# 意图识别使用function_call，缺点：需要所选择的LLM支持function_call，优点：按需调用工具、速度快 # 意图识别使用function_call，缺点：需要所选择的LLM支持function_call，优点：按需调用工具、速度快
# 如果意图识别设置成 function_call，建议把LLM设置成：DoubaoLLM，使用的具体model_name是：doubao-pro-32k-functioncall-241028 # 如果意图识别设置成 function_call，建议把LLM设置成：DoubaoLLM，使用的具体model_name是：doubao-pro-32k-functioncall-241028
Intent: nointent Intent: function_call
# 意图识别，是用于理解用户意图的模块，例如：播放音乐 # 意图识别，是用于理解用户意图的模块，例如：播放音乐
Intent: Intent:
@@ -157,7 +157,7 @@ LLM:
# 可在这里找到你的api key https://bigmodel.cn/usercenter/proj-mgmt/apikeys # 可在这里找到你的api key https://bigmodel.cn/usercenter/proj-mgmt/apikeys
model_name: glm-4-flash model_name: glm-4-flash
url: https://open.bigmodel.cn/api/paas/v4/ url: https://open.bigmodel.cn/api/paas/v4/
api_key: 你的chat-glm web key api_key: f04500d80a234c9085be2b4020dce25f.gnnnWr3BRLuqeDHr
OllamaLLM: OllamaLLM:
# 定义LLM API类型 # 定义LLM API类型
type: ollama type: ollama
@@ -208,8 +208,8 @@ LLM:
k2: "v2" k2: "v2"
TTS_SET: TTS_SET:
TTS_STREAM: false #是否启动流响应 true/false，默认false TTS_STREAM: true #是否启动流响应 true/false，默认false
MAX_WORKERS: 4 #并发请求tts的数量，这个调太大，本地tts会变慢 MAX_WORKERS: 5 #并发请求tts的数量，这个调太大，本地tts会变慢
TTS: TTS:
# 当前支持的type为edge、doubao，可自行适配 # 当前支持的type为edge、doubao，可自行适配
@@ -283,8 +283,8 @@ TTS:
seed: null seed: null
channels: 1 channels: 1
rate: 44100 rate: 44100
api_key: "你的api_key" api_key: "xxxx"
api_url: "http://127.0.0.1:8080/v1/tts" api_url: "http://192.168.101.242:10000/tts-fishspeech/v1/tts"
GPT_SOVITS_V2: GPT_SOVITS_V2:
# 定义TTS API类型 # 定义TTS API类型
#启动tts方法： #启动tts方法：
main/xiaozhi-server/core/connection.py
+56 -25
View File
@@ -257,7 +257,7 @@ class ConnectionHandler:
current_text = full_text[processed_chars:] # 从未处理的位置开始 current_text = full_text[processed_chars:] # 从未处理的位置开始
# 查找最后一个有效标点 # 查找最后一个有效标点
punctuations = ("。", "？", "！", "；", "：", ".", "?", "!", ";", ":"," ") punctuations = ("。", "？", "！", "；", "：", ".", "?", "!", ";", ":", " ")
last_punct_pos = -1 last_punct_pos = -1
for punct in punctuations: for punct in punctuations:
pos = current_text.rfind(punct) pos = current_text.rfind(punct)
@@ -328,7 +328,7 @@ class ConnectionHandler:
response_message = [] response_message = []
processed_chars = 0 # 跟踪已处理的字符位置 processed_chars = 0 # 跟踪已处理的字符位置
try: try:
start_time = time.time() start_time = time.time()
@@ -336,7 +336,7 @@ class ConnectionHandler:
future = asyncio.run_coroutine_threadsafe(self.memory.query_memory(query), self.loop) future = asyncio.run_coroutine_threadsafe(self.memory.query_memory(query), self.loop)
memory_str = future.result() memory_str = future.result()
#self.logger.bind(tag=TAG).info(f"对话记录: {self.dialogue.get_llm_dialogue_with_memory(memory_str)}") # self.logger.bind(tag=TAG).info(f"对话记录: {self.dialogue.get_llm_dialogue_with_memory(memory_str)}")
# 使用支持functions的streaming接口 # 使用支持functions的streaming接口
llm_responses = self.llm.response_with_functions( llm_responses = self.llm.response_with_functions(
@@ -359,8 +359,8 @@ class ConnectionHandler:
content_arguments = "" content_arguments = ""
for response in llm_responses: for response in llm_responses:
content, tools_call = response content, tools_call = response
if content is not None and len(content)>0: if content is not None and len(content) > 0:
if len(response_message)<=0 and content=="```": if len(response_message) <= 0 and content == "```":
tool_call_flag = True tool_call_flag = True
if tools_call is not None: if tools_call is not None:
@@ -374,7 +374,7 @@ class ConnectionHandler:
if content is not None and len(content) > 0: if content is not None and len(content) > 0:
if tool_call_flag: if tool_call_flag:
content_arguments+=content content_arguments += content
else: else:
response_message.append(content) response_message.append(content)
@@ -390,7 +390,7 @@ class ConnectionHandler:
current_text = full_text[processed_chars:] # 从未处理的位置开始 current_text = full_text[processed_chars:] # 从未处理的位置开始
# 查找最后一个有效标点 # 查找最后一个有效标点
punctuations = ("。", "？", "！", "；", "：") punctuations = ("。", "？", "！", "；", "：", ".", "?", "!", ";", ":", " ")
last_punct_pos = -1 last_punct_pos = -1
for punct in punctuations: for punct in punctuations:
pos = current_text.rfind(punct) pos = current_text.rfind(punct)
@@ -404,8 +404,17 @@ class ConnectionHandler:
if segment_text: if segment_text:
text_index += 1 text_index += 1
self.recode_first_last_text(segment_text, text_index) self.recode_first_last_text(segment_text, text_index)
future = self.executor.submit(self.speak_and_play, segment_text, text_index) if self.tts_stream:
self.tts_queue.put(future) stream_queue = queue.Queue()
self.executor.submit(self.speak_and_play_stream, segment_text, stream_queue)
self.tts_queue_stream.put({
"text": segment_text,
"chunk_queque": stream_queue,
"text_index": text_index
})
else:
future = self.executor.submit(self.speak_and_play, segment_text, text_index)
self.tts_queue.put(future)
processed_chars += len(segment_text_raw) # 更新已处理字符位置 processed_chars += len(segment_text_raw) # 更新已处理字符位置
# 处理最后剩余的文本 # 处理最后剩余的文本
@@ -415,12 +424,21 @@ class ConnectionHandler:
segment_text = get_string_no_punctuation_or_emoji(remaining_text) segment_text = get_string_no_punctuation_or_emoji(remaining_text)
if segment_text: if segment_text:
text_index += 1 text_index += 1
self.recode_first_last_text(segment_text, text_index) if self.tts_stream:
future = self.executor.submit(self.speak_and_play, segment_text, text_index) stream_queue = queue.Queue()
self.executor.submit(self.speak_and_play_stream, segment_text, stream_queue, text_index)
self.tts_queue_stream.put({
"text": segment_text,
"chunk_queque": stream_queue,
"text_index": text_index
})
else:
self.recode_first_last_text(segment_text, text_index)
future = self.executor.submit(self.speak_and_play, segment_text, text_index)
self.tts_queue.put(future) self.tts_queue.put(future)
# 存储对话内容 # 存储对话内容
if len(response_message)>0: if len(response_message) > 0:
self.dialogue.put(Message(role="assistant", content="".join(response_message))) self.dialogue.put(Message(role="assistant", content="".join(response_message)))
# 处理function call # 处理function call
@@ -435,14 +453,15 @@ class ConnectionHandler:
else: else:
return [] return []
function_arguments = json.loads(function_arguments) function_arguments = json.loads(function_arguments)
self.logger.bind(tag=TAG).info(f"function_name={function_name}, function_id={function_id}, function_arguments={function_arguments}") self.logger.bind(tag=TAG).info(
f"function_name={function_name}, function_id={function_id}, function_arguments={function_arguments}")
function_call_data = { function_call_data = {
"name": function_name, "name": function_name,
"id": function_id, "id": function_id,
"arguments": function_arguments "arguments": function_arguments
} }
result = handle_llm_function_call(self, function_call_data) result = handle_llm_function_call(self, function_call_data)
self._handle_function_result(result, function_call_data, text_index+1) self._handle_function_result(result, function_call_data, text_index + 1)
self.llm_finish_task = True self.llm_finish_task = True
self.logger.bind(tag=TAG).debug(json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False)) self.logger.bind(tag=TAG).debug(json.dumps(self.dialogue.get_llm_dialogue(), indent=4, ensure_ascii=False))
@@ -450,19 +469,26 @@ class ConnectionHandler:
return True return True
def _handle_function_result(self, result, function_call_data, text_index): def _handle_function_result(self, result, function_call_data, text_index):
if result.action == Action.RESPONSE: # 直接回复前端 if result.action == Action.RESPONSE: # 直接回复前端
text = result.response text = result.response
self.recode_first_last_text(text, text_index) self.recode_first_last_text(text, text_index)
future = self.executor.submit(self.speak_and_play, text, text_index) if self.tts_stream:
self.tts_queue.put(future) stream_queue = queue.Queue()
self.executor.submit(self.speak_and_play_stream, text, stream_queue, text_index)
self.tts_queue_stream.put({
"text": text,
"chunk_queque": stream_queue,
"text_index": text_index
})
else:
future = self.executor.submit(self.speak_and_play, text, text_index)
self.tts_queue.put(future)
self.dialogue.put(Message(role="assistant", content=text)) self.dialogue.put(Message(role="assistant", content=text))
if result.action == Action.REQLLM: # 调用函数后再请求llm生成回复 if result.action == Action.REQLLM: # 调用函数后再请求llm生成回复
text = result.response text = result.response
if result.action == Action.NOTFOUND: if result.action == Action.NOTFOUND:
text = result.response text = result.response
def _tts_priority_thread(self): def _tts_priority_thread(self):
if self.tts_stream: if self.tts_stream:
self._tts_priority_thread_stream() self._tts_priority_thread_stream()
@@ -540,11 +566,16 @@ class ConnectionHandler:
text = None text = None
try: try:
if self.tts_stream: if self.tts_stream:
chunk_queque, text, text_index = self.audio_play_queue.get() data, text, text_index = self.audio_play_queue.get()
future = asyncio.run_coroutine_threadsafe( if isinstance(data, list):
sendAudioMessageStream(self, chunk_queque, text, text_index), future = asyncio.run_coroutine_threadsafe(sendAudioMessage(self, data, text, text_index),
self.loop) self.loop)
future.result() future.result()
else:
future = asyncio.run_coroutine_threadsafe(
sendAudioMessageStream(self, data, text, text_index),
self.loop)
future.result()
else: else:
opus_datas, text, text_index = self.audio_play_queue.get() opus_datas, text, text_index = self.audio_play_queue.get()
future = asyncio.run_coroutine_threadsafe(sendAudioMessage(self, opus_datas, text, text_index), future = asyncio.run_coroutine_threadsafe(sendAudioMessage(self, opus_datas, text, text_index),
main/xiaozhi-server/core/providers/llm/openai/openai.py
+2 -1
View File
@@ -2,6 +2,7 @@ import openai
from core.utils.util import check_model_key from core.utils.util import check_model_key
from core.providers.llm.base import LLMProviderBase from core.providers.llm.base import LLMProviderBase
TAG = __name__
class LLMProvider(LLMProviderBase): class LLMProvider(LLMProviderBase):
def __init__(self, config): def __init__(self, config):
@@ -42,7 +43,7 @@ class LLMProvider(LLMProviderBase):
yield content yield content
except Exception as e: except Exception as e:
logger.bind(tag=TAG).error(f"Error in response generation: {e}") self.logger.bind(tag=TAG).error(f"Error in response generation: {e}")
def response_with_functions(self, session_id, dialogue, functions=None): def response_with_functions(self, session_id, dialogue, functions=None):
try: try: