Merge branch 'refs/heads/main' into perf-tool-call-optimization

This commit is contained in:
DaGou12138
2026-04-27 14:23:28 +08:00
27 changed files with 490 additions and 342 deletions
+47 -1
View File
@@ -396,7 +396,28 @@ ASR:
type: doubao_stream
appid: 你的火山引擎语音合成服务appid
access_token: 你的火山引擎语音合成服务access_token
cluster: volcengine_input_common
# 资源ID:小时版 volc.bigasr.sauc.duration,并发版 volc.bigasr.sauc.concurrent
resource_id: volc.bigasr.sauc.duration
# 热词、替换词使用流程:https://www.volcengine.com/docs/6561/155738
boosting_table_name: (选填)你的热词文件名称
correct_table_name: (选填)你的替换词文件名称
# 是否开启多语种识别模式
enable_multilingual: False
# 多语种识别当该键为空时,该模型支持中英文、上海话、闽南语,四川、陕西、粤语识别。当将其设置为特定键时,它可以识别指定语言。
# 详细语言列表参考 https://www.volcengine.com/docs/6561/1354869
# language: zh-cn
# 静音判定时长(ms),默认200ms
end_window_size: 200
output_dir: tmp/
DoubaoStreamASRV2:
# 豆包语音识别模型2.0(基于火山引擎seed-asr
# 开通地址:https://console.volcengine.com/speech/service/10038
# 价格更为便宜,建议在高并发场景下使用
type: doubao_stream
appid: 你的火山引擎语音合成服务appid
access_token: 你的火山引擎语音合成服务access_token
# 资源ID:小时版 volc.seedasr.sauc.duration,并发版 volc.seedasr.sauc.concurrent
resource_id: volc.seedasr.sauc.duration
# 热词、替换词使用流程:https://www.volcengine.com/docs/6561/155738
boosting_table_name: (选填)你的热词文件名称
correct_table_name: (选填)你的替换词文件名称
@@ -796,6 +817,31 @@ TTS:
# - source_speaker: zh_male_ahu_conversation_wvae_bigtts
# mix_factor: 0.4
# language: "中文" # 指定输出语种,如:中文、英语、日语、韩语等,请根据所选音色支持的语言进行设置,不填则默认为中文
#豆包语音合成模型2.0,支持双向流式tts(基于seed-tts-2.0资源)
HuoshanDoubleStreamTTSV2:
type: huoshan_double_stream
# 访问 https://console.volcengine.com/speech/service/10035 开通语音合成大模型,购买音色
# 在页面底部获取appid和access_token
# 资源ID固定为:seed-tts-2.0(豆包语音合成模型2.0
# 如果是机智云,把接口地址换成wss://bytedance.gizwitsapi.com/api/v3/tts/bidirection
# 机智云不需要天填 appid
ws_url: wss://openspeech.bytedance.com/api/v3/tts/bidirection
appid: 你的火山引擎语音合成服务appid
access_token: 你的火山引擎语音合成服务access_token
resource_id: seed-tts-2.0
speaker: zh_female_xiaohe_uranus_bigtts
# 开启WebSocket连接复用,默认复用(注意:复用后设备处于聆听状态时空闲链接会占并发数)
enable_ws_reuse: True
# 相关参数文档:https://www.volcengine.com/docs/6561/1329505
# 音频输出配置(audio_params)- 用户可自定义添加火山引擎支持的任何音频参数
audio_params:
speech_rate: 0 # 语速(-50~100)
loudness_rate: 0 # 音量(-50~100)
# 高级文本处理配置(additions)- 用户可自定义添加火山引擎支持的任何高级参数
additions:
post_process:
pitch: 0 # 音高(-12~12)
# language: "中文" # 指定输出语种,如:中文、英语、日语、韩语等,请根据所选音色支持的语言进行设置,不填则默认为中文
CosyVoiceSiliconflow:
type: siliconflow
# 硅基流动TTS
+1 -1
View File
@@ -5,7 +5,7 @@ from config.config_loader import load_config
from config.settings import check_config_file
from datetime import datetime
SERVER_VERSION = "0.9.2"
SERVER_VERSION = "0.9.3"
_logger_initialized = False
-6
View File
@@ -116,8 +116,6 @@ class ConnectionHandler:
self.memory = _memory
self.intent = _intent
self.is_exiting = False # 标记是否正在执行退出流程
# 为每个连接单独管理声纹识别
self.voiceprint_provider = None
@@ -312,10 +310,6 @@ class ConnectionHandler:
async def _route_message(self, message):
"""消息路由"""
# 退出状态丢弃所有消息
if self.is_exiting:
return
# 检查是否已经获取到真实的绑定状态
if not self.bind_completed_event.is_set():
# 还没有获取到真实状态,等待直到获取到真实状态或超时
@@ -7,12 +7,9 @@ TAG = __name__
async def handleAbortMessage(conn: "ConnectionHandler"):
if conn.close_after_chat or conn.is_exiting:
conn.logger.bind(tag=TAG).info("退出流程中被打断,直接关闭连接")
return
conn.logger.bind(tag=TAG).info("Abort message received")
# 设置成打断状态,会自动打断llm、tts任务
conn.close_after_chat = False
conn.client_abort = True
conn.clear_queues()
# 打断客户端说话状态
@@ -33,10 +33,6 @@ async def handle_user_intent(conn: "ConnectionHandler", text):
if await check_direct_exit(conn, filtered_text):
return True
# 明确再见不被打断
if conn.is_exiting:
return True
# 检查是否是唤醒词
if await checkWakeupWords(conn, filtered_text):
return True
@@ -62,7 +58,6 @@ async def check_direct_exit(conn: "ConnectionHandler", text):
if text == cmd:
conn.logger.bind(tag=TAG).info(f"识别到明确的退出命令: {text}")
await send_stt_message(conn, text)
conn.is_exiting = True
await conn.close()
return True
return False
@@ -15,8 +15,6 @@ TAG = __name__
async def handleAudioMessage(conn: "ConnectionHandler", audio):
if conn.is_exiting:
return
# 当前片段是否有人说话
have_voice = conn.vad.is_vad(conn, audio)
# 如果设备刚刚被唤醒,短暂忽略VAD检测
@@ -84,12 +84,6 @@ class ASRProviderBase(ABC):
async def handle_voice_stop(self, conn: "ConnectionHandler", asr_audio_task: List[bytes]):
"""并行处理ASR和声纹识别"""
try:
# 如果处于退出流程中,直接关闭连接,不处理新消息
if conn.close_after_chat or conn.is_exiting:
logger.bind(tag=TAG).info("退出流程中收到新消息,直接关闭连接")
await conn.close()
return
total_start_time = time.monotonic()
# 准备音频数据
@@ -26,11 +26,14 @@ class ASRProvider(ASRProviderBase):
self.asr_ws = None
self.forward_task = None
self.is_processing = False # 添加处理状态标志
self._is_stopping = False # 添加停止标志,防止竞态条件
# 配置参数
self.appid = str(config.get("appid"))
self.cluster = config.get("cluster")
self.access_token = config.get("access_token")
# 资源ID,用于区分不同的ASR模型(默认1.0模型小时版,v2版本使用seed-asr)
self.resource_id = config.get("resource_id", "volc.bigasr.sauc.duration")
self.boosting_table_name = config.get("boosting_table_name", "")
self.correct_table_name = config.get("correct_table_name", "")
self.output_dir = config.get("output_dir", "tmp/")
@@ -44,7 +47,7 @@ class ASRProvider(ASRProviderBase):
if self.enable_multilingual:
self.ws_url = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_nostream"
else:
self.ws_url = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel"
self.ws_url = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_async"
self.uid = config.get("uid", "streaming_asr_service")
self.workflow = config.get(
"workflow", "audio_in,resample,partition,vad,fe,decode,itn,nlu_punctuate"
@@ -146,7 +149,7 @@ class ASRProvider(ASRProviderBase):
return
# 发送当前音频数据
if self.asr_ws and self.is_processing:
if self.asr_ws and self.is_processing and not self._is_stopping:
try:
pcm_frame = self.decoder.decode(audio, 960)
payload = gzip.compress(pcm_frame)
@@ -254,6 +257,7 @@ class ASRProvider(ASRProviderBase):
await self.asr_ws.close()
self.asr_ws = None
self.is_processing = False
self._is_stopping = False
# 重置所有音频相关状态
conn.reset_audio_states()
@@ -262,9 +266,11 @@ class ASRProvider(ASRProviderBase):
asyncio.create_task(self.asr_ws.close())
self.asr_ws = None
self.is_processing = False
self._is_stopping = False
async def _send_stop_request(self):
"""发送最后一个音频帧以通知服务器结束"""
self._is_stopping = True # 先标记为停止状态,阻止后续音频发送
if self.asr_ws:
try:
# 发送结束标记的音频帧(gzip压缩的空数据)
@@ -283,7 +289,6 @@ class ASRProvider(ASRProviderBase):
req = {
"app": {
"appid": self.appid,
"cluster": self.cluster,
"token": self.access_token,
},
"user": {"uid": self.uid},
@@ -322,7 +327,7 @@ class ASRProvider(ASRProviderBase):
return {
"X-Api-App-Key": self.appid,
"X-Api-Access-Key": self.access_token,
"X-Api-Resource-Id": "volc.bigasr.sauc.duration",
"X-Api-Resource-Id": self.resource_id,
"X-Api-Connect-Id": str(uuid.uuid4()),
}
@@ -385,9 +390,17 @@ class ASRProvider(ASRProviderBase):
"payload_msg": error_msg,
}
# 获取JSON数据(跳过12字节头部)
# 获取JSON数据
try:
json_data = res[12:].decode("utf-8")
# 检查字节8-11是否为有效的JSON长度字段
# 格式:4字节头 + 4字节序列号 + 4字节长度 + JSON数据
length = int.from_bytes(res[8:12], "big")
if length > 0 and length <= len(res) - 12:
# 有长度字段,从字节12开始读取指定长度的JSON
json_data = res[12:12 + length].decode("utf-8")
else:
# 无长度字段或长度无效,尝试直接解析
json_data = res[8:].decode("utf-8")
result = json.loads(json_data)
logger.bind(tag=TAG).debug(f"成功解析JSON响应: {result}")
return {"payload_msg": result}
@@ -39,13 +39,12 @@ class LLMProvider(LLMProviderBase):
}
# 发起 POST 请求
response = requests.post(self.api_url, json=payload, headers=headers)
with requests.post(self.api_url, json=payload, headers=headers) as response:
# 检查请求是否成功
response.raise_for_status()
# 检查请求是否成功
response.raise_for_status()
# 解析返回数据
data = response.json()
# 解析返回数据
data = response.json()
speech = (
data.get("response", {})
.get("speech", {})
@@ -50,43 +50,46 @@ class LLMProvider(LLMProviderBase):
# 用于处理跨chunk的标签
buffer = ""
for chunk in responses:
try:
delta = (
chunk.choices[0].delta
if getattr(chunk, "choices", None)
else None
)
content = delta.content if hasattr(delta, "content") else ""
try:
for chunk in responses:
try:
delta = (
chunk.choices[0].delta
if getattr(chunk, "choices", None)
else None
)
content = delta.content if hasattr(delta, "content") else ""
if content:
# 将内容添加到缓冲区
buffer += content
if content:
# 将内容添加到缓冲区
buffer += content
# 处理缓冲区中的标签
while "<think>" in buffer and "</think>" in buffer:
# 找到完整的<think></think>标签并移除
pre = buffer.split("<think>", 1)[0]
post = buffer.split("</think>", 1)[1]
buffer = pre + post
# 处理缓冲区中的标签
while "<think>" in buffer and "</think>" in buffer:
# 找到完整的<think></think>标签并移除
pre = buffer.split("<think>", 1)[0]
post = buffer.split("</think>", 1)[1]
buffer = pre + post
# 处理只有开始标签的情况
if "<think>" in buffer:
is_active = False
buffer = buffer.split("<think>", 1)[0]
# 处理只有开始标签的情况
if "<think>" in buffer:
is_active = False
buffer = buffer.split("<think>", 1)[0]
# 处理只有结束标签的情况
if "</think>" in buffer:
is_active = True
buffer = buffer.split("</think>", 1)[1]
# 处理只有结束标签的情况
if "</think>" in buffer:
is_active = True
buffer = buffer.split("</think>", 1)[1]
# 如果当前处于活动状态且缓冲区有内容,则输出
if is_active and buffer:
yield buffer
buffer = "" # 清空缓冲区
# 如果当前处于活动状态且缓冲区有内容,则输出
if is_active and buffer:
yield buffer
buffer = "" # 清空缓冲区
except Exception as e:
logger.bind(tag=TAG).error(f"Error processing chunk: {e}")
except Exception as e:
logger.bind(tag=TAG).error(f"Error processing chunk: {e}")
finally:
responses.close()
def response_with_functions(self, session_id, dialogue, functions=None):
# 如果是qwen3模型,在用户最后一条消息中添加/no_think指令
@@ -117,49 +120,52 @@ class LLMProvider(LLMProviderBase):
is_active = True
buffer = ""
for chunk in stream:
try:
delta = (
chunk.choices[0].delta
if getattr(chunk, "choices", None)
else None
)
content = delta.content if hasattr(delta, "content") else None
tool_calls = (
delta.tool_calls if hasattr(delta, "tool_calls") else None
)
try:
for chunk in stream:
try:
delta = (
chunk.choices[0].delta
if getattr(chunk, "choices", None)
else None
)
content = delta.content if hasattr(delta, "content") else None
tool_calls = (
delta.tool_calls if hasattr(delta, "tool_calls") else None
)
# 如果是工具调用,直接传递
if tool_calls:
yield None, tool_calls
# 如果是工具调用,直接传递
if tool_calls:
yield None, tool_calls
continue
# 处理文本内容
if content:
# 将内容添加到缓冲区
buffer += content
# 处理缓冲区中的标签
while "<think>" in buffer and "</think>" in buffer:
# 找到完整的<think></think>标签并移除
pre = buffer.split("<think>", 1)[0]
post = buffer.split("</think>", 1)[1]
buffer = pre + post
# 处理只有开始标签的情况
if "<think>" in buffer:
is_active = False
buffer = buffer.split("<think>", 1)[0]
# 处理只有结束标签的情况
if "</think>" in buffer:
is_active = True
buffer = buffer.split("</think>", 1)[1]
# 如果当前处于活动状态且缓冲区有内容,则输出
if is_active and buffer:
yield buffer, None
buffer = "" # 清空缓冲区
except Exception as e:
logger.bind(tag=TAG).error(f"Error processing function chunk: {e}")
continue
# 处理文本内容
if content:
# 将内容添加到缓冲区
buffer += content
# 处理缓冲区中的标签
while "<think>" in buffer and "</think>" in buffer:
# 找到完整的<think></think>标签并移除
pre = buffer.split("<think>", 1)[0]
post = buffer.split("</think>", 1)[1]
buffer = pre + post
# 处理只有开始标签的情况
if "<think>" in buffer:
is_active = False
buffer = buffer.split("<think>", 1)[0]
# 处理只有结束标签的情况
if "</think>" in buffer:
is_active = True
buffer = buffer.split("</think>", 1)[1]
# 如果当前处于活动状态且缓冲区有内容,则输出
if is_active and buffer:
yield buffer, None
buffer = "" # 清空缓冲区
except Exception as e:
logger.bind(tag=TAG).error(f"Error processing function chunk: {e}")
continue
finally:
stream.close()
@@ -115,21 +115,24 @@ class LLMProvider(LLMProviderBase):
responses = self.client.chat.completions.create(**request_params)
is_active = True
for chunk in responses:
try:
delta = chunk.choices[0].delta if getattr(chunk, "choices", None) else None
content = getattr(delta, "content", "") if delta else ""
except IndexError:
content = ""
if content:
if "<think>" in content:
is_active = False
content = content.split("<think>")[0]
if "</think>" in content:
is_active = True
content = content.split("</think>")[-1]
if is_active:
yield content
try:
for chunk in responses:
try:
delta = chunk.choices[0].delta if getattr(chunk, "choices", None) else None
content = getattr(delta, "content", "") if delta else ""
except IndexError:
content = ""
if content:
if "<think>" in content:
is_active = False
content = content.split("<think>")[0]
if "</think>" in content:
is_active = True
content = content.split("</think>")[-1]
if is_active:
yield content
finally:
responses.close()
def response_with_functions(self, session_id, dialogue, functions=None, **kwargs):
dialogue = self.normalize_dialogue(dialogue)
@@ -157,16 +160,19 @@ class LLMProvider(LLMProviderBase):
stream = self.client.chat.completions.create(**request_params)
for chunk in stream:
if getattr(chunk, "choices", None):
delta = chunk.choices[0].delta
content = getattr(delta, "content", "")
tool_calls = getattr(delta, "tool_calls", None)
yield content, tool_calls
elif isinstance(getattr(chunk, "usage", None), CompletionUsage):
usage_info = getattr(chunk, "usage", None)
logger.bind(tag=TAG).info(
f"Token 消耗:输入 {getattr(usage_info, 'prompt_tokens', '未知')}"
f"输出 {getattr(usage_info, 'completion_tokens', '未知')}"
f"共计 {getattr(usage_info, 'total_tokens', '未知')}"
)
try:
for chunk in stream:
if getattr(chunk, "choices", None):
delta = chunk.choices[0].delta
content = getattr(delta, "content", "")
tool_calls = getattr(delta, "tool_calls", None)
yield content, tool_calls
elif isinstance(getattr(chunk, "usage", None), CompletionUsage):
usage_info = getattr(chunk, "usage", None)
logger.bind(tag=TAG).info(
f"Token 消耗:输入 {getattr(usage_info, 'prompt_tokens', '未知')}"
f"输出 {getattr(usage_info, 'completion_tokens', '未知')}"
f"共计 {getattr(usage_info, 'total_tokens', '未知')}"
)
finally:
stream.close()
@@ -74,12 +74,15 @@ class LLMProvider(LLMProviderBase):
tools=functions,
)
for chunk in stream:
delta = chunk.choices[0].delta
content = delta.content
tool_calls = delta.tool_calls
try:
for chunk in stream:
delta = chunk.choices[0].delta
content = delta.content
tool_calls = delta.tool_calls
if content:
yield content, tool_calls
elif tool_calls:
yield None, tool_calls
if content:
yield content, tool_calls
elif tool_calls:
yield None, tool_calls
finally:
stream.close()
@@ -65,69 +65,8 @@ class BaseASRTester:
class DoubaoStreamASRTester(BaseASRTester):
def __init__(self):
super().__init__("DoubaoStreamASR")
def _generate_header(
self,
version=0x01,
message_type=0x01,
message_type_specific_flags=0x00,
serial_method=0x01,
compression_type=0x01,
reserved_data=0x00,
extension_header: bytes = b"",
):
"""生成协议头(修复:使用正确的Header格式)"""
header = bytearray()
header_size = int(len(extension_header) / 4) + 1
header.append((version << 4) | header_size)
header.append((message_type << 4) | message_type_specific_flags)
header.append((serial_method << 4) | compression_type)
header.append(reserved_data)
header.extend(extension_header)
return header
def _generate_audio_default_header(self):
"""生成音频数据Header"""
return self._generate_header(
version=0x01,
message_type=0x02,
message_type_specific_flags=0x00, # 普通音频帧
serial_method=0x01,
compression_type=0x01,
)
def _generate_last_audio_header(self):
"""生成最后一帧音频的Header(标记音频结束)"""
return self._generate_header(
version=0x01,
message_type=0x02,
message_type_specific_flags=0x02, # 0x02表示这是最后一帧
serial_method=0x01,
compression_type=0x01,
)
def _parse_response(self, res: bytes) -> dict:
try:
if len(res) < 4:
return {"error": "响应数据长度不足"}
header = res[:4]
message_type = header[1] >> 4
if message_type == 0x0F:
code = int.from_bytes(res[4:8], "big", signed=False)
msg_length = int.from_bytes(res[8:12], "big", signed=False)
error_msg = json.loads(res[12:].decode("utf-8"))
return {
"code": code,
"msg_length": msg_length,
"payload_msg": error_msg
}
try:
json_data = res[12:].decode("utf-8")
return {"payload_msg": json.loads(json_data)}
except (UnicodeDecodeError, json.JSONDecodeError):
return {"error": "JSON解析失败"}
except Exception:
return {"error": "解析响应失败"}
from core.providers.asr.doubao_stream import ASRProvider as DoubaoStreamProvider
self.provider = DoubaoStreamProvider(self.asr_config, delete_audio_file=False)
async def test(self, test_count=5):
if not self.test_audio_files:
@@ -138,59 +77,29 @@ class DoubaoStreamASRTester(BaseASRTester):
latencies = []
for i in range(test_count):
try:
ws_url = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel"
appid = self.asr_config["appid"]
access_token = self.asr_config["access_token"]
cluster = self.asr_config.get("cluster", "volcengine_input_common")
uid = self.asr_config.get("uid", "streaming_asr_service")
start_time = time.time()
headers = {
"X-Api-App-Key": appid,
"X-Api-Access-Key": access_token,
"X-Api-Resource-Id": "volc.bigasr.sauc.duration",
"X-Api-Connect-Id": str(uuid.uuid4())
}
headers = self.provider.token_auth()
async with websockets.connect(
ws_url,
self.provider.ws_url,
additional_headers=headers,
max_size=1000000000,
ping_interval=None,
ping_timeout=None,
close_timeout=10
) as ws:
request_params = {
"app": {"appid": appid, "cluster": cluster, "token": access_token},
"user": {"uid": uid},
"request": {
"reqid": str(uuid.uuid4()),
"workflow": "audio_in,resample,partition,vad,fe,decode,itn,nlu_punctuate",
"show_utterances": True,
"result_type": "single",
"sequence": 1
},
"audio": {
"format": "pcm",
"codec": "pcm",
"rate": 16000,
"language": "zh-CN",
"bits": 16,
"channel": 1,
"sample_rate": 16000
}
}
request_params = self.provider.construct_request(str(uuid.uuid4()))
payload_bytes = str.encode(json.dumps(request_params))
payload_bytes = gzip.compress(payload_bytes)
full_client_request = self._generate_header()
full_client_request = bytearray(self.provider.generate_header())
full_client_request.extend((len(payload_bytes)).to_bytes(4, "big"))
full_client_request.extend(payload_bytes)
await ws.send(full_client_request)
init_res = await ws.recv()
result = self._parse_response(init_res)
result = self.provider.parse_response(init_res)
if "code" in result and result["code"] != 1000:
raise Exception(f"初始化失败: {result.get('payload_msg', {}).get('error', '未知错误')}")
@@ -200,7 +109,7 @@ class DoubaoStreamASRTester(BaseASRTester):
# 发送音频数据(使用最后一帧标记,告诉服务端音频已结束)
payload = gzip.compress(audio_data)
audio_request = bytearray(self._generate_last_audio_header()) # 修复:使用最后一帧Header
audio_request = bytearray(self.provider.generate_last_audio_default_header())
audio_request.extend(len(payload).to_bytes(4, "big"))
audio_request.extend(payload)
await ws.send(audio_request)
@@ -31,12 +31,12 @@ handle_exit_intent_function_desc = {
"handle_exit_intent", handle_exit_intent_function_desc, ToolType.SYSTEM_CTL
)
def handle_exit_intent(conn: "ConnectionHandler", say_goodbye: str | None = None):
conn.is_exiting = True
# 处理退出意图
try:
if say_goodbye is None:
say_goodbye = "再见,祝您生活愉快!"
conn.close_after_chat = True
if not conn.close_after_chat:
conn.close_after_chat = True
logger.bind(tag=TAG).info(f"退出意图已处理:{say_goodbye}")
return ActionResponse(
action=Action.RESPONSE, result="退出意图已处理", response=say_goodbye