mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-26 09:03:54 +08:00
Merge branch 'refs/heads/main' into perf-tool-call-optimization
This commit is contained in:
@@ -396,7 +396,28 @@ ASR:
|
||||
type: doubao_stream
|
||||
appid: 你的火山引擎语音合成服务appid
|
||||
access_token: 你的火山引擎语音合成服务access_token
|
||||
cluster: volcengine_input_common
|
||||
# 资源ID:小时版 volc.bigasr.sauc.duration,并发版 volc.bigasr.sauc.concurrent
|
||||
resource_id: volc.bigasr.sauc.duration
|
||||
# 热词、替换词使用流程:https://www.volcengine.com/docs/6561/155738
|
||||
boosting_table_name: (选填)你的热词文件名称
|
||||
correct_table_name: (选填)你的替换词文件名称
|
||||
# 是否开启多语种识别模式
|
||||
enable_multilingual: False
|
||||
# 多语种识别当该键为空时,该模型支持中英文、上海话、闽南语,四川、陕西、粤语识别。当将其设置为特定键时,它可以识别指定语言。
|
||||
# 详细语言列表参考 https://www.volcengine.com/docs/6561/1354869
|
||||
# language: zh-cn
|
||||
# 静音判定时长(ms),默认200ms
|
||||
end_window_size: 200
|
||||
output_dir: tmp/
|
||||
DoubaoStreamASRV2:
|
||||
# 豆包语音识别模型2.0(基于火山引擎seed-asr)
|
||||
# 开通地址:https://console.volcengine.com/speech/service/10038
|
||||
# 价格更为便宜,建议在高并发场景下使用
|
||||
type: doubao_stream
|
||||
appid: 你的火山引擎语音合成服务appid
|
||||
access_token: 你的火山引擎语音合成服务access_token
|
||||
# 资源ID:小时版 volc.seedasr.sauc.duration,并发版 volc.seedasr.sauc.concurrent
|
||||
resource_id: volc.seedasr.sauc.duration
|
||||
# 热词、替换词使用流程:https://www.volcengine.com/docs/6561/155738
|
||||
boosting_table_name: (选填)你的热词文件名称
|
||||
correct_table_name: (选填)你的替换词文件名称
|
||||
@@ -796,6 +817,31 @@ TTS:
|
||||
# - source_speaker: zh_male_ahu_conversation_wvae_bigtts
|
||||
# mix_factor: 0.4
|
||||
# language: "中文" # 指定输出语种,如:中文、英语、日语、韩语等,请根据所选音色支持的语言进行设置,不填则默认为中文
|
||||
#豆包语音合成模型2.0,支持双向流式tts(基于seed-tts-2.0资源)
|
||||
HuoshanDoubleStreamTTSV2:
|
||||
type: huoshan_double_stream
|
||||
# 访问 https://console.volcengine.com/speech/service/10035 开通语音合成大模型,购买音色
|
||||
# 在页面底部获取appid和access_token
|
||||
# 资源ID固定为:seed-tts-2.0(豆包语音合成模型2.0)
|
||||
# 如果是机智云,把接口地址换成wss://bytedance.gizwitsapi.com/api/v3/tts/bidirection
|
||||
# 机智云不需要天填 appid
|
||||
ws_url: wss://openspeech.bytedance.com/api/v3/tts/bidirection
|
||||
appid: 你的火山引擎语音合成服务appid
|
||||
access_token: 你的火山引擎语音合成服务access_token
|
||||
resource_id: seed-tts-2.0
|
||||
speaker: zh_female_xiaohe_uranus_bigtts
|
||||
# 开启WebSocket连接复用,默认复用(注意:复用后设备处于聆听状态时空闲链接会占并发数)
|
||||
enable_ws_reuse: True
|
||||
# 相关参数文档:https://www.volcengine.com/docs/6561/1329505
|
||||
# 音频输出配置(audio_params)- 用户可自定义添加火山引擎支持的任何音频参数
|
||||
audio_params:
|
||||
speech_rate: 0 # 语速(-50~100)
|
||||
loudness_rate: 0 # 音量(-50~100)
|
||||
# 高级文本处理配置(additions)- 用户可自定义添加火山引擎支持的任何高级参数
|
||||
additions:
|
||||
post_process:
|
||||
pitch: 0 # 音高(-12~12)
|
||||
# language: "中文" # 指定输出语种,如:中文、英语、日语、韩语等,请根据所选音色支持的语言进行设置,不填则默认为中文
|
||||
CosyVoiceSiliconflow:
|
||||
type: siliconflow
|
||||
# 硅基流动TTS
|
||||
|
||||
@@ -5,7 +5,7 @@ from config.config_loader import load_config
|
||||
from config.settings import check_config_file
|
||||
from datetime import datetime
|
||||
|
||||
SERVER_VERSION = "0.9.2"
|
||||
SERVER_VERSION = "0.9.3"
|
||||
_logger_initialized = False
|
||||
|
||||
|
||||
|
||||
@@ -116,8 +116,6 @@ class ConnectionHandler:
|
||||
self.memory = _memory
|
||||
self.intent = _intent
|
||||
|
||||
self.is_exiting = False # 标记是否正在执行退出流程
|
||||
|
||||
# 为每个连接单独管理声纹识别
|
||||
self.voiceprint_provider = None
|
||||
|
||||
@@ -312,10 +310,6 @@ class ConnectionHandler:
|
||||
|
||||
async def _route_message(self, message):
|
||||
"""消息路由"""
|
||||
# 退出状态丢弃所有消息
|
||||
if self.is_exiting:
|
||||
return
|
||||
|
||||
# 检查是否已经获取到真实的绑定状态
|
||||
if not self.bind_completed_event.is_set():
|
||||
# 还没有获取到真实状态,等待直到获取到真实状态或超时
|
||||
|
||||
@@ -7,12 +7,9 @@ TAG = __name__
|
||||
|
||||
|
||||
async def handleAbortMessage(conn: "ConnectionHandler"):
|
||||
if conn.close_after_chat or conn.is_exiting:
|
||||
conn.logger.bind(tag=TAG).info("退出流程中被打断,直接关闭连接")
|
||||
return
|
||||
|
||||
conn.logger.bind(tag=TAG).info("Abort message received")
|
||||
# 设置成打断状态,会自动打断llm、tts任务
|
||||
conn.close_after_chat = False
|
||||
conn.client_abort = True
|
||||
conn.clear_queues()
|
||||
# 打断客户端说话状态
|
||||
|
||||
@@ -33,10 +33,6 @@ async def handle_user_intent(conn: "ConnectionHandler", text):
|
||||
if await check_direct_exit(conn, filtered_text):
|
||||
return True
|
||||
|
||||
# 明确再见不被打断
|
||||
if conn.is_exiting:
|
||||
return True
|
||||
|
||||
# 检查是否是唤醒词
|
||||
if await checkWakeupWords(conn, filtered_text):
|
||||
return True
|
||||
@@ -62,7 +58,6 @@ async def check_direct_exit(conn: "ConnectionHandler", text):
|
||||
if text == cmd:
|
||||
conn.logger.bind(tag=TAG).info(f"识别到明确的退出命令: {text}")
|
||||
await send_stt_message(conn, text)
|
||||
conn.is_exiting = True
|
||||
await conn.close()
|
||||
return True
|
||||
return False
|
||||
|
||||
@@ -15,8 +15,6 @@ TAG = __name__
|
||||
|
||||
|
||||
async def handleAudioMessage(conn: "ConnectionHandler", audio):
|
||||
if conn.is_exiting:
|
||||
return
|
||||
# 当前片段是否有人说话
|
||||
have_voice = conn.vad.is_vad(conn, audio)
|
||||
# 如果设备刚刚被唤醒,短暂忽略VAD检测
|
||||
|
||||
@@ -84,12 +84,6 @@ class ASRProviderBase(ABC):
|
||||
async def handle_voice_stop(self, conn: "ConnectionHandler", asr_audio_task: List[bytes]):
|
||||
"""并行处理ASR和声纹识别"""
|
||||
try:
|
||||
# 如果处于退出流程中,直接关闭连接,不处理新消息
|
||||
if conn.close_after_chat or conn.is_exiting:
|
||||
logger.bind(tag=TAG).info("退出流程中收到新消息,直接关闭连接")
|
||||
await conn.close()
|
||||
return
|
||||
|
||||
total_start_time = time.monotonic()
|
||||
|
||||
# 准备音频数据
|
||||
|
||||
@@ -26,11 +26,14 @@ class ASRProvider(ASRProviderBase):
|
||||
self.asr_ws = None
|
||||
self.forward_task = None
|
||||
self.is_processing = False # 添加处理状态标志
|
||||
self._is_stopping = False # 添加停止标志,防止竞态条件
|
||||
|
||||
# 配置参数
|
||||
self.appid = str(config.get("appid"))
|
||||
self.cluster = config.get("cluster")
|
||||
self.access_token = config.get("access_token")
|
||||
# 资源ID,用于区分不同的ASR模型(默认1.0模型小时版,v2版本使用seed-asr)
|
||||
self.resource_id = config.get("resource_id", "volc.bigasr.sauc.duration")
|
||||
|
||||
self.boosting_table_name = config.get("boosting_table_name", "")
|
||||
self.correct_table_name = config.get("correct_table_name", "")
|
||||
self.output_dir = config.get("output_dir", "tmp/")
|
||||
@@ -44,7 +47,7 @@ class ASRProvider(ASRProviderBase):
|
||||
if self.enable_multilingual:
|
||||
self.ws_url = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_nostream"
|
||||
else:
|
||||
self.ws_url = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel"
|
||||
self.ws_url = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_async"
|
||||
self.uid = config.get("uid", "streaming_asr_service")
|
||||
self.workflow = config.get(
|
||||
"workflow", "audio_in,resample,partition,vad,fe,decode,itn,nlu_punctuate"
|
||||
@@ -146,7 +149,7 @@ class ASRProvider(ASRProviderBase):
|
||||
return
|
||||
|
||||
# 发送当前音频数据
|
||||
if self.asr_ws and self.is_processing:
|
||||
if self.asr_ws and self.is_processing and not self._is_stopping:
|
||||
try:
|
||||
pcm_frame = self.decoder.decode(audio, 960)
|
||||
payload = gzip.compress(pcm_frame)
|
||||
@@ -254,6 +257,7 @@ class ASRProvider(ASRProviderBase):
|
||||
await self.asr_ws.close()
|
||||
self.asr_ws = None
|
||||
self.is_processing = False
|
||||
self._is_stopping = False
|
||||
# 重置所有音频相关状态
|
||||
conn.reset_audio_states()
|
||||
|
||||
@@ -262,9 +266,11 @@ class ASRProvider(ASRProviderBase):
|
||||
asyncio.create_task(self.asr_ws.close())
|
||||
self.asr_ws = None
|
||||
self.is_processing = False
|
||||
self._is_stopping = False
|
||||
|
||||
async def _send_stop_request(self):
|
||||
"""发送最后一个音频帧以通知服务器结束"""
|
||||
self._is_stopping = True # 先标记为停止状态,阻止后续音频发送
|
||||
if self.asr_ws:
|
||||
try:
|
||||
# 发送结束标记的音频帧(gzip压缩的空数据)
|
||||
@@ -283,7 +289,6 @@ class ASRProvider(ASRProviderBase):
|
||||
req = {
|
||||
"app": {
|
||||
"appid": self.appid,
|
||||
"cluster": self.cluster,
|
||||
"token": self.access_token,
|
||||
},
|
||||
"user": {"uid": self.uid},
|
||||
@@ -322,7 +327,7 @@ class ASRProvider(ASRProviderBase):
|
||||
return {
|
||||
"X-Api-App-Key": self.appid,
|
||||
"X-Api-Access-Key": self.access_token,
|
||||
"X-Api-Resource-Id": "volc.bigasr.sauc.duration",
|
||||
"X-Api-Resource-Id": self.resource_id,
|
||||
"X-Api-Connect-Id": str(uuid.uuid4()),
|
||||
}
|
||||
|
||||
@@ -385,9 +390,17 @@ class ASRProvider(ASRProviderBase):
|
||||
"payload_msg": error_msg,
|
||||
}
|
||||
|
||||
# 获取JSON数据(跳过12字节头部)
|
||||
# 获取JSON数据
|
||||
try:
|
||||
json_data = res[12:].decode("utf-8")
|
||||
# 检查字节8-11是否为有效的JSON长度字段
|
||||
# 格式:4字节头 + 4字节序列号 + 4字节长度 + JSON数据
|
||||
length = int.from_bytes(res[8:12], "big")
|
||||
if length > 0 and length <= len(res) - 12:
|
||||
# 有长度字段,从字节12开始读取指定长度的JSON
|
||||
json_data = res[12:12 + length].decode("utf-8")
|
||||
else:
|
||||
# 无长度字段或长度无效,尝试直接解析
|
||||
json_data = res[8:].decode("utf-8")
|
||||
result = json.loads(json_data)
|
||||
logger.bind(tag=TAG).debug(f"成功解析JSON响应: {result}")
|
||||
return {"payload_msg": result}
|
||||
|
||||
@@ -39,13 +39,12 @@ class LLMProvider(LLMProviderBase):
|
||||
}
|
||||
|
||||
# 发起 POST 请求
|
||||
response = requests.post(self.api_url, json=payload, headers=headers)
|
||||
with requests.post(self.api_url, json=payload, headers=headers) as response:
|
||||
# 检查请求是否成功
|
||||
response.raise_for_status()
|
||||
|
||||
# 检查请求是否成功
|
||||
response.raise_for_status()
|
||||
|
||||
# 解析返回数据
|
||||
data = response.json()
|
||||
# 解析返回数据
|
||||
data = response.json()
|
||||
speech = (
|
||||
data.get("response", {})
|
||||
.get("speech", {})
|
||||
|
||||
@@ -50,43 +50,46 @@ class LLMProvider(LLMProviderBase):
|
||||
# 用于处理跨chunk的标签
|
||||
buffer = ""
|
||||
|
||||
for chunk in responses:
|
||||
try:
|
||||
delta = (
|
||||
chunk.choices[0].delta
|
||||
if getattr(chunk, "choices", None)
|
||||
else None
|
||||
)
|
||||
content = delta.content if hasattr(delta, "content") else ""
|
||||
try:
|
||||
for chunk in responses:
|
||||
try:
|
||||
delta = (
|
||||
chunk.choices[0].delta
|
||||
if getattr(chunk, "choices", None)
|
||||
else None
|
||||
)
|
||||
content = delta.content if hasattr(delta, "content") else ""
|
||||
|
||||
if content:
|
||||
# 将内容添加到缓冲区
|
||||
buffer += content
|
||||
if content:
|
||||
# 将内容添加到缓冲区
|
||||
buffer += content
|
||||
|
||||
# 处理缓冲区中的标签
|
||||
while "<think>" in buffer and "</think>" in buffer:
|
||||
# 找到完整的<think></think>标签并移除
|
||||
pre = buffer.split("<think>", 1)[0]
|
||||
post = buffer.split("</think>", 1)[1]
|
||||
buffer = pre + post
|
||||
# 处理缓冲区中的标签
|
||||
while "<think>" in buffer and "</think>" in buffer:
|
||||
# 找到完整的<think></think>标签并移除
|
||||
pre = buffer.split("<think>", 1)[0]
|
||||
post = buffer.split("</think>", 1)[1]
|
||||
buffer = pre + post
|
||||
|
||||
# 处理只有开始标签的情况
|
||||
if "<think>" in buffer:
|
||||
is_active = False
|
||||
buffer = buffer.split("<think>", 1)[0]
|
||||
# 处理只有开始标签的情况
|
||||
if "<think>" in buffer:
|
||||
is_active = False
|
||||
buffer = buffer.split("<think>", 1)[0]
|
||||
|
||||
# 处理只有结束标签的情况
|
||||
if "</think>" in buffer:
|
||||
is_active = True
|
||||
buffer = buffer.split("</think>", 1)[1]
|
||||
# 处理只有结束标签的情况
|
||||
if "</think>" in buffer:
|
||||
is_active = True
|
||||
buffer = buffer.split("</think>", 1)[1]
|
||||
|
||||
# 如果当前处于活动状态且缓冲区有内容,则输出
|
||||
if is_active and buffer:
|
||||
yield buffer
|
||||
buffer = "" # 清空缓冲区
|
||||
# 如果当前处于活动状态且缓冲区有内容,则输出
|
||||
if is_active and buffer:
|
||||
yield buffer
|
||||
buffer = "" # 清空缓冲区
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"Error processing chunk: {e}")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"Error processing chunk: {e}")
|
||||
finally:
|
||||
responses.close()
|
||||
|
||||
def response_with_functions(self, session_id, dialogue, functions=None):
|
||||
# 如果是qwen3模型,在用户最后一条消息中添加/no_think指令
|
||||
@@ -117,49 +120,52 @@ class LLMProvider(LLMProviderBase):
|
||||
is_active = True
|
||||
buffer = ""
|
||||
|
||||
for chunk in stream:
|
||||
try:
|
||||
delta = (
|
||||
chunk.choices[0].delta
|
||||
if getattr(chunk, "choices", None)
|
||||
else None
|
||||
)
|
||||
content = delta.content if hasattr(delta, "content") else None
|
||||
tool_calls = (
|
||||
delta.tool_calls if hasattr(delta, "tool_calls") else None
|
||||
)
|
||||
try:
|
||||
for chunk in stream:
|
||||
try:
|
||||
delta = (
|
||||
chunk.choices[0].delta
|
||||
if getattr(chunk, "choices", None)
|
||||
else None
|
||||
)
|
||||
content = delta.content if hasattr(delta, "content") else None
|
||||
tool_calls = (
|
||||
delta.tool_calls if hasattr(delta, "tool_calls") else None
|
||||
)
|
||||
|
||||
# 如果是工具调用,直接传递
|
||||
if tool_calls:
|
||||
yield None, tool_calls
|
||||
# 如果是工具调用,直接传递
|
||||
if tool_calls:
|
||||
yield None, tool_calls
|
||||
continue
|
||||
|
||||
# 处理文本内容
|
||||
if content:
|
||||
# 将内容添加到缓冲区
|
||||
buffer += content
|
||||
|
||||
# 处理缓冲区中的标签
|
||||
while "<think>" in buffer and "</think>" in buffer:
|
||||
# 找到完整的<think></think>标签并移除
|
||||
pre = buffer.split("<think>", 1)[0]
|
||||
post = buffer.split("</think>", 1)[1]
|
||||
buffer = pre + post
|
||||
|
||||
# 处理只有开始标签的情况
|
||||
if "<think>" in buffer:
|
||||
is_active = False
|
||||
buffer = buffer.split("<think>", 1)[0]
|
||||
|
||||
# 处理只有结束标签的情况
|
||||
if "</think>" in buffer:
|
||||
is_active = True
|
||||
buffer = buffer.split("</think>", 1)[1]
|
||||
|
||||
# 如果当前处于活动状态且缓冲区有内容,则输出
|
||||
if is_active and buffer:
|
||||
yield buffer, None
|
||||
buffer = "" # 清空缓冲区
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"Error processing function chunk: {e}")
|
||||
continue
|
||||
|
||||
# 处理文本内容
|
||||
if content:
|
||||
# 将内容添加到缓冲区
|
||||
buffer += content
|
||||
|
||||
# 处理缓冲区中的标签
|
||||
while "<think>" in buffer and "</think>" in buffer:
|
||||
# 找到完整的<think></think>标签并移除
|
||||
pre = buffer.split("<think>", 1)[0]
|
||||
post = buffer.split("</think>", 1)[1]
|
||||
buffer = pre + post
|
||||
|
||||
# 处理只有开始标签的情况
|
||||
if "<think>" in buffer:
|
||||
is_active = False
|
||||
buffer = buffer.split("<think>", 1)[0]
|
||||
|
||||
# 处理只有结束标签的情况
|
||||
if "</think>" in buffer:
|
||||
is_active = True
|
||||
buffer = buffer.split("</think>", 1)[1]
|
||||
|
||||
# 如果当前处于活动状态且缓冲区有内容,则输出
|
||||
if is_active and buffer:
|
||||
yield buffer, None
|
||||
buffer = "" # 清空缓冲区
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"Error processing function chunk: {e}")
|
||||
continue
|
||||
finally:
|
||||
stream.close()
|
||||
|
||||
@@ -115,21 +115,24 @@ class LLMProvider(LLMProviderBase):
|
||||
responses = self.client.chat.completions.create(**request_params)
|
||||
|
||||
is_active = True
|
||||
for chunk in responses:
|
||||
try:
|
||||
delta = chunk.choices[0].delta if getattr(chunk, "choices", None) else None
|
||||
content = getattr(delta, "content", "") if delta else ""
|
||||
except IndexError:
|
||||
content = ""
|
||||
if content:
|
||||
if "<think>" in content:
|
||||
is_active = False
|
||||
content = content.split("<think>")[0]
|
||||
if "</think>" in content:
|
||||
is_active = True
|
||||
content = content.split("</think>")[-1]
|
||||
if is_active:
|
||||
yield content
|
||||
try:
|
||||
for chunk in responses:
|
||||
try:
|
||||
delta = chunk.choices[0].delta if getattr(chunk, "choices", None) else None
|
||||
content = getattr(delta, "content", "") if delta else ""
|
||||
except IndexError:
|
||||
content = ""
|
||||
if content:
|
||||
if "<think>" in content:
|
||||
is_active = False
|
||||
content = content.split("<think>")[0]
|
||||
if "</think>" in content:
|
||||
is_active = True
|
||||
content = content.split("</think>")[-1]
|
||||
if is_active:
|
||||
yield content
|
||||
finally:
|
||||
responses.close()
|
||||
|
||||
def response_with_functions(self, session_id, dialogue, functions=None, **kwargs):
|
||||
dialogue = self.normalize_dialogue(dialogue)
|
||||
@@ -157,16 +160,19 @@ class LLMProvider(LLMProviderBase):
|
||||
|
||||
stream = self.client.chat.completions.create(**request_params)
|
||||
|
||||
for chunk in stream:
|
||||
if getattr(chunk, "choices", None):
|
||||
delta = chunk.choices[0].delta
|
||||
content = getattr(delta, "content", "")
|
||||
tool_calls = getattr(delta, "tool_calls", None)
|
||||
yield content, tool_calls
|
||||
elif isinstance(getattr(chunk, "usage", None), CompletionUsage):
|
||||
usage_info = getattr(chunk, "usage", None)
|
||||
logger.bind(tag=TAG).info(
|
||||
f"Token 消耗:输入 {getattr(usage_info, 'prompt_tokens', '未知')},"
|
||||
f"输出 {getattr(usage_info, 'completion_tokens', '未知')},"
|
||||
f"共计 {getattr(usage_info, 'total_tokens', '未知')}"
|
||||
)
|
||||
try:
|
||||
for chunk in stream:
|
||||
if getattr(chunk, "choices", None):
|
||||
delta = chunk.choices[0].delta
|
||||
content = getattr(delta, "content", "")
|
||||
tool_calls = getattr(delta, "tool_calls", None)
|
||||
yield content, tool_calls
|
||||
elif isinstance(getattr(chunk, "usage", None), CompletionUsage):
|
||||
usage_info = getattr(chunk, "usage", None)
|
||||
logger.bind(tag=TAG).info(
|
||||
f"Token 消耗:输入 {getattr(usage_info, 'prompt_tokens', '未知')},"
|
||||
f"输出 {getattr(usage_info, 'completion_tokens', '未知')},"
|
||||
f"共计 {getattr(usage_info, 'total_tokens', '未知')}"
|
||||
)
|
||||
finally:
|
||||
stream.close()
|
||||
|
||||
@@ -74,12 +74,15 @@ class LLMProvider(LLMProviderBase):
|
||||
tools=functions,
|
||||
)
|
||||
|
||||
for chunk in stream:
|
||||
delta = chunk.choices[0].delta
|
||||
content = delta.content
|
||||
tool_calls = delta.tool_calls
|
||||
try:
|
||||
for chunk in stream:
|
||||
delta = chunk.choices[0].delta
|
||||
content = delta.content
|
||||
tool_calls = delta.tool_calls
|
||||
|
||||
if content:
|
||||
yield content, tool_calls
|
||||
elif tool_calls:
|
||||
yield None, tool_calls
|
||||
if content:
|
||||
yield content, tool_calls
|
||||
elif tool_calls:
|
||||
yield None, tool_calls
|
||||
finally:
|
||||
stream.close()
|
||||
|
||||
@@ -65,69 +65,8 @@ class BaseASRTester:
|
||||
class DoubaoStreamASRTester(BaseASRTester):
|
||||
def __init__(self):
|
||||
super().__init__("DoubaoStreamASR")
|
||||
|
||||
def _generate_header(
|
||||
self,
|
||||
version=0x01,
|
||||
message_type=0x01,
|
||||
message_type_specific_flags=0x00,
|
||||
serial_method=0x01,
|
||||
compression_type=0x01,
|
||||
reserved_data=0x00,
|
||||
extension_header: bytes = b"",
|
||||
):
|
||||
"""生成协议头(修复:使用正确的Header格式)"""
|
||||
header = bytearray()
|
||||
header_size = int(len(extension_header) / 4) + 1
|
||||
header.append((version << 4) | header_size)
|
||||
header.append((message_type << 4) | message_type_specific_flags)
|
||||
header.append((serial_method << 4) | compression_type)
|
||||
header.append(reserved_data)
|
||||
header.extend(extension_header)
|
||||
return header
|
||||
|
||||
def _generate_audio_default_header(self):
|
||||
"""生成音频数据Header"""
|
||||
return self._generate_header(
|
||||
version=0x01,
|
||||
message_type=0x02,
|
||||
message_type_specific_flags=0x00, # 普通音频帧
|
||||
serial_method=0x01,
|
||||
compression_type=0x01,
|
||||
)
|
||||
|
||||
def _generate_last_audio_header(self):
|
||||
"""生成最后一帧音频的Header(标记音频结束)"""
|
||||
return self._generate_header(
|
||||
version=0x01,
|
||||
message_type=0x02,
|
||||
message_type_specific_flags=0x02, # 0x02表示这是最后一帧
|
||||
serial_method=0x01,
|
||||
compression_type=0x01,
|
||||
)
|
||||
|
||||
def _parse_response(self, res: bytes) -> dict:
|
||||
try:
|
||||
if len(res) < 4:
|
||||
return {"error": "响应数据长度不足"}
|
||||
header = res[:4]
|
||||
message_type = header[1] >> 4
|
||||
if message_type == 0x0F:
|
||||
code = int.from_bytes(res[4:8], "big", signed=False)
|
||||
msg_length = int.from_bytes(res[8:12], "big", signed=False)
|
||||
error_msg = json.loads(res[12:].decode("utf-8"))
|
||||
return {
|
||||
"code": code,
|
||||
"msg_length": msg_length,
|
||||
"payload_msg": error_msg
|
||||
}
|
||||
try:
|
||||
json_data = res[12:].decode("utf-8")
|
||||
return {"payload_msg": json.loads(json_data)}
|
||||
except (UnicodeDecodeError, json.JSONDecodeError):
|
||||
return {"error": "JSON解析失败"}
|
||||
except Exception:
|
||||
return {"error": "解析响应失败"}
|
||||
from core.providers.asr.doubao_stream import ASRProvider as DoubaoStreamProvider
|
||||
self.provider = DoubaoStreamProvider(self.asr_config, delete_audio_file=False)
|
||||
|
||||
async def test(self, test_count=5):
|
||||
if not self.test_audio_files:
|
||||
@@ -138,59 +77,29 @@ class DoubaoStreamASRTester(BaseASRTester):
|
||||
latencies = []
|
||||
for i in range(test_count):
|
||||
try:
|
||||
ws_url = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel"
|
||||
appid = self.asr_config["appid"]
|
||||
access_token = self.asr_config["access_token"]
|
||||
cluster = self.asr_config.get("cluster", "volcengine_input_common")
|
||||
uid = self.asr_config.get("uid", "streaming_asr_service")
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
headers = {
|
||||
"X-Api-App-Key": appid,
|
||||
"X-Api-Access-Key": access_token,
|
||||
"X-Api-Resource-Id": "volc.bigasr.sauc.duration",
|
||||
"X-Api-Connect-Id": str(uuid.uuid4())
|
||||
}
|
||||
headers = self.provider.token_auth()
|
||||
|
||||
async with websockets.connect(
|
||||
ws_url,
|
||||
self.provider.ws_url,
|
||||
additional_headers=headers,
|
||||
max_size=1000000000,
|
||||
ping_interval=None,
|
||||
ping_timeout=None,
|
||||
close_timeout=10
|
||||
) as ws:
|
||||
request_params = {
|
||||
"app": {"appid": appid, "cluster": cluster, "token": access_token},
|
||||
"user": {"uid": uid},
|
||||
"request": {
|
||||
"reqid": str(uuid.uuid4()),
|
||||
"workflow": "audio_in,resample,partition,vad,fe,decode,itn,nlu_punctuate",
|
||||
"show_utterances": True,
|
||||
"result_type": "single",
|
||||
"sequence": 1
|
||||
},
|
||||
"audio": {
|
||||
"format": "pcm",
|
||||
"codec": "pcm",
|
||||
"rate": 16000,
|
||||
"language": "zh-CN",
|
||||
"bits": 16,
|
||||
"channel": 1,
|
||||
"sample_rate": 16000
|
||||
}
|
||||
}
|
||||
request_params = self.provider.construct_request(str(uuid.uuid4()))
|
||||
|
||||
payload_bytes = str.encode(json.dumps(request_params))
|
||||
payload_bytes = gzip.compress(payload_bytes)
|
||||
full_client_request = self._generate_header()
|
||||
full_client_request = bytearray(self.provider.generate_header())
|
||||
full_client_request.extend((len(payload_bytes)).to_bytes(4, "big"))
|
||||
full_client_request.extend(payload_bytes)
|
||||
await ws.send(full_client_request)
|
||||
|
||||
init_res = await ws.recv()
|
||||
result = self._parse_response(init_res)
|
||||
result = self.provider.parse_response(init_res)
|
||||
if "code" in result and result["code"] != 1000:
|
||||
raise Exception(f"初始化失败: {result.get('payload_msg', {}).get('error', '未知错误')}")
|
||||
|
||||
@@ -200,7 +109,7 @@ class DoubaoStreamASRTester(BaseASRTester):
|
||||
|
||||
# 发送音频数据(使用最后一帧标记,告诉服务端音频已结束)
|
||||
payload = gzip.compress(audio_data)
|
||||
audio_request = bytearray(self._generate_last_audio_header()) # 修复:使用最后一帧Header
|
||||
audio_request = bytearray(self.provider.generate_last_audio_default_header())
|
||||
audio_request.extend(len(payload).to_bytes(4, "big"))
|
||||
audio_request.extend(payload)
|
||||
await ws.send(audio_request)
|
||||
|
||||
@@ -31,12 +31,12 @@ handle_exit_intent_function_desc = {
|
||||
"handle_exit_intent", handle_exit_intent_function_desc, ToolType.SYSTEM_CTL
|
||||
)
|
||||
def handle_exit_intent(conn: "ConnectionHandler", say_goodbye: str | None = None):
|
||||
conn.is_exiting = True
|
||||
# 处理退出意图
|
||||
try:
|
||||
if say_goodbye is None:
|
||||
say_goodbye = "再见,祝您生活愉快!"
|
||||
conn.close_after_chat = True
|
||||
if not conn.close_after_chat:
|
||||
conn.close_after_chat = True
|
||||
logger.bind(tag=TAG).info(f"退出意图已处理:{say_goodbye}")
|
||||
return ActionResponse(
|
||||
action=Action.RESPONSE, result="退出意图已处理", response=say_goodbye
|
||||
|
||||
Reference in New Issue
Block a user