合并main分支

This commit is contained in:
hrz
2025-05-21 14:13:52 +08:00
20 changed files with 246 additions and 98 deletions
+10 -12
View File
@@ -9,26 +9,24 @@ assignees: ''
## 🐛 问题描述
<!-- 清晰简洁地描述问题是什么 -->
## 🔍 复现步骤
<!-- 详细描述复现问题的步骤 -->
## 🖥️ 环境信息
- 部署方式: 全模块部署 还是 单Server部署
- 版本号: 例如 0.3.x
## 🔍 告诉我们,应该怎么复现这个问题
<!-- 这个很重要,方便我们快速定位 -->
1. 打开 '...'
2. 点击 '...'
3. 滚动到 '...'
4. 看到错误
## 🤔 预期行为
## 🤔 你原本希望是怎么样的
<!-- 简要描述预期的正确行为 -->
## 😯 截图
## 😯 提供一些截图
<!-- 如果适用,添加问题的截图 -->
## 🖥️ 环境信息
- 操作系统: [例如 Windows 10]
- 浏览器: [例如 Chrome 89]
- 项目版本: [例如 1.0.0]
- Python版本: [例如 3.9.13]
- Jdk版本:[例如 java 21.0.5 2024-10-15 LTS]
- Nodejs版本:[例如 v20.14.0]
1. 比如日志截图,越多越好
2. 比如界面反应
## 📋 其他信息
<!-- 在此添加关于此问题的任何其他上下文信息 -->
+2 -2
View File
@@ -58,7 +58,7 @@ jobs:
file: Dockerfile-server
push: true
tags: |
${{ env.IS_VERSION == 'true' && format('ghcr.io/{0}:server_{1}\nghcr.io/{0}:server_latest', github.repository, env.VERSION) || format('ghcr.io/{0}:server_latest', github.repository) }}
${{ env.IS_VERSION == 'true' && format('ghcr.io/{0}:server_{1},ghcr.io/{0}:server_latest', github.repository, env.VERSION) || format('ghcr.io/{0}:server_latest', github.repository) }}
platforms: linux/amd64,linux/arm64
# 构建 manager-api 镜像
@@ -69,5 +69,5 @@ jobs:
file: Dockerfile-web
push: true
tags: |
${{ env.IS_VERSION == 'true' && format('ghcr.io/{0}:web_{1}\nghcr.io/{0}:web_latest', github.repository, env.VERSION) || format('ghcr.io/{0}:web_latest', github.repository) }}
${{ env.IS_VERSION == 'true' && format('ghcr.io/{0}:web_{1},ghcr.io/{0}:web_latest', github.repository, env.VERSION) || format('ghcr.io/{0}:web_latest', github.repository) }}
platforms: linux/amd64,linux/arm64
+5
View File
@@ -162,9 +162,14 @@ main/xiaozhi-server/models/sherpa-onnx*
/main/xiaozhi-server/asr-models/iic/SenseVoiceSmall/
/models/SenseVoiceSmall/model.pt
my_wakeup_words.mp3
!main/xiaozhi-server/config/assets/bind_code.wav
!main/xiaozhi-server/config/assets/bind_not_found.wav
!main/xiaozhi-server/config/assets/bind_code/*.wav
!main/xiaozhi-server/config/assets/max_output_size.wav
main/manager-api/.vscode
# Ignore webpack cache directory
main/manager-web/.webpack_cache/
main/xiaozhi-server/mysql
uploadfile
*.json
.vscode
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,18 +1,22 @@
from config.logger import setup_logging
import time
import copy
from core.utils.util import remove_punctuation_and_length
from core.handle.sendAudioHandle import send_stt_message
from core.handle.intentHandler import handle_user_intent
from core.utils.output_counter import check_device_output_limit
from core.handle.reportHandle import enqueue_asr_report
from core.utils.util import audio_to_data
TAG = __name__
logger = setup_logging()
async def handleAudioMessage(conn, audio):
if not conn.asr_server_receive:
logger.bind(tag=TAG).debug(f"前期数据处理中,暂停接收")
if conn.vad is None:
return
if conn.client_listen_mode == "auto":
if not conn.asr_server_receive:
conn.logger.bind(tag=TAG).debug(f"前期数据处理中,暂停接收")
return
if conn.client_listen_mode == "auto" or conn.client_listen_mode == "realtime":
have_voice = conn.vad.is_vad(conn, audio)
else:
have_voice = conn.client_have_voice
@@ -35,12 +39,13 @@ async def handleAudioMessage(conn, audio):
if len(conn.asr_audio) < 15:
conn.asr_server_receive = True
else:
text, file_path = await conn.asr.speech_to_text(
conn.asr_audio, conn.session_id
)
logger.bind(tag=TAG).info(f"识别文本: {text}")
text, _ = await conn.asr.speech_to_text(conn.asr_audio, conn.session_id)
conn.logger.bind(tag=TAG).info(f"识别文本: {text}")
text_len, _ = remove_punctuation_and_length(text)
if text_len > 0:
# 使用自定义模块进行上报
enqueue_asr_report(conn, text, copy.deepcopy(conn.asr_audio))
await startToChat(conn, text)
else:
conn.asr_server_receive = True
@@ -49,6 +54,18 @@ async def handleAudioMessage(conn, audio):
async def startToChat(conn, text):
if conn.need_bind:
await check_bind_device(conn)
return
# 如果当日的输出字数大于限定的字数
if conn.max_output_size > 0:
if check_device_output_limit(
conn.headers.get("device-id"), conn.max_output_size
):
await max_out_size(conn)
return
# 首先进行意图分析
intent_handled = await handle_user_intent(conn, text)
@@ -59,7 +76,7 @@ async def startToChat(conn, text):
# 意图未被处理,继续常规聊天流程
await send_stt_message(conn, text)
if conn.use_function_call_mode:
if conn.intent_type == "function_call":
# 使用支持function calling的聊天方法
conn.executor.submit(conn.chat_with_function_calling, text)
else:
@@ -71,8 +88,8 @@ async def no_voice_close_connect(conn):
conn.client_no_voice_last_time = time.time() * 1000
else:
no_voice_time = time.time() * 1000 - conn.client_no_voice_last_time
close_connection_no_voice_time = conn.config.get(
"close_connection_no_voice_time", 120
close_connection_no_voice_time = int(
conn.config.get("close_connection_no_voice_time", 120)
)
if (
not conn.close_after_chat
@@ -81,7 +98,65 @@ async def no_voice_close_connect(conn):
conn.close_after_chat = True
conn.client_abort = False
conn.asr_server_receive = False
prompt = (
"请你以“时间过得真快”未来头,用富有感情、依依不舍的话来结束这场对话吧。"
)
end_prompt = conn.config.get("end_prompt", {})
if end_prompt and end_prompt.get("enable", True) is False:
conn.logger.bind(tag=TAG).info("结束对话,无需发送结束提示语")
await conn.close()
return
prompt = end_prompt.get("prompt")
if not prompt:
prompt = "请你以“时间过得真快”未来头,用富有感情、依依不舍的话来结束这场对话吧。!"
await startToChat(conn, prompt)
async def max_out_size(conn):
text = "不好意思,我现在有点事情要忙,明天这个时候我们再聊,约好了哦!明天不见不散,拜拜!"
await send_stt_message(conn, text)
conn.tts_first_text_index = 0
conn.tts_last_text_index = 0
conn.llm_finish_task = True
file_path = "config/assets/max_output_size.wav"
opus_packets, _ = audio_to_data(file_path)
conn.audio_play_queue.put((opus_packets, text, 0))
conn.close_after_chat = True
async def check_bind_device(conn):
if conn.bind_code:
# 确保bind_code是6位数字
if len(conn.bind_code) != 6:
conn.logger.bind(tag=TAG).error(f"无效的绑定码格式: {conn.bind_code}")
text = "绑定码格式错误,请检查配置。"
await send_stt_message(conn, text)
return
text = f"请登录控制面板,输入{conn.bind_code},绑定设备。"
await send_stt_message(conn, text)
conn.tts_first_text_index = 0
conn.tts_last_text_index = 6
conn.llm_finish_task = True
# 播放提示音
music_path = "config/assets/bind_code.wav"
opus_packets, _ = audio_to_data(music_path)
conn.audio_play_queue.put((opus_packets, text, 0))
# 逐个播放数字
for i in range(6): # 确保只播放6位数字
try:
digit = conn.bind_code[i]
num_path = f"config/assets/bind_code/{digit}.wav"
num_packets, _ = audio_to_data(num_path)
conn.audio_play_queue.put((num_packets, None, i + 1))
except Exception as e:
conn.logger.bind(tag=TAG).error(f"播放数字音频失败: {e}")
continue
else:
text = f"没有找到该设备的版本信息,请正确配置 OTA地址,然后重新编译固件。"
await send_stt_message(conn, text)
conn.tts_first_text_index = 0
conn.tts_last_text_index = 0
conn.llm_finish_task = True
music_path = "config/assets/bind_not_found.wav"
opus_packets, _ = audio_to_data(music_path)
conn.audio_play_queue.put((opus_packets, text, 0))
@@ -10,40 +10,62 @@ from pydub import AudioSegment
from core.providers.tts.dto.dto import TTSMessageDTO, MsgType, SentenceType
from core.utils.util import check_model_key
from core.providers.tts.base import TTSProviderBase
from config.logger import setup_logging
TAG = __name__
logger = setup_logging()
class TTSProvider(TTSProviderBase):
def __init__(self, config, delete_audio_file):
super().__init__(config, delete_audio_file)
self.appid = config.get("appid")
if config.get("appid"):
self.appid = int(config.get("appid"))
else:
self.appid = ""
self.access_token = config.get("access_token")
self.cluster = config.get("cluster")
self.voice = config.get("voice")
if config.get("private_voice"):
self.voice = config.get("private_voice")
else:
self.voice = config.get("voice")
# 处理空字符串的情况
speed_ratio = config.get("speed_ratio", "1.0")
volume_ratio = config.get("volume_ratio", "1.0")
pitch_ratio = config.get("pitch_ratio", "1.0")
self.speed_ratio = float(speed_ratio) if speed_ratio else 1.0
self.volume_ratio = float(volume_ratio) if volume_ratio else 1.0
self.pitch_ratio = float(pitch_ratio) if pitch_ratio else 1.0
self.api_url = config.get("api_url")
self.authorization = config.get("authorization")
self.header = {"Authorization": f"{self.authorization}{self.access_token}"}
check_model_key("TTS", self.access_token)
def generate_filename(self, extension=".wav"):
return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
return os.path.join(
self.output_file,
f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
)
async def text_to_speak(self, u_id, text, is_last_text=False, is_first_text=False):
tmp_file = self.generate_filename()
request_json = {
"app": {
"appid": f"{self.appid}",
"token": "access_token",
"cluster": self.cluster
},
"user": {
"uid": "1"
"token": self.access_token,
"cluster": self.cluster,
},
"user": {"uid": "1"},
"audio": {
"voice_type": self.voice,
"encoding": "wav",
"speed_ratio": 1.0,
"volume_ratio": 1.0,
"pitch_ratio": 1.0,
"speed_ratio": self.speed_ratio,
"volume_ratio": self.volume_ratio,
"pitch_ratio": self.pitch_ratio,
},
"request": {
"reqid": str(uuid.uuid4()),
@@ -51,18 +73,22 @@ class TTSProvider(TTSProviderBase):
"text_type": "plain",
"operation": "query",
"with_frontend": 1,
"frontend_type": "unitTson"
}
"frontend_type": "unitTson",
},
}
try:
resp = requests.post(self.api_url, json.dumps(request_json), headers=self.header)
resp = requests.post(
self.api_url, json.dumps(request_json), headers=self.header
)
if "data" in resp.json():
data = resp.json()["data"]
file_to_save = open(tmp_file, "wb")
file_to_save.write(base64.b64decode(data))
else:
raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}")
raise Exception(
f"{__name__} status_code: {resp.status_code} response: {resp.content}"
)
except Exception as e:
raise Exception(f"{__name__} error: {e}")
# 使用 pydub 读取临时文件
@@ -16,7 +16,10 @@ class TTSProvider(TTSProviderBase):
self.appid = config.get("appid")
self.secret_id = config.get("secret_id")
self.secret_key = config.get("secret_key")
self.voice = config.get("voice")
if config.get("private_voice"):
self.voice = config.get("private_voice")
else:
self.voice = int(config.get("voice"))
self.api_url = "https://tts.tencentcloudapi.com" # 正确的API端点
self.region = config.get("region")
self.output_file = config.get("output_dir")
@@ -25,35 +28,36 @@ class TTSProvider(TTSProviderBase):
"""生成鉴权请求头"""
# 获取当前UTC时间戳
timestamp = int(time.time())
# 使用UTC时间计算日期
utc_date = datetime.fromtimestamp(timestamp, tz=timezone.utc).strftime('%Y-%m-%d')
utc_date = datetime.fromtimestamp(timestamp, tz=timezone.utc).strftime(
"%Y-%m-%d"
)
# 服务名称必须是 "tts"
service = "tts"
# 拼接凭证范围
credential_scope = f"{utc_date}/{service}/tc3_request"
# 使用TC3-HMAC-SHA256签名方法
algorithm = "TC3-HMAC-SHA256"
# 构建规范请求字符串
http_request_method = "POST"
canonical_uri = "/"
canonical_querystring = ""
# 请求头必须包含host和content-type,且按字典序排列
canonical_headers = (
f"content-type:application/json\n"
f"host:tts.tencentcloudapi.com\n"
f"content-type:application/json\n" f"host:tts.tencentcloudapi.com\n"
)
signed_headers = "content-type;host"
# 请求体哈希值
payload = json.dumps(request_body)
payload_hash = hashlib.sha256(payload.encode('utf-8')).hexdigest()
payload_hash = hashlib.sha256(payload.encode("utf-8")).hexdigest()
# 构建规范请求字符串
canonical_request = (
f"{http_request_method}\n"
@@ -63,10 +67,12 @@ class TTSProvider(TTSProviderBase):
f"{signed_headers}\n"
f"{payload_hash}"
)
# 计算规范请求的哈希值
hashed_canonical_request = hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()
hashed_canonical_request = hashlib.sha256(
canonical_request.encode("utf-8")
).hexdigest()
# 构建待签名字符串
string_to_sign = (
f"{algorithm}\n"
@@ -74,19 +80,19 @@ class TTSProvider(TTSProviderBase):
f"{credential_scope}\n"
f"{hashed_canonical_request}"
)
# 计算签名密钥
secret_date = self._hmac_sha256(f"TC3{self.secret_key}".encode('utf-8'), utc_date)
secret_date = self._hmac_sha256(
f"TC3{self.secret_key}".encode("utf-8"), utc_date
)
secret_service = self._hmac_sha256(secret_date, service)
secret_signing = self._hmac_sha256(secret_service, "tc3_request")
# 计算签名
signature = hmac.new(
secret_signing,
string_to_sign.encode('utf-8'),
hashlib.sha256
secret_signing, string_to_sign.encode("utf-8"), hashlib.sha256
).hexdigest()
# 构建授权头
authorization = (
f"{algorithm} "
@@ -94,7 +100,7 @@ class TTSProvider(TTSProviderBase):
f"SignedHeaders={signed_headers}, "
f"Signature={signature}"
)
# 构建请求头
headers = {
"Content-Type": "application/json",
@@ -104,19 +110,22 @@ class TTSProvider(TTSProviderBase):
"X-TC-Timestamp": str(timestamp),
"X-TC-Version": "2019-08-23",
"X-TC-Region": self.region,
"X-TC-Language": "zh-CN"
"X-TC-Language": "zh-CN",
}
return headers
def _hmac_sha256(self, key, msg):
"""HMAC-SHA256加密"""
if isinstance(msg, str):
msg = msg.encode('utf-8')
msg = msg.encode("utf-8")
return hmac.new(key, msg, hashlib.sha256).digest()
def generate_filename(self, extension=".wav"):
return os.path.join(self.output_file, f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}")
return os.path.join(
self.output_file,
f"tts-{datetime.now().date()}@{uuid.uuid4().hex}{extension}",
)
async def text_to_speak(self, text, output_file):
# 构建请求体
@@ -129,19 +138,23 @@ class TTSProvider(TTSProviderBase):
try:
# 获取请求头(每次请求都重新生成,以确保时间戳和签名是最新的)
headers = self._get_auth_headers(request_json)
# 发送请求
resp = requests.post(self.api_url, json.dumps(request_json), headers=headers)
resp = requests.post(
self.api_url, json.dumps(request_json), headers=headers
)
# 检查响应
if resp.status_code == 200:
response_data = resp.json()
# 检查是否成功
if response_data.get("Response", {}).get("Error") is not None:
error_info = response_data["Response"]["Error"]
raise Exception(f"API返回错误: {error_info['Code']}: {error_info['Message']}")
raise Exception(
f"API返回错误: {error_info['Code']}: {error_info['Message']}"
)
# 提取音频数据
audio_data = response_data["Response"].get("Audio")
if audio_data:
@@ -151,6 +164,8 @@ class TTSProvider(TTSProviderBase):
else:
raise Exception(f"{__name__}: 没有返回音频数据: {response_data}")
else:
raise Exception(f"{__name__} status_code: {resp.status_code} response: {resp.content}")
raise Exception(
f"{__name__} status_code: {resp.status_code} response: {resp.content}"
)
except Exception as e:
raise Exception(f"{__name__} error: {e}")
raise Exception(f"{__name__} error: {e}")
@@ -12,9 +12,9 @@ from core.providers.tts.dto.dto import TTSMessageDTO, MsgType
from core.utils import p3
from core.handle.sendAudioHandle import send_stt_message
from plugins_func.register import register_function, ToolType, ActionResponse, Action
from core.utils.dialogue import Message
TAG = __name__
logger = setup_logging()
MUSIC_CACHE = {}
@@ -22,13 +22,13 @@ play_music_function_desc = {
"type": "function",
"function": {
"name": "play_music",
"description": "唱歌、听歌、播放音乐方法。比如用户说播放音乐,参数为:random,比如用户说播放两只老虎,参数为:两只老虎",
"description": "唱歌、听歌、播放音乐方法。",
"parameters": {
"type": "object",
"properties": {
"song_name": {
"type": "string",
"description": "歌曲名称,如果没有指定具体歌名则为'random'",
"description": "歌曲名称,如果用户没有指定具体歌名则为'random', 明确指定的时返回音乐的名字 示例: ```用户:播放两只老虎\n参数:两只老虎``` ```用户:播放音乐 \n参数:random ```",
}
},
"required": ["song_name"],
@@ -46,7 +46,7 @@ def play_music(conn, song_name: str):
# 检查事件循环状态
if not conn.loop.is_running():
logger.bind(tag=TAG).error("事件循环未运行,无法提交任务")
conn.logger.bind(tag=TAG).error("事件循环未运行,无法提交任务")
return ActionResponse(
action=Action.RESPONSE, result="系统繁忙", response="请稍后再试"
)
@@ -60,9 +60,9 @@ def play_music(conn, song_name: str):
def handle_done(f):
try:
f.result() # 可在此处理成功逻辑
logger.bind(tag=TAG).info("播放完成")
conn.logger.bind(tag=TAG).info("播放完成")
except Exception as e:
logger.bind(tag=TAG).error(f"播放失败: {e}")
conn.logger.bind(tag=TAG).error(f"播放失败: {e}")
future.add_done_callback(handle_done)
@@ -70,7 +70,7 @@ def play_music(conn, song_name: str):
action=Action.NONE, result="指令已接收", response="正在为您播放音乐"
)
except Exception as e:
logger.bind(tag=TAG).error(f"处理音乐意图错误: {e}")
conn.logger.bind(tag=TAG).error(f"处理音乐意图错误: {e}")
return ActionResponse(
action=Action.RESPONSE, result=str(e), response="播放音乐时出错了"
)
@@ -151,7 +151,7 @@ async def handle_music_command(conn, text):
"""处理音乐播放指令"""
clean_text = re.sub(r"[^\w\s]", "", text).strip()
logger.bind(tag=TAG).debug(f"检查是否是音乐命令: {clean_text}")
conn.logger.bind(tag=TAG).debug(f"检查是否是音乐命令: {clean_text}")
# 尝试匹配具体歌名
if os.path.exists(MUSIC_CACHE["music_dir"]):
@@ -166,7 +166,7 @@ async def handle_music_command(conn, text):
if potential_song:
best_match = _find_best_match(potential_song, MUSIC_CACHE["music_files"])
if best_match:
logger.bind(tag=TAG).info(f"找到最匹配的歌曲: {best_match}")
conn.logger.bind(tag=TAG).info(f"找到最匹配的歌曲: {best_match}")
await play_local_music(conn, specific_file=best_match)
return True
# 检查是否是通用播放音乐命令
@@ -174,12 +174,31 @@ async def handle_music_command(conn, text):
return True
def _get_random_play_prompt(song_name):
"""生成随机播放引导语"""
# 移除文件扩展名
clean_name = os.path.splitext(song_name)[0]
prompts = [
f"正在为您播放,{clean_name}",
f"请欣赏歌曲,{clean_name}",
f"即将为您播放,{clean_name}",
f"为您带来,{clean_name}",
f"让我们聆听,{clean_name}",
f"接下来请欣赏,{clean_name}",
f"为您献上,{clean_name}",
]
# 直接使用random.choice,不设置seed
return random.choice(prompts)
async def play_local_music(conn, specific_file=None):
global MUSIC_CACHE
"""播放本地音乐文件"""
try:
if not os.path.exists(MUSIC_CACHE["music_dir"]):
logger.bind(tag=TAG).error(f"音乐目录不存在: " + MUSIC_CACHE["music_dir"])
conn.logger.bind(tag=TAG).error(
f"音乐目录不存在: " + MUSIC_CACHE["music_dir"]
)
return
# 确保路径正确性
@@ -188,23 +207,33 @@ async def play_local_music(conn, specific_file=None):
music_path = os.path.join(MUSIC_CACHE["music_dir"], specific_file)
else:
if not MUSIC_CACHE["music_files"]:
logger.bind(tag=TAG).error("未找到MP3音乐文件")
conn.logger.bind(tag=TAG).error("未找到MP3音乐文件")
return
selected_music = random.choice(MUSIC_CACHE["music_files"])
music_path = os.path.join(MUSIC_CACHE["music_dir"], selected_music)
if not os.path.exists(music_path):
logger.bind(tag=TAG).error(f"选定的音乐文件不存在: {music_path}")
conn.logger.bind(tag=TAG).error(f"选定的音乐文件不存在: {music_path}")
return
text = f"正在播放{selected_music}"
text = _get_random_play_prompt(selected_music)
await send_stt_message(conn, text)
conn.dialogue.put(Message(role="assistant", content=text))
conn.tts_first_text_index = 0
conn.tts_last_text_index = 0
tts_file = await asyncio.to_thread(conn.tts.to_tts, text)
if tts_file is not None and os.path.exists(tts_file):
conn.tts_last_text_index = 1
opus_packets, _ = conn.tts.audio_to_opus_data(tts_file)
conn.audio_play_queue.put((opus_packets, None, 0))
os.remove(tts_file)
conn.llm_finish_task = True
if music_path.endswith(".p3"):
opus_packets, duration = p3.decode_opus_from_file(music_path)
opus_packets, _ = p3.decode_opus_from_file(music_path)
else:
opus_packets, duration = conn.tts.audio_to_opus_data(music_path)
opus_packets, _ = conn.tts.audio_to_opus_data(music_path)
conn.tts.tts_audio_queue.put(
TTSMessageDTO(
u_id=conn.u_id,
@@ -226,5 +255,5 @@ async def play_local_music(conn, specific_file=None):
)
except Exception as e:
logger.bind(tag=TAG).error(f"播放音乐失败: {str(e)}")
logger.bind(tag=TAG).error(f"详细错误: {traceback.format_exc()}")
conn.logger.bind(tag=TAG).error(f"播放音乐失败: {str(e)}")
conn.logger.bind(tag=TAG).error(f"详细错误: {traceback.format_exc()}")