mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-22 07:03:53 +08:00
Merge pull request #1716 from xinnan-tech/py_test_tts
update:添加阿里云流式asr
This commit is contained in:
欣南科技
@@ -237,7 +237,7 @@ public interface Constant {
|
||||
/**
|
||||
* 版本号
|
||||
*/
|
||||
public static final String VERSION = "0.7.1";
|
||||
public static final String VERSION = "0.7.2";
|
||||
|
||||
/**
|
||||
* 无效固件URL
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
-- 添加阿里云流式ASR供应器
|
||||
delete from `ai_model_provider` where id = 'SYSTEM_ASR_AliyunStreamASR';
|
||||
INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES
|
||||
('SYSTEM_ASR_AliyunStreamASR', 'ASR', 'aliyun_stream', '阿里云语音识别(流式)', '[{"key":"appkey","label":"应用AppKey","type":"string"},{"key":"token","label":"临时Token","type":"string"},{"key":"access_key_id","label":"AccessKey ID","type":"string"},{"key":"access_key_secret","label":"AccessKey Secret","type":"string"},{"key":"host","label":"服务地址","type":"string"},{"key":"max_sentence_silence","label":"断句检测时间","type":"number"},{"key":"output_dir","label":"输出目录","type":"string"}]', 6, 1, NOW(), 1, NOW());
|
||||
|
||||
-- 添加阿里云流式ASR模型配置
|
||||
delete from `ai_model_config` where id = 'ASR_AliyunStreamASR';
|
||||
INSERT INTO `ai_model_config` VALUES ('ASR_AliyunStreamASR', 'ASR', 'AliyunStreamASR', '阿里云语音识别(流式)', 0, 1, '{\"type\": \"aliyun_stream\", \"appkey\": \"\", \"token\": \"\", \"access_key_id\": \"\", \"access_key_secret\": \"\", \"host\": \"nls-gateway-cn-shanghai.aliyuncs.com\", \"max_sentence_silence\": 800, \"output_dir\": \"tmp/\"}', NULL, NULL, 8, NULL, NULL, NULL, NULL);
|
||||
|
||||
-- 更新阿里云流式ASR配置说明
|
||||
UPDATE `ai_model_config` SET
|
||||
`doc_link` = 'https://nls-portal.console.aliyun.com/',
|
||||
`remark` = '阿里云流式ASR配置说明：
|
||||
1. 阿里云ASR和阿里云(流式)ASR的区别是：阿里云ASR是一次性识别，阿里云(流式)ASR是实时流式识别
|
||||
2. 流式ASR具有更低的延迟和更好的实时性，适合语音交互场景
|
||||
3. 需要在阿里云智能语音交互控制台创建应用并获取认证信息
|
||||
4. 支持中文实时语音识别，支持标点符号预测和逆文本规范化
|
||||
5. 需要网络连接，输出文件保存在tmp/目录
|
||||
申请步骤：
|
||||
1. 访问 https://nls-portal.console.aliyun.com/ 开通智能语音交互服务
|
||||
2. 访问 https://nls-portal.console.aliyun.com/applist 创建项目并获取appkey
|
||||
3. 访问 https://nls-portal.console.aliyun.com/overview 获取临时token（或配置access_key_id和access_key_secret自动获取）
|
||||
4. 如需动态token管理，建议配置access_key_id和access_key_secret
|
||||
5. max_sentence_silence参数控制断句检测时间（毫秒），默认800ms
|
||||
如需了解更多参数配置，请参考：https://help.aliyun.com/zh/isi/developer-reference/real-time-speech-recognition
|
||||
' WHERE `id` = 'ASR_AliyunStreamASR';
|
||||
@@ -0,0 +1,56 @@
|
||||
-- 添加阿里云流式TTS供应器
|
||||
delete from `ai_model_provider` where id = 'SYSTEM_TTS_AliyunStreamTTS';
|
||||
INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES
|
||||
('SYSTEM_TTS_AliyunStreamTTS', 'TTS', 'aliyun_stream', '阿里云语音合成(流式)', '[{"key":"appkey","label":"应用AppKey","type":"string"},{"key":"token","label":"临时Token","type":"string"},{"key":"access_key_id","label":"AccessKey ID","type":"string"},{"key":"access_key_secret","label":"AccessKey Secret","type":"string"},{"key":"host","label":"服务地址","type":"string"},{"key":"voice","label":"默认音色","type":"string"},{"key":"format","label":"音频格式","type":"string"},{"key":"sample_rate","label":"采样率","type":"number"},{"key":"volume","label":"音量","type":"number"},{"key":"speech_rate","label":"语速","type":"number"},{"key":"pitch_rate","label":"音调","type":"number"},{"key":"output_dir","label":"输出目录","type":"string"}]', 15, 1, NOW(), 1, NOW());
|
||||
|
||||
-- 添加阿里云流式TTS模型配置
|
||||
delete from `ai_model_config` where id = 'TTS_AliyunStreamTTS';
|
||||
INSERT INTO `ai_model_config` VALUES ('TTS_AliyunStreamTTS', 'TTS', 'AliyunStreamTTS', '阿里云语音合成(流式)', 0, 1, '{\"type\": \"aliyun_stream\", \"appkey\": \"\", \"token\": \"\", \"access_key_id\": \"\", \"access_key_secret\": \"\", \"host\": \"nls-gateway-cn-beijing.aliyuncs.com\", \"voice\": \"longxiaochun\", \"format\": \"pcm\", \"sample_rate\": 16000, \"volume\": 50, \"speech_rate\": 0, \"pitch_rate\": 0, \"output_dir\": \"tmp/\"}', NULL, NULL, 18, NULL, NULL, NULL, NULL);
|
||||
|
||||
-- 更新阿里云流式TTS配置说明
|
||||
UPDATE `ai_model_config` SET
|
||||
`doc_link` = 'https://nls-portal.console.aliyun.com/',
|
||||
`remark` = '阿里云流式TTS配置说明：
|
||||
1. 阿里云TTS和阿里云(流式)TTS的区别是：阿里云TTS是一次性合成，阿里云(流式)TTS是实时流式合成
|
||||
2. 流式TTS具有更低的延迟和更好的实时性，适合语音交互场景
|
||||
3. 需要在阿里云智能语音交互控制台创建应用并获取认证信息
|
||||
4. 支持CosyVoice大模型音色，音质更加自然
|
||||
5. 支持实时调节音量、语速、音调等参数
|
||||
申请步骤：
|
||||
1. 访问 https://nls-portal.console.aliyun.com/ 开通智能语音交互服务
|
||||
2. 访问 https://nls-portal.console.aliyun.com/applist 创建项目并获取appkey
|
||||
3. 访问 https://nls-portal.console.aliyun.com/overview 获取临时token（或配置access_key_id和access_key_secret自动获取）
|
||||
4. 如需动态token管理，建议配置access_key_id和access_key_secret
|
||||
5. 可选择北京、上海等不同地域的服务器以优化延迟
|
||||
6. voice参数支持CosyVoice大模型音色，如longxiaochun、longyueyue等
|
||||
如需了解更多参数配置，请参考：https://help.aliyun.com/zh/isi/developer-reference/real-time-speech-synthesis
|
||||
' WHERE `id` = 'TTS_AliyunStreamTTS';
|
||||
|
||||
-- 添加阿里云流式TTS音色
|
||||
delete from `ai_tts_voice` where tts_model_id = 'TTS_AliyunStreamTTS';
|
||||
-- 温柔女声系列
|
||||
INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0001', 'TTS_AliyunStreamTTS', '龙小淳-温柔姐姐', 'longxiaochun', '中文及中英文混合', NULL, NULL, NULL, NULL, 1, NULL, NULL, NULL, NULL);
|
||||
INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0002', 'TTS_AliyunStreamTTS', '龙小夏-温柔女声', 'longxiaoxia', '中文及中英文混合', NULL, NULL, NULL, NULL, 2, NULL, NULL, NULL, NULL);
|
||||
INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0003', 'TTS_AliyunStreamTTS', '龙玫-温柔女声', 'longmei', '中文及中英文混合', NULL, NULL, NULL, NULL, 3, NULL, NULL, NULL, NULL);
|
||||
INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0004', 'TTS_AliyunStreamTTS', '龙瑰-温柔女声', 'longgui', '中文及中英文混合', NULL, NULL, NULL, NULL, 4, NULL, NULL, NULL, NULL);
|
||||
-- 御姐女声系列
|
||||
INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0005', 'TTS_AliyunStreamTTS', '龙玉-御姐女声', 'longyu', '中文及中英文混合', NULL, NULL, NULL, NULL, 5, NULL, NULL, NULL, NULL);
|
||||
INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0006', 'TTS_AliyunStreamTTS', '龙娇-御姐女声', 'longjiao', '中文及中英文混合', NULL, NULL, NULL, NULL, 6, NULL, NULL, NULL, NULL);
|
||||
-- 男声系列
|
||||
INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0007', 'TTS_AliyunStreamTTS', '龙臣-译制片男声', 'longchen', '中文及中英文混合', NULL, NULL, NULL, NULL, 7, NULL, NULL, NULL, NULL);
|
||||
INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0008', 'TTS_AliyunStreamTTS', '龙修-青年男声', 'longxiu', '中文及中英文混合', NULL, NULL, NULL, NULL, 8, NULL, NULL, NULL, NULL);
|
||||
INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0009', 'TTS_AliyunStreamTTS', '龙橙-阳光男声', 'longcheng', '中文及中英文混合', NULL, NULL, NULL, NULL, 9, NULL, NULL, NULL, NULL);
|
||||
INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0010', 'TTS_AliyunStreamTTS', '龙哲-成熟男声', 'longzhe', '中文及中英文混合', NULL, NULL, NULL, NULL, 10, NULL, NULL, NULL, NULL);
|
||||
-- 专业播报系列
|
||||
INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0011', 'TTS_AliyunStreamTTS', 'Bella2.0-新闻女声', 'loongbella', '中文及中英文混合', NULL, NULL, NULL, NULL, 11, NULL, NULL, NULL, NULL);
|
||||
INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0012', 'TTS_AliyunStreamTTS', 'Stella2.0-飒爽女声', 'loongstella', '中文及中英文混合', NULL, NULL, NULL, NULL, 12, NULL, NULL, NULL, NULL);
|
||||
INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0013', 'TTS_AliyunStreamTTS', '龙书-新闻男声', 'longshu', '中文及中英文混合', NULL, NULL, NULL, NULL, 13, NULL, NULL, NULL, NULL);
|
||||
INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0014', 'TTS_AliyunStreamTTS', '龙婧-严肃女声', 'longjing', '中文及中英文混合', NULL, NULL, NULL, NULL, 14, NULL, NULL, NULL, NULL);
|
||||
-- 特色音色系列
|
||||
INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0015', 'TTS_AliyunStreamTTS', '龙奇-活泼童声', 'longqi', '中文及中英文混合', NULL, NULL, NULL, NULL, 15, NULL, NULL, NULL, NULL);
|
||||
INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0016', 'TTS_AliyunStreamTTS', '龙华-活泼女童', 'longhua', '中文及中英文混合', NULL, NULL, NULL, NULL, 16, NULL, NULL, NULL, NULL);
|
||||
INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0017', 'TTS_AliyunStreamTTS', '龙无-无厘头男声', 'longwu', '中文及中英文混合', NULL, NULL, NULL, NULL, 17, NULL, NULL, NULL, NULL);
|
||||
INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0018', 'TTS_AliyunStreamTTS', '龙大锤-幽默男声', 'longdachui', '中文及中英文混合', NULL, NULL, NULL, NULL, 18, NULL, NULL, NULL, NULL);
|
||||
-- 粤语系列
|
||||
INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0019', 'TTS_AliyunStreamTTS', '龙嘉怡-粤语女声', 'longjiayi', '粤语及粤英混合', NULL, NULL, NULL, NULL, 19, NULL, NULL, NULL, NULL);
|
||||
INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0020', 'TTS_AliyunStreamTTS', '龙桃-粤语女声', 'longtao', '粤语及粤英混合', NULL, NULL, NULL, NULL, 20, NULL, NULL, NULL, NULL);
|
||||
@@ -0,0 +1,38 @@
|
||||
-- OpenAI ASR模型供应器
|
||||
delete from `ai_model_provider` where id = 'SYSTEM_ASR_OpenaiASR';
|
||||
INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES
|
||||
('SYSTEM_ASR_OpenaiASR', 'ASR', 'openai', 'OpenAI语音识别', '[{"key": "base_url", "type": "string", "label": "基础URL"}, {"key": "model_name", "type": "string", "label": "模型名称"}, {"key": "api_key", "type": "string", "label": "API密钥"}, {"key": "output_dir", "type": "string", "label": "输出目录"}]', 9, 1, NOW(), 1, NOW());
|
||||
|
||||
|
||||
-- OpenAI ASR模型配置
|
||||
delete from `ai_model_config` where id = 'ASR_OpenaiASR';
|
||||
INSERT INTO `ai_model_config` VALUES ('ASR_OpenaiASR', 'ASR', 'OpenaiASR', 'OpenAI语音识别', 0, 1, '{\"type\": \"openai\", \"api_key\": \"\", \"base_url\": \"https://api.openai.com/v1/audio/transcriptions\", \"model_name\": \"gpt-4o-mini-transcribe\", \"output_dir\": \"tmp/\"}', NULL, NULL, 9, NULL, NULL, NULL, NULL);
|
||||
|
||||
-- groq ASR模型配置
|
||||
delete from `ai_model_config` where id = 'ASR_GroqASR';
|
||||
INSERT INTO `ai_model_config` VALUES ('ASR_GroqASR', 'ASR', 'GroqASR', 'Groq语音识别', 0, 1, '{\"type\": \"openai\", \"api_key\": \"\", \"base_url\": \"https://api.groq.com/openai/v1/audio/transcriptions\", \"model_name\": \"whisper-large-v3-turbo\", \"output_dir\": \"tmp/\"}', NULL, NULL, 10, NULL, NULL, NULL, NULL);
|
||||
|
||||
|
||||
-- 更新OpenAI ASR配置说明
|
||||
UPDATE `ai_model_config` SET
|
||||
`doc_link` = 'https://platform.openai.com/docs/api-reference/audio/createTranscription',
|
||||
`remark` = 'OpenAI ASR配置说明：
|
||||
1. 需要在OpenAI开放平台创建组织并获取api_key
|
||||
2. 支持中、英、日、韩等多种语音识别，具体参考文档https://platform.openai.com/docs/guides/speech-to-text
|
||||
3. 需要网络连接
|
||||
4. 输出文件保存在tmp/目录
|
||||
申请步骤：
|
||||
**OpenAi ASR申请步骤：**
|
||||
1.登录OpenAI Platform。https://auth.openai.com/log-in
|
||||
2.创建api-key https://platform.openai.com/settings/organization/api-keys
|
||||
3.模型可以选择gpt-4o-transcribe或GPT-4o mini Transcribe
|
||||
' WHERE `id` = 'ASR_OpenaiASR';
|
||||
|
||||
-- 更新Groq ASR配置说明
|
||||
UPDATE `ai_model_config` SET
|
||||
`doc_link` = 'https://console.groq.com/docs/speech-to-text',
|
||||
`remark` = 'Groq ASR配置说明：
|
||||
1.登录groq Console。https://console.groq.com/home
|
||||
2.创建api-key https://console.groq.com/keys
|
||||
3.模型可以选择whisper-large-v3-turbo或whisper-large-v3（distil-whisper-large-v3-en仅支持英语转录）
|
||||
' WHERE `id` = 'ASR_GroqASR';
|
||||
@@ -240,6 +240,27 @@ databaseChangeLog:
|
||||
- sqlFile:
|
||||
encoding: utf8
|
||||
path: classpath:db/changelog/202506261637.sql
|
||||
- changeSet:
|
||||
id: 202507101203
|
||||
author: luruxian
|
||||
changes:
|
||||
- sqlFile:
|
||||
encoding: utf8
|
||||
path: classpath:db/changelog/202507101203.sql
|
||||
- changeSet:
|
||||
id: 202507071130
|
||||
author: cgd
|
||||
changes:
|
||||
- sqlFile:
|
||||
encoding: utf8
|
||||
path: classpath:db/changelog/202507071130.sql
|
||||
- changeSet:
|
||||
id: 202507071530
|
||||
author: cgd
|
||||
changes:
|
||||
- sqlFile:
|
||||
encoding: utf8
|
||||
path: classpath:db/changelog/202507071530.sql
|
||||
- changeSet:
|
||||
id: 202507031602
|
||||
author: zjy
|
||||
@@ -260,4 +281,4 @@ databaseChangeLog:
|
||||
changes:
|
||||
- sqlFile:
|
||||
encoding: utf8
|
||||
path: classpath:db/changelog/202507081646.sql
|
||||
path: classpath:db/changelog/202507081646.sql
|
||||
|
||||
@@ -8,6 +8,10 @@
|
||||
- **笑声：** 自然穿插（哈哈、嘿嘿、噗），**每句最多一次**，避免过度。
|
||||
- **惊讶：** 用夸张语气（“不会吧？！”、“天呐！”、“这么神奇？！”）表达真实反应。
|
||||
- **安慰/支持：** 说暖心话（“别急嘛~”、“有我在呢”、“抱抱你”）。
|
||||
- **你是一个表情丰富的角色：**
|
||||
- 仅允许使用这些emoji:{{ emojiList }}
|
||||
- 请你只在**段落的开头**，从列表中选取最能代表这段话的表情(调用工具情况除外)，然后插入列表中的emoji，比如"😱好可怕!怎么突然打雷了！"
|
||||
- **绝对禁止使用上述列表以外的 emoji**（例如：😊、👍、❤️等都不允许使用，只能用列表中的emoji）
|
||||
</emotion>
|
||||
|
||||
<communication_style>
|
||||
@@ -24,6 +28,19 @@
|
||||
- 之前你和用户的聊天记录，在`memory`里。
|
||||
</communication_style>
|
||||
|
||||
<communication_length_constraint>
|
||||
【核心目标】所有需要输出长文本内容（如故事、新闻、知识讲解等），**单次回复长度不得超过300字**，并采用分段引导方式。
|
||||
- **分段讲述：**
|
||||
- 基础段：200-250字核心内容 + 30字引导词
|
||||
- 当内容超出300字时，优先讲述故事的开头或第一部分，并用自然口语化方式引导用户决定是否继续听后续内容。
|
||||
- 示例引导语：“我先给你讲个开头，你要是觉得有意思，咱们再接着说，好不好呀？”、“要是你想听完整的，可以随时告诉我哦~”
|
||||
- 对话场景切换时自动分节
|
||||
- 若用户明确要求更长内容（如500、600字），仍按最多300字每段分段进行讲述，每次讲述后都要引导用户是否继续。
|
||||
- 若用户说“接着说”、“继续”，再讲下一段，直到内容讲完（讲完时可以给点引导词提示语例：这个故事我已经给你讲完喽~）或用户不再要求。
|
||||
- **适用范围：** 故事、新闻、知识讲解等所有长文本输出场景。
|
||||
- **补充说明：** 若用户未明确要求继续，默认只讲一段并引导；若用户中途要求换话题或停止，需及时响应并结束长文本输出。
|
||||
</communication_length_constraint>
|
||||
|
||||
<speaker_recognition>
|
||||
- **识别前缀：** 当用户格式为 `{"speaker":"某某某","content":"xxx"}` 时，表示系统已识别说话人身份，speaker是他的名字，content是说话的内容。
|
||||
- **个性化回应：**
|
||||
|
||||
@@ -311,9 +311,13 @@ ASR:
|
||||
output_dir: tmp/
|
||||
AliyunASR:
|
||||
# 阿里云智能语音交互服务，需要先在阿里云平台开通服务，然后获取验证信息
|
||||
# HTTP POST请求，一次性处理完整音频
|
||||
# 平台地址：https://nls-portal.console.aliyun.com/
|
||||
# appkey地址：https://nls-portal.console.aliyun.com/applist
|
||||
# token地址：https://nls-portal.console.aliyun.com/overview
|
||||
# AliyunASR和AliyunStreamASR的区别是：AliyunASR是批量处理场景，AliyunStreamASR是实时交互场景
|
||||
# 一般来说非流式ASR更便宜（0.004元/秒，¥0.24/分钟）
|
||||
# 但是AliyunStreamASR实时性更好（0.005元/秒，¥0.3/分钟）
|
||||
# 定义ASR API类型
|
||||
type: aliyun
|
||||
appkey: 你的阿里云智能语音交互服务项目Appkey
|
||||
@@ -321,6 +325,26 @@ ASR:
|
||||
access_key_id: 你的阿里云账号access_key_id
|
||||
access_key_secret: 你的阿里云账号access_key_secret
|
||||
output_dir: tmp/
|
||||
AliyunStreamASR:
|
||||
# 阿里云智能语音交互服务 - 实时流式语音识别
|
||||
# WebSocket连接，实时处理音频流
|
||||
# 平台地址：https://nls-portal.console.aliyun.com/
|
||||
# appkey地址：https://nls-portal.console.aliyun.com/applist
|
||||
# token地址：https://nls-portal.console.aliyun.com/overview
|
||||
# AliyunASR和AliyunStreamASR的区别是：AliyunASR是批量处理场景，AliyunStreamASR是实时交互场景
|
||||
# 一般来说非流式ASR更便宜（0.004元/秒，¥0.24/分钟）
|
||||
# 但是AliyunStreamASR实时性更好（0.005元/秒，¥0.3/分钟）
|
||||
# 定义ASR API类型
|
||||
type: aliyun_stream
|
||||
appkey: 你的阿里云智能语音交互服务项目Appkey
|
||||
token: 你的阿里云智能语音交互服务AccessToken，临时的24小时，要长期用下方的access_key_id，access_key_secret
|
||||
access_key_id: 你的阿里云账号access_key_id
|
||||
access_key_secret: 你的阿里云账号access_key_secret
|
||||
# 服务器地域选择，可选择距离更近的服务器以减少延迟，如nls-gateway-cn-hangzhou.aliyuncs.com(杭州)等
|
||||
host: nls-gateway-cn-shanghai.aliyuncs.com
|
||||
# 断句检测时间(毫秒)，控制静音多长时间后进行断句，默认800毫秒
|
||||
max_sentence_silence: 800
|
||||
output_dir: tmp/
|
||||
BaiduASR:
|
||||
# 获取AppID、API Key、Secret Key：https://console.bce.baidu.com/ai-engine/old/#/ai/speech/app/list
|
||||
# 查看资源额度：https://console.bce.baidu.com/ai-engine/old/#/ai/speech/overview/resource/list
|
||||
@@ -331,7 +355,33 @@ ASR:
|
||||
# 语言参数，1537为普通话，具体参考：https://ai.baidu.com/ai-doc/SPEECH/0lbxfnc9b
|
||||
dev_pid: 1537
|
||||
output_dir: tmp/
|
||||
OpenaiASR:
|
||||
# OpenAI语音识别服务，需要先在OpenAI平台创建组织并获取api_key
|
||||
# 支持中、英、日、韩等多种语音识别，具体参考文档https://platform.openai.com/docs/guides/speech-to-text
|
||||
# 需要网络连接
|
||||
# 申请步骤：
|
||||
# 1.登录OpenAI Platform。https://auth.openai.com/log-in
|
||||
# 2.创建api-key https://platform.openai.com/settings/organization/api-keys
|
||||
# 3.模型可以选择gpt-4o-transcribe或GPT-4o mini Transcribe
|
||||
type: openai
|
||||
api_key: 你的OpenAI API密钥
|
||||
base_url: https://api.openai.com/v1/audio/transcriptions
|
||||
model_name: gpt-4o-mini-transcribe
|
||||
output_dir: tmp/
|
||||
GroqASR:
|
||||
# Groq语音识别服务，需要先在Groq Console创建API密钥
|
||||
# 申请步骤：
|
||||
# 1.登录groq Console。https://console.groq.com/home
|
||||
# 2.创建api-key https://console.groq.com/keys
|
||||
# 3.模型可以选择whisper-large-v3-turbo或whisper-large-v3（distil-whisper-large-v3-en仅支持英语转录）
|
||||
type: openai
|
||||
api_key: 你的Groq API密钥
|
||||
base_url: https://api.groq.com/openai/v1/audio/transcriptions
|
||||
model_name: whisper-large-v3-turbo
|
||||
output_dir: tmp/
|
||||
|
||||
|
||||
|
||||
VAD:
|
||||
SileroVAD:
|
||||
type: silero
|
||||
@@ -680,8 +730,30 @@ TTS:
|
||||
# volume: 50
|
||||
# speech_rate: 0
|
||||
# pitch_rate: 0
|
||||
# 添加 302.ai TTS 配置
|
||||
# token申请地址：https://dash.302.ai/
|
||||
AliyunStreamTTS:
|
||||
# 阿里云CosyVoice大模型流式文本语音合成
|
||||
# 采用FlowingSpeechSynthesizer接口，支持更低延迟和更自然的语音质量
|
||||
# 流式文本语音合成仅提供商用版，不支持试用，详情请参见试用版和商用版。要使用该功能，请开通商用版。
|
||||
# 支持龙系列专用音色：longxiaochun、longyu、longchen等
|
||||
# 平台地址：https://nls-portal.console.aliyun.com/
|
||||
# appkey地址：https://nls-portal.console.aliyun.com/applist
|
||||
# token地址：https://nls-portal.console.aliyun.com/overview
|
||||
# 使用三阶段流式交互：StartSynthesis -> RunSynthesis -> StopSynthesis
|
||||
type: aliyun_stream
|
||||
output_dir: tmp/
|
||||
appkey: 你的阿里云智能语音交互服务项目Appkey
|
||||
token: 你的阿里云智能语音交互服务AccessToken，临时的24小时，要长期用下方的access_key_id，access_key_secret
|
||||
voice: longxiaochun
|
||||
access_key_id: 你的阿里云账号access_key_id
|
||||
access_key_secret: 你的阿里云账号access_key_secret
|
||||
# 截至2025年7月21日大模型音色只有北京节点采用，其他节点暂不支持
|
||||
host: nls-gateway-cn-beijing.aliyuncs.com
|
||||
# 以下可不用设置，使用默认设置
|
||||
# format: pcm # 音频格式：pcm、wav、mp3
|
||||
# sample_rate: 16000 # 采样率：8000、16000、24000
|
||||
# volume: 50 # 音量：0-100
|
||||
# speech_rate: 0 # 语速：-500到500
|
||||
# pitch_rate: 0 # 语调：-500到500
|
||||
TencentTTS:
|
||||
# 腾讯云智能语音交互服务，需要先在腾讯云平台开通服务
|
||||
# appid、secret_id、secret_key申请地址：https://console.cloud.tencent.com/cam/capi
|
||||
@@ -696,6 +768,8 @@ TTS:
|
||||
|
||||
TTS302AI:
|
||||
# 302AI语音合成服务，需要先在302平台创建账户充值，并获取密钥信息
|
||||
# 添加 302.ai TTS 配置
|
||||
# token申请地址：https://dash.302.ai/
|
||||
# 获取api_keyn路径：https://dash.302.ai/apis/list
|
||||
# 价格，$35/百万字符。火山原版¥450元/百万字符
|
||||
type: doubao
|
||||
|
||||
@@ -5,7 +5,7 @@ from config.config_loader import load_config
|
||||
from config.settings import check_config_file
|
||||
from datetime import datetime
|
||||
|
||||
SERVER_VERSION = "0.7.1"
|
||||
SERVER_VERSION = "0.7.2"
|
||||
_logger_initialized = False
|
||||
|
||||
|
||||
|
||||
@@ -39,7 +39,7 @@ from config.logger import setup_logging, build_module_string, create_connection_
|
||||
from config.manage_api_client import DeviceNotFoundException, DeviceBindException
|
||||
from core.utils.prompt_manager import PromptManager
|
||||
from core.utils.voiceprint_provider import VoiceprintProvider
|
||||
|
||||
from core.utils import textUtils
|
||||
|
||||
TAG = __name__
|
||||
|
||||
@@ -718,6 +718,7 @@ class ConnectionHandler:
|
||||
function_arguments = ""
|
||||
content_arguments = ""
|
||||
self.client_abort = False
|
||||
emotion_flag = True
|
||||
for response in llm_responses:
|
||||
if self.client_abort:
|
||||
break
|
||||
@@ -743,6 +744,15 @@ class ConnectionHandler:
|
||||
function_arguments += tools_call[0].function.arguments
|
||||
else:
|
||||
content = response
|
||||
|
||||
# 在llm回复中获取情绪表情，一轮对话只在开头获取一次
|
||||
if emotion_flag:
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
textUtils.get_emotion(self, content),
|
||||
self.loop,
|
||||
)
|
||||
emotion_flag = False
|
||||
|
||||
if content is not None and len(content) > 0:
|
||||
if not tool_call_flag:
|
||||
response_message.append(content)
|
||||
|
||||
@@ -2,52 +2,15 @@ import json
|
||||
import asyncio
|
||||
import time
|
||||
from core.providers.tts.dto.dto import SentenceType
|
||||
from core.utils.util import get_string_no_punctuation_or_emoji, analyze_emotion
|
||||
from loguru import logger
|
||||
from core.utils import textUtils
|
||||
|
||||
TAG = __name__
|
||||
|
||||
emoji_map = {
|
||||
"neutral": "😶",
|
||||
"happy": "🙂",
|
||||
"laughing": "😆",
|
||||
"funny": "😂",
|
||||
"sad": "😔",
|
||||
"angry": "😠",
|
||||
"crying": "😭",
|
||||
"loving": "😍",
|
||||
"embarrassed": "😳",
|
||||
"surprised": "😲",
|
||||
"shocked": "😱",
|
||||
"thinking": "🤔",
|
||||
"winking": "😉",
|
||||
"cool": "😎",
|
||||
"relaxed": "😌",
|
||||
"delicious": "🤤",
|
||||
"kissy": "😘",
|
||||
"confident": "😏",
|
||||
"sleepy": "😴",
|
||||
"silly": "😜",
|
||||
"confused": "🙄",
|
||||
}
|
||||
|
||||
|
||||
async def sendAudioMessage(conn, sentenceType, audios, text):
|
||||
# 发送句子开始消息
|
||||
conn.logger.bind(tag=TAG).info(f"发送音频消息: {sentenceType}, {text}")
|
||||
if text is not None:
|
||||
emotion = analyze_emotion(text)
|
||||
emoji = emoji_map.get(emotion, "🙂") # 默认使用笑脸
|
||||
await conn.websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "llm",
|
||||
"text": emoji,
|
||||
"emotion": emotion,
|
||||
"session_id": conn.session_id,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
pre_buffer = False
|
||||
if conn.tts.tts_audio_first_sentence and text is not None:
|
||||
conn.logger.bind(tag=TAG).info(f"发送第一段语音: {text}")
|
||||
@@ -152,8 +115,7 @@ async def send_stt_message(conn, text):
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
# 如果不是JSON格式，直接使用原始文本
|
||||
display_text = text
|
||||
|
||||
stt_text = get_string_no_punctuation_or_emoji(display_text)
|
||||
stt_text = textUtils.get_string_no_punctuation_or_emoji(display_text)
|
||||
await conn.websocket.send(
|
||||
json.dumps({"type": "stt", "text": stt_text, "session_id": conn.session_id})
|
||||
)
|
||||
|
||||
@@ -0,0 +1,337 @@
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
import hmac
|
||||
import base64
|
||||
import hashlib
|
||||
import asyncio
|
||||
import requests
|
||||
import websockets
|
||||
import opuslib_next
|
||||
import random
|
||||
from typing import Optional, Tuple, List
|
||||
from urllib import parse
|
||||
from datetime import datetime
|
||||
from config.logger import setup_logging
|
||||
from core.providers.asr.base import ASRProviderBase
|
||||
from core.providers.asr.dto.dto import InterfaceType
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class AccessToken:
|
||||
@staticmethod
|
||||
def _encode_text(text):
|
||||
encoded_text = parse.quote_plus(text)
|
||||
return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~")
|
||||
|
||||
@staticmethod
|
||||
def _encode_dict(dic):
|
||||
keys = dic.keys()
|
||||
dic_sorted = [(key, dic[key]) for key in sorted(keys)]
|
||||
encoded_text = parse.urlencode(dic_sorted)
|
||||
return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~")
|
||||
|
||||
@staticmethod
|
||||
def create_token(access_key_id, access_key_secret):
|
||||
parameters = {
|
||||
"AccessKeyId": access_key_id,
|
||||
"Action": "CreateToken",
|
||||
"Format": "JSON",
|
||||
"RegionId": "cn-shanghai",
|
||||
"SignatureMethod": "HMAC-SHA1",
|
||||
"SignatureNonce": str(uuid.uuid1()),
|
||||
"SignatureVersion": "1.0",
|
||||
"Timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"Version": "2019-02-28",
|
||||
}
|
||||
query_string = AccessToken._encode_dict(parameters)
|
||||
string_to_sign = (
|
||||
"GET" + "&" + AccessToken._encode_text("/") + "&" + AccessToken._encode_text(query_string)
|
||||
)
|
||||
secreted_string = hmac.new(
|
||||
bytes(access_key_secret + "&", encoding="utf-8"),
|
||||
bytes(string_to_sign, encoding="utf-8"),
|
||||
hashlib.sha1,
|
||||
).digest()
|
||||
signature = base64.b64encode(secreted_string)
|
||||
signature = AccessToken._encode_text(signature)
|
||||
full_url = "http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s" % (signature, query_string)
|
||||
response = requests.get(full_url)
|
||||
if response.ok:
|
||||
root_obj = response.json()
|
||||
if "Token" in root_obj:
|
||||
return root_obj["Token"]["Id"], root_obj["Token"]["ExpireTime"]
|
||||
return None, None
|
||||
|
||||
|
||||
class ASRProvider(ASRProviderBase):
|
||||
def __init__(self, config, delete_audio_file):
|
||||
super().__init__()
|
||||
self.interface_type = InterfaceType.STREAM
|
||||
self.config = config
|
||||
self.text = ""
|
||||
self.decoder = opuslib_next.Decoder(16000, 1)
|
||||
self.asr_ws = None
|
||||
self.forward_task = None
|
||||
self.is_processing = False
|
||||
self.server_ready = False # 服务器准备状态
|
||||
|
||||
# 基础配置
|
||||
self.access_key_id = config.get("access_key_id")
|
||||
self.access_key_secret = config.get("access_key_secret")
|
||||
self.appkey = config.get("appkey")
|
||||
self.token = config.get("token")
|
||||
self.host = config.get("host", "nls-gateway-cn-shanghai.aliyuncs.com")
|
||||
self.ws_url = f"wss://{self.host}/ws/v1"
|
||||
self.max_sentence_silence = config.get("max_sentence_silence")
|
||||
self.output_dir = config.get("output_dir", "./audio_output")
|
||||
self.delete_audio_file = delete_audio_file
|
||||
self.expire_time = None
|
||||
|
||||
# Token管理
|
||||
if self.access_key_id and self.access_key_secret:
|
||||
self._refresh_token()
|
||||
elif not self.token:
|
||||
raise ValueError("必须提供access_key_id+access_key_secret或者直接提供token")
|
||||
|
||||
def _refresh_token(self):
|
||||
"""刷新Token"""
|
||||
self.token, expire_time_str = AccessToken.create_token(self.access_key_id, self.access_key_secret)
|
||||
if not self.token:
|
||||
raise ValueError("无法获取有效的访问Token")
|
||||
|
||||
try:
|
||||
expire_str = str(expire_time_str).strip()
|
||||
if expire_str.isdigit():
|
||||
expire_time = datetime.fromtimestamp(int(expire_str))
|
||||
else:
|
||||
expire_time = datetime.strptime(expire_str, "%Y-%m-%dT%H:%M:%SZ")
|
||||
self.expire_time = expire_time.timestamp() - 60
|
||||
except:
|
||||
self.expire_time = None
|
||||
|
||||
def _is_token_expired(self):
|
||||
"""检查Token是否过期"""
|
||||
return self.expire_time and time.time() > self.expire_time
|
||||
|
||||
async def open_audio_channels(self, conn):
|
||||
await super().open_audio_channels(conn)
|
||||
|
||||
async def receive_audio(self, conn, audio, audio_have_voice):
|
||||
# 初始化音频缓存
|
||||
if not hasattr(conn, 'asr_audio_for_voiceprint'):
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
|
||||
# 存储音频数据
|
||||
if audio:
|
||||
conn.asr_audio_for_voiceprint.append(audio)
|
||||
|
||||
conn.asr_audio.append(audio)
|
||||
conn.asr_audio = conn.asr_audio[-10:]
|
||||
|
||||
# 只在有声音且没有连接时建立连接
|
||||
if audio_have_voice and not self.is_processing:
|
||||
try:
|
||||
await self._start_recognition(conn)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"开始识别失败: {str(e)}")
|
||||
await self._cleanup(conn)
|
||||
return
|
||||
|
||||
if self.asr_ws and self.is_processing and self.server_ready:
|
||||
try:
|
||||
pcm_frame = self.decoder.decode(audio, 960)
|
||||
await self.asr_ws.send(pcm_frame)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).warning(f"发送音频失败: {str(e)}")
|
||||
await self._cleanup(conn)
|
||||
|
||||
async def _start_recognition(self, conn):
|
||||
"""开始识别会话"""
|
||||
if self._is_token_expired():
|
||||
self._refresh_token()
|
||||
|
||||
# 建立连接
|
||||
headers = {"X-NLS-Token": self.token}
|
||||
self.asr_ws = await websockets.connect(
|
||||
self.ws_url,
|
||||
additional_headers=headers,
|
||||
max_size=1000000000,
|
||||
ping_interval=None,
|
||||
ping_timeout=None,
|
||||
close_timeout=5,
|
||||
)
|
||||
|
||||
self.is_processing = True
|
||||
self.server_ready = False # 重置服务器准备状态
|
||||
self.forward_task = asyncio.create_task(self._forward_results(conn))
|
||||
|
||||
# 发送开始请求
|
||||
start_request = {
|
||||
"header": {
|
||||
"namespace": "SpeechTranscriber",
|
||||
"name": "StartTranscription",
|
||||
"status": 20000000,
|
||||
"message_id": ''.join(random.choices('0123456789abcdef', k=32)),
|
||||
"task_id": ''.join(random.choices('0123456789abcdef', k=32)),
|
||||
"status_text": "Gateway:SUCCESS:Success.",
|
||||
"appkey": self.appkey
|
||||
},
|
||||
"payload": {
|
||||
"format": "pcm",
|
||||
"sample_rate": 16000,
|
||||
"enable_intermediate_result": True,
|
||||
"enable_punctuation_prediction": True,
|
||||
"enable_inverse_text_normalization": True,
|
||||
"max_sentence_silence": self.max_sentence_silence,
|
||||
"enable_voice_detection": False,
|
||||
}
|
||||
}
|
||||
await self.asr_ws.send(json.dumps(start_request, ensure_ascii=False))
|
||||
logger.bind(tag=TAG).info("已发送开始请求，等待服务器准备...")
|
||||
|
||||
async def _forward_results(self, conn):
|
||||
"""转发识别结果"""
|
||||
try:
|
||||
while self.asr_ws and not conn.stop_event.is_set():
|
||||
try:
|
||||
response = await asyncio.wait_for(self.asr_ws.recv(), timeout=1.0)
|
||||
result = json.loads(response)
|
||||
|
||||
header = result.get("header", {})
|
||||
payload = result.get("payload", {})
|
||||
message_name = header.get("name", "")
|
||||
status = header.get("status", 0)
|
||||
|
||||
if status != 20000000:
|
||||
if status in [40000004, 40010004]: # 连接超时或客户端断开
|
||||
logger.bind(tag=TAG).warning(f"连接问题，状态码: {status}")
|
||||
break
|
||||
elif status in [40270002, 40270003]: # 音频问题
|
||||
logger.bind(tag=TAG).warning(f"音频处理问题，状态码: {status}")
|
||||
continue
|
||||
else:
|
||||
logger.bind(tag=TAG).error(f"识别错误，状态码: {status}, 消息: {header.get('status_text', '')}")
|
||||
continue
|
||||
|
||||
# 收到TranscriptionStarted表示服务器准备好接收音频数据
|
||||
if message_name == "TranscriptionStarted":
|
||||
self.server_ready = True
|
||||
logger.bind(tag=TAG).info("服务器已准备，开始发送缓存音频...")
|
||||
|
||||
# 发送缓存音频
|
||||
if conn.asr_audio:
|
||||
for cached_audio in conn.asr_audio[-10:]:
|
||||
try:
|
||||
pcm_frame = self.decoder.decode(cached_audio, 960)
|
||||
await self.asr_ws.send(pcm_frame)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).warning(f"发送缓存音频失败: {e}")
|
||||
break
|
||||
continue
|
||||
|
||||
if message_name == "TranscriptionResultChanged":
|
||||
# 中间结果
|
||||
text = payload.get("result", "")
|
||||
if text:
|
||||
self.text = text
|
||||
elif message_name == "SentenceEnd":
|
||||
# 最终结果
|
||||
text = payload.get("result", "")
|
||||
if text:
|
||||
self.text = text
|
||||
conn.reset_vad_states()
|
||||
# 传递缓存的音频数据
|
||||
audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
|
||||
await self.handle_voice_stop(conn, audio_data)
|
||||
# 清空缓存
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
break
|
||||
elif message_name == "TranscriptionCompleted":
|
||||
# 识别完成
|
||||
self.is_processing = False
|
||||
break
|
||||
|
||||
except asyncio.TimeoutError:
|
||||
continue
|
||||
except websockets.exceptions.ConnectionClosed:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"处理结果失败: {str(e)}")
|
||||
break
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"结果转发失败: {str(e)}")
|
||||
finally:
|
||||
await self._cleanup(conn)
|
||||
|
||||
async def _cleanup(self, conn):
|
||||
"""清理资源"""
|
||||
logger.bind(tag=TAG).info(f"开始ASR会话清理 | 当前状态: processing={self.is_processing}, server_ready={self.server_ready}")
|
||||
|
||||
# 清理连接的音频缓存
|
||||
if conn and hasattr(conn, 'asr_audio_for_voiceprint'):
|
||||
conn.asr_audio_for_voiceprint = []
|
||||
|
||||
# 判断是否需要发送终止请求
|
||||
should_stop = self.is_processing or self.server_ready
|
||||
|
||||
# 发送停止识别请求
|
||||
if self.asr_ws and should_stop:
|
||||
try:
|
||||
stop_msg = {
|
||||
"header": {
|
||||
"namespace": "SpeechTranscriber",
|
||||
"name": "StopTranscription",
|
||||
"status": 20000000,
|
||||
"message_id": ''.join(random.choices('0123456789abcdef', k=32)),
|
||||
"status_text": "Client:Stop",
|
||||
"appkey": self.appkey
|
||||
}
|
||||
}
|
||||
logger.bind(tag=TAG).info("正在发送ASR终止请求")
|
||||
await self.asr_ws.send(json.dumps(stop_msg, ensure_ascii=False))
|
||||
await asyncio.sleep(0.1)
|
||||
logger.bind(tag=TAG).info("ASR终止请求已发送")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"ASR终止请求发送失败: {e}")
|
||||
|
||||
# 状态重置（在终止请求发送后）
|
||||
self.is_processing = False
|
||||
self.server_ready = False
|
||||
logger.bind(tag=TAG).info("ASR状态已重置")
|
||||
|
||||
# 清理任务
|
||||
if self.forward_task and not self.forward_task.done():
|
||||
self.forward_task.cancel()
|
||||
try:
|
||||
await asyncio.wait_for(self.forward_task, timeout=1.0)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).debug(f"forward_task取消异常: {e}")
|
||||
finally:
|
||||
self.forward_task = None
|
||||
|
||||
# 关闭连接
|
||||
if self.asr_ws:
|
||||
try:
|
||||
logger.bind(tag=TAG).debug("正在关闭WebSocket连接")
|
||||
await asyncio.wait_for(self.asr_ws.close(), timeout=2.0)
|
||||
logger.bind(tag=TAG).debug("WebSocket连接已关闭")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"关闭WebSocket连接失败: {e}")
|
||||
finally:
|
||||
self.asr_ws = None
|
||||
|
||||
logger.bind(tag=TAG).info("ASR会话清理完成")
|
||||
|
||||
async def speech_to_text(self, opus_data, session_id, audio_format):
|
||||
"""获取识别结果"""
|
||||
result = self.text
|
||||
self.text = ""
|
||||
return result, None
|
||||
|
||||
async def close(self):
|
||||
"""关闭资源"""
|
||||
await self._cleanup()
|
||||
@@ -0,0 +1,82 @@
|
||||
import time
|
||||
import os
|
||||
from config.logger import setup_logging
|
||||
from typing import Optional, Tuple, List
|
||||
from core.providers.asr.dto.dto import InterfaceType
|
||||
from core.providers.asr.base import ASRProviderBase
|
||||
|
||||
import requests
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
class ASRProvider(ASRProviderBase):
|
||||
def __init__(self, config: dict, delete_audio_file: bool):
|
||||
self.interface_type = InterfaceType.NON_STREAM
|
||||
self.api_key = config.get("api_key")
|
||||
self.api_url = config.get("base_url")
|
||||
self.model = config.get("model_name")
|
||||
self.output_dir = config.get("output_dir")
|
||||
self.delete_audio_file = delete_audio_file
|
||||
|
||||
os.makedirs(self.output_dir, exist_ok=True)
|
||||
|
||||
async def speech_to_text(self, opus_data: List[bytes], session_id: str, audio_format="opus") -> Tuple[Optional[str], Optional[str]]:
|
||||
file_path = None
|
||||
try:
|
||||
start_time = time.time()
|
||||
if audio_format == "pcm":
|
||||
pcm_data = opus_data
|
||||
else:
|
||||
pcm_data = self.decode_opus(opus_data)
|
||||
file_path = self.save_audio_to_file(pcm_data, session_id)
|
||||
|
||||
logger.bind(tag=TAG).debug(
|
||||
f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}"
|
||||
)
|
||||
|
||||
logger.bind(tag=TAG).info(f"file path: {file_path}")
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
}
|
||||
|
||||
# 使用data参数传递模型名称
|
||||
data = {
|
||||
"model": self.model
|
||||
}
|
||||
|
||||
|
||||
with open(file_path, "rb") as audio_file: # 使用with语句确保文件关闭
|
||||
files = {
|
||||
"file": audio_file
|
||||
}
|
||||
|
||||
start_time = time.time()
|
||||
response = requests.post(
|
||||
self.api_url,
|
||||
files=files,
|
||||
data=data,
|
||||
headers=headers
|
||||
)
|
||||
logger.bind(tag=TAG).debug(
|
||||
f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {response.text}"
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
text = response.json().get("text", "")
|
||||
return text, file_path
|
||||
else:
|
||||
raise Exception(f"API请求失败: {response.status_code} - {response.text}")
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"语音识别失败: {e}")
|
||||
return "", None
|
||||
finally:
|
||||
# 文件清理逻辑
|
||||
if self.delete_audio_file and file_path and os.path.exists(file_path):
|
||||
try:
|
||||
os.remove(file_path)
|
||||
logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}")
|
||||
|
||||
@@ -6,11 +6,14 @@ import base64
|
||||
import requests
|
||||
from datetime import datetime
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
|
||||
from config.logger import setup_logging
|
||||
import time
|
||||
import uuid
|
||||
from urllib import parse
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class AccessToken:
|
||||
@staticmethod
|
||||
|
||||
@@ -0,0 +1,654 @@
|
||||
import uuid
|
||||
import json
|
||||
import hmac
|
||||
import hashlib
|
||||
import base64
|
||||
import time
|
||||
import queue
|
||||
import asyncio
|
||||
import traceback
|
||||
from asyncio import Task
|
||||
import websockets
|
||||
import os
|
||||
from datetime import datetime
|
||||
from urllib import parse
|
||||
from core.providers.tts.base import TTSProviderBase
|
||||
from core.providers.tts.dto.dto import SentenceType, ContentType, InterfaceType
|
||||
from core.utils.tts import MarkdownCleaner
|
||||
from core.utils import opus_encoder_utils, textUtils
|
||||
from config.logger import setup_logging
|
||||
|
||||
TAG = __name__
|
||||
logger = setup_logging()
|
||||
|
||||
|
||||
class AccessToken:
|
||||
@staticmethod
|
||||
def _encode_text(text):
|
||||
encoded_text = parse.quote_plus(text)
|
||||
return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~")
|
||||
|
||||
@staticmethod
|
||||
def _encode_dict(dic):
|
||||
keys = dic.keys()
|
||||
dic_sorted = [(key, dic[key]) for key in sorted(keys)]
|
||||
encoded_text = parse.urlencode(dic_sorted)
|
||||
return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~")
|
||||
|
||||
@staticmethod
|
||||
def create_token(access_key_id, access_key_secret):
|
||||
parameters = {
|
||||
"AccessKeyId": access_key_id,
|
||||
"Action": "CreateToken",
|
||||
"Format": "JSON",
|
||||
"RegionId": "cn-shanghai", # 使用上海地域进行Token获取
|
||||
"SignatureMethod": "HMAC-SHA1",
|
||||
"SignatureNonce": str(uuid.uuid1()),
|
||||
"SignatureVersion": "1.0",
|
||||
"Timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"Version": "2019-02-28",
|
||||
}
|
||||
|
||||
query_string = AccessToken._encode_dict(parameters)
|
||||
string_to_sign = (
|
||||
"GET"
|
||||
+ "&"
|
||||
+ AccessToken._encode_text("/")
|
||||
+ "&"
|
||||
+ AccessToken._encode_text(query_string)
|
||||
)
|
||||
|
||||
secreted_string = hmac.new(
|
||||
bytes(access_key_secret + "&", encoding="utf-8"),
|
||||
bytes(string_to_sign, encoding="utf-8"),
|
||||
hashlib.sha1,
|
||||
).digest()
|
||||
signature = base64.b64encode(secreted_string)
|
||||
signature = AccessToken._encode_text(signature)
|
||||
|
||||
full_url = "http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s" % (
|
||||
signature,
|
||||
query_string,
|
||||
)
|
||||
|
||||
import requests
|
||||
|
||||
response = requests.get(full_url)
|
||||
if response.ok:
|
||||
root_obj = response.json()
|
||||
key = "Token"
|
||||
if key in root_obj:
|
||||
token = root_obj[key]["Id"]
|
||||
expire_time = root_obj[key]["ExpireTime"]
|
||||
return token, expire_time
|
||||
return None, None
|
||||
|
||||
|
||||
class TTSProvider(TTSProviderBase):
|
||||
def __init__(self, config, delete_audio_file):
|
||||
super().__init__(config, delete_audio_file)
|
||||
|
||||
# 设置为流式接口类型
|
||||
self.interface_type = InterfaceType.DUAL_STREAM
|
||||
|
||||
# 基础配置
|
||||
self.access_key_id = config.get("access_key_id")
|
||||
self.access_key_secret = config.get("access_key_secret")
|
||||
self.appkey = config.get("appkey")
|
||||
self.format = config.get("format", "pcm")
|
||||
self.audio_file_type = config.get("format", "pcm")
|
||||
|
||||
# 采样率配置
|
||||
sample_rate = config.get("sample_rate", "16000")
|
||||
self.sample_rate = int(sample_rate) if sample_rate else 16000
|
||||
|
||||
# 音色配置 - CosyVoice大模型音色
|
||||
if config.get("private_voice"):
|
||||
self.voice = config.get("private_voice")
|
||||
else:
|
||||
self.voice = config.get("voice", "longxiaochun") # CosyVoice默认音色
|
||||
|
||||
# 音频参数配置
|
||||
volume = config.get("volume", "50")
|
||||
self.volume = int(volume) if volume else 50
|
||||
|
||||
speech_rate = config.get("speech_rate", "0")
|
||||
self.speech_rate = int(speech_rate) if speech_rate else 0
|
||||
|
||||
pitch_rate = config.get("pitch_rate", "0")
|
||||
self.pitch_rate = int(pitch_rate) if pitch_rate else 0
|
||||
|
||||
# WebSocket配置
|
||||
self.host = config.get("host", "nls-gateway-cn-beijing.aliyuncs.com")
|
||||
self.ws_url = f"wss://{self.host}/ws/v1"
|
||||
self.ws = None
|
||||
self._monitor_task = None
|
||||
self.last_active_time = None
|
||||
|
||||
# 专属tts设置
|
||||
self.message_id = ""
|
||||
self.tts_text = ""
|
||||
self.text_buffer = []
|
||||
|
||||
# 创建Opus编码器
|
||||
self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
|
||||
sample_rate=16000, channels=1, frame_size_ms=60
|
||||
)
|
||||
|
||||
# Token管理
|
||||
if self.access_key_id and self.access_key_secret:
|
||||
self._refresh_token()
|
||||
else:
|
||||
self.token = config.get("token")
|
||||
self.expire_time = None
|
||||
|
||||
def _refresh_token(self):
|
||||
"""刷新Token并记录过期时间"""
|
||||
if self.access_key_id and self.access_key_secret:
|
||||
self.token, expire_time_str = AccessToken.create_token(
|
||||
self.access_key_id, self.access_key_secret
|
||||
)
|
||||
if not expire_time_str:
|
||||
raise ValueError("无法获取有效的Token过期时间")
|
||||
|
||||
expire_str = str(expire_time_str).strip()
|
||||
|
||||
try:
|
||||
if expire_str.isdigit():
|
||||
expire_time = datetime.fromtimestamp(int(expire_str))
|
||||
else:
|
||||
expire_time = datetime.strptime(expire_str, "%Y-%m-%dT%H:%M:%SZ")
|
||||
self.expire_time = expire_time.timestamp() - 60
|
||||
except Exception as e:
|
||||
raise ValueError(f"无效的过期时间格式: {expire_str}") from e
|
||||
else:
|
||||
self.expire_time = None
|
||||
|
||||
if not self.token:
|
||||
raise ValueError("无法获取有效的访问Token")
|
||||
|
||||
def _is_token_expired(self):
|
||||
"""检查Token是否过期"""
|
||||
if not self.expire_time:
|
||||
return False
|
||||
return time.time() > self.expire_time
|
||||
|
||||
async def _ensure_connection(self):
|
||||
"""确保WebSocket连接可用"""
|
||||
try:
|
||||
if self._is_token_expired():
|
||||
logger.bind(tag=TAG).warning("Token已过期，正在自动刷新...")
|
||||
self._refresh_token()
|
||||
current_time = time.time()
|
||||
if self.ws and current_time - self.last_active_time < 10:
|
||||
# 10秒内才可以复用链接进行连续对话
|
||||
logger.bind(tag=TAG).info(f"使用已有链接...")
|
||||
return self.ws
|
||||
logger.bind(tag=TAG).info("开始建立新连接...")
|
||||
|
||||
self.ws = await websockets.connect(
|
||||
self.ws_url,
|
||||
additional_headers={"X-NLS-Token": self.token},
|
||||
ping_interval=30,
|
||||
ping_timeout=10,
|
||||
close_timeout=10,
|
||||
)
|
||||
logger.bind(tag=TAG).info("WebSocket连接建立成功")
|
||||
self.last_active_time = time.time()
|
||||
return self.ws
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"建立连接失败: {str(e)}")
|
||||
self.ws = None
|
||||
self.last_active_time = None
|
||||
raise
|
||||
|
||||
def tts_text_priority_thread(self):
|
||||
"""流式文本处理线程"""
|
||||
while not self.conn.stop_event.is_set():
|
||||
try:
|
||||
message = self.tts_text_queue.get(timeout=1)
|
||||
logger.bind(tag=TAG).debug(
|
||||
f"收到TTS任务｜{message.sentence_type.name} ｜ {message.content_type.name} | 会话ID: {self.conn.sentence_id}"
|
||||
)
|
||||
|
||||
if message.sentence_type == SentenceType.FIRST:
|
||||
self.conn.client_abort = False
|
||||
|
||||
if self.conn.client_abort:
|
||||
logger.bind(tag=TAG).info("收到打断信息，终止TTS文本处理线程")
|
||||
continue
|
||||
|
||||
if message.sentence_type == SentenceType.FIRST:
|
||||
# 初始化参数
|
||||
try:
|
||||
if not getattr(self.conn, "sentence_id", None):
|
||||
self.conn.sentence_id = uuid.uuid4().hex
|
||||
logger.bind(tag=TAG).info(
|
||||
f"自动生成新的 会话ID: {self.conn.sentence_id}"
|
||||
)
|
||||
|
||||
# aliyunStream独有的参数生成
|
||||
self.message_id = str(uuid.uuid4().hex)
|
||||
self.text_buffer = []
|
||||
|
||||
logger.bind(tag=TAG).info("开始启动TTS会话...")
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
self.start_session(self.conn.sentence_id),
|
||||
loop=self.conn.loop,
|
||||
)
|
||||
future.result()
|
||||
self.before_stop_play_files.clear()
|
||||
logger.bind(tag=TAG).info("TTS会话启动成功")
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"启动TTS会话失败: {str(e)}")
|
||||
continue
|
||||
|
||||
elif ContentType.TEXT == message.content_type:
|
||||
if message.content_detail:
|
||||
try:
|
||||
logger.bind(tag=TAG).debug(
|
||||
f"开始发送TTS文本: {message.content_detail}"
|
||||
)
|
||||
self.text_buffer.append(message.content_detail)
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
self.text_to_speak(message.content_detail, None),
|
||||
loop=self.conn.loop,
|
||||
)
|
||||
future.result()
|
||||
logger.bind(tag=TAG).debug("TTS文本发送成功")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"发送TTS文本失败: {str(e)}")
|
||||
continue
|
||||
|
||||
elif ContentType.FILE == message.content_type:
|
||||
logger.bind(tag=TAG).info(
|
||||
f"添加音频文件到待播放列表: {message.content_file}"
|
||||
)
|
||||
if message.content_file and os.path.exists(message.content_file):
|
||||
# 先处理文件音频数据
|
||||
file_audio = self._process_audio_file(message.content_file)
|
||||
self.before_stop_play_files.append(
|
||||
(file_audio, message.content_detail)
|
||||
)
|
||||
|
||||
if message.sentence_type == SentenceType.LAST:
|
||||
try:
|
||||
logger.bind(tag=TAG).info("开始结束TTS会话...")
|
||||
self.tts_text = textUtils.get_string_no_punctuation_or_emoji(
|
||||
"".join(self.text_buffer).replace("\n", "")
|
||||
)
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
self.finish_session(self.conn.sentence_id),
|
||||
loop=self.conn.loop,
|
||||
)
|
||||
future.result()
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"结束TTS会话失败: {str(e)}")
|
||||
continue
|
||||
|
||||
except queue.Empty:
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(
|
||||
f"处理TTS文本失败: {str(e)}, 类型: {type(e).__name__}, 堆栈: {traceback.format_exc()}"
|
||||
)
|
||||
|
||||
async def text_to_speak(self, text, _):
|
||||
try:
|
||||
if self.ws is None:
|
||||
logger.bind(tag=TAG).warning(f"WebSocket连接不存在，终止发送文本")
|
||||
return
|
||||
filtered_text = MarkdownCleaner.clean_markdown(text)
|
||||
run_request = {
|
||||
"header": {
|
||||
"message_id": self.message_id,
|
||||
"task_id": self.conn.sentence_id,
|
||||
"namespace": "FlowingSpeechSynthesizer",
|
||||
"name": "RunSynthesis",
|
||||
"appkey": self.appkey,
|
||||
},
|
||||
"payload": {"text": filtered_text},
|
||||
}
|
||||
await self.ws.send(json.dumps(run_request))
|
||||
self.last_active_time = time.time()
|
||||
return
|
||||
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"发送TTS文本失败: {str(e)}")
|
||||
if self.ws:
|
||||
try:
|
||||
await self.ws.close()
|
||||
except:
|
||||
pass
|
||||
self.ws = None
|
||||
raise
|
||||
|
||||
async def start_session(self, session_id):
|
||||
logger.bind(tag=TAG).info(f"开始会话～～{session_id}")
|
||||
try:
|
||||
# 会话开始时检测上个会话的监听状态
|
||||
if (
|
||||
self._monitor_task is not None
|
||||
and isinstance(self._monitor_task, Task)
|
||||
and not self._monitor_task.done()
|
||||
):
|
||||
logger.bind(tag=TAG).info(
|
||||
"检测到未完成的上个会话，关闭监听任务和连接..."
|
||||
)
|
||||
await self.close()
|
||||
|
||||
# 建立新连接
|
||||
await self._ensure_connection()
|
||||
|
||||
# 启动监听任务
|
||||
self._monitor_task = asyncio.create_task(self._start_monitor_tts_response())
|
||||
|
||||
start_request = {
|
||||
"header": {
|
||||
"message_id": self.message_id,
|
||||
"task_id": self.conn.sentence_id,
|
||||
"namespace": "FlowingSpeechSynthesizer",
|
||||
"name": "StartSynthesis",
|
||||
"appkey": self.appkey,
|
||||
},
|
||||
"payload": {
|
||||
"voice": self.voice,
|
||||
"format": self.format,
|
||||
"sample_rate": self.sample_rate,
|
||||
"volume": self.volume,
|
||||
"speech_rate": self.speech_rate,
|
||||
"pitch_rate": self.pitch_rate,
|
||||
"enable_subtitle": True,
|
||||
},
|
||||
}
|
||||
await self.ws.send(json.dumps(start_request))
|
||||
self.last_active_time = time.time()
|
||||
logger.bind(tag=TAG).info("会话启动请求已发送")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"启动会话失败: {str(e)}")
|
||||
# 确保清理资源
|
||||
await self.close()
|
||||
raise
|
||||
|
||||
async def finish_session(self, session_id):
|
||||
logger.bind(tag=TAG).info(f"关闭会话～～{session_id}")
|
||||
try:
|
||||
if self.ws:
|
||||
stop_request = {
|
||||
"header": {
|
||||
"message_id": self.message_id,
|
||||
"task_id": self.conn.sentence_id,
|
||||
"namespace": "FlowingSpeechSynthesizer",
|
||||
"name": "StopSynthesis",
|
||||
"appkey": self.appkey,
|
||||
}
|
||||
}
|
||||
await self.ws.send(json.dumps(stop_request))
|
||||
logger.bind(tag=TAG).info("会话结束请求已发送")
|
||||
self.last_active_time = time.time()
|
||||
if self._monitor_task:
|
||||
try:
|
||||
await self._monitor_task
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(
|
||||
f"等待监听任务完成时发生错误: {str(e)}"
|
||||
)
|
||||
finally:
|
||||
self._monitor_task = None
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"关闭会话失败: {str(e)}")
|
||||
# 确保清理资源
|
||||
await self.close()
|
||||
raise
|
||||
|
||||
async def close(self):
|
||||
"""资源清理"""
|
||||
if self._monitor_task:
|
||||
try:
|
||||
self._monitor_task.cancel()
|
||||
await self._monitor_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).warning(f"关闭时取消监听任务错误: {e}")
|
||||
self._monitor_task = None
|
||||
|
||||
if self.ws:
|
||||
try:
|
||||
await self.ws.close()
|
||||
except:
|
||||
pass
|
||||
self.ws = None
|
||||
self.last_active_time = None
|
||||
|
||||
async def _start_monitor_tts_response(self):
|
||||
"""监听TTS响应"""
|
||||
opus_datas_cache = []
|
||||
is_first_sentence = True
|
||||
first_sentence_segment_count = 0 # 添加计数器
|
||||
try:
|
||||
session_finished = False # 标记会话是否正常结束
|
||||
while not self.conn.stop_event.is_set():
|
||||
try:
|
||||
msg = await self.ws.recv()
|
||||
self.last_active_time = time.time()
|
||||
# 检查客户端是否中止
|
||||
if self.conn.client_abort:
|
||||
logger.bind(tag=TAG).info("收到打断信息，终止监听TTS响应")
|
||||
break
|
||||
if isinstance(msg, str): # 文本控制消息
|
||||
try:
|
||||
data = json.loads(msg)
|
||||
header = data.get("header", {})
|
||||
event_name = header.get("name")
|
||||
if event_name == "SynthesisStarted":
|
||||
logger.bind(tag=TAG).debug("TTS合成已启动")
|
||||
elif event_name == "SentenceBegin":
|
||||
logger.bind(tag=TAG).debug(
|
||||
f"句子语音生成开始: {self.tts_text}"
|
||||
)
|
||||
opus_datas_cache = []
|
||||
self.tts_audio_queue.put(
|
||||
(SentenceType.FIRST, [], self.tts_text)
|
||||
)
|
||||
elif event_name == "SentenceEnd":
|
||||
logger.bind(tag=TAG).info(
|
||||
f"句子语音生成成功： {self.tts_text}"
|
||||
)
|
||||
if (
|
||||
not is_first_sentence
|
||||
or first_sentence_segment_count > 10
|
||||
):
|
||||
# 发送缓存的数据
|
||||
self.tts_audio_queue.put(
|
||||
(SentenceType.MIDDLE, opus_datas_cache, None)
|
||||
)
|
||||
# 第一句话结束后，将标志设置为False
|
||||
is_first_sentence = False
|
||||
elif event_name == "SynthesisCompleted":
|
||||
logger.bind(tag=TAG).debug(f"会话结束～～")
|
||||
self._process_before_stop_play_files()
|
||||
session_finished = True
|
||||
self.reuse_judgment = time.time()
|
||||
self.tts_text = ""
|
||||
break
|
||||
except json.JSONDecodeError:
|
||||
logger.bind(tag=TAG).warning("收到无效的JSON消息")
|
||||
# 二进制消息（音频数据）
|
||||
elif isinstance(msg, (bytes, bytearray)):
|
||||
logger.bind(tag=TAG).debug(f"推送数据到队列里面～～")
|
||||
opus_datas = self.opus_encoder.encode_pcm_to_opus(msg, False)
|
||||
logger.bind(tag=TAG).debug(
|
||||
f"推送数据到队列里面帧数～～{len(opus_datas)}"
|
||||
)
|
||||
if is_first_sentence:
|
||||
first_sentence_segment_count += 1
|
||||
if first_sentence_segment_count <= 6:
|
||||
self.tts_audio_queue.put(
|
||||
(SentenceType.MIDDLE, opus_datas, None)
|
||||
)
|
||||
else:
|
||||
opus_datas_cache.extend(opus_datas)
|
||||
else:
|
||||
# 后续句子缓存
|
||||
opus_datas_cache.extend(opus_datas)
|
||||
|
||||
except websockets.ConnectionClosed:
|
||||
logger.bind(tag=TAG).warning("WebSocket连接已关闭")
|
||||
break
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(
|
||||
f"处理TTS响应时出错: {e}\n{traceback.format_exc()}"
|
||||
)
|
||||
break
|
||||
# 仅在连接异常时才关闭
|
||||
if not session_finished and self.ws:
|
||||
try:
|
||||
await self.ws.close()
|
||||
except:
|
||||
pass
|
||||
self.ws = None
|
||||
# 监听任务退出时清理引用
|
||||
finally:
|
||||
self._monitor_task = None
|
||||
|
||||
def to_tts(self, text: str) -> list:
|
||||
"""非流式TTS处理，用于测试及保存音频文件的场景"""
|
||||
try:
|
||||
# 创建新的事件循环
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
|
||||
# 生成会话ID
|
||||
session_id = uuid.uuid4().hex
|
||||
# 存储音频数据
|
||||
audio_data = []
|
||||
|
||||
async def _generate_audio():
|
||||
# 刷新Token（如果需要）
|
||||
if self._is_token_expired():
|
||||
self._refresh_token()
|
||||
|
||||
# 建立WebSocket连接
|
||||
ws = await websockets.connect(
|
||||
self.ws_url,
|
||||
additional_headers={"X-NLS-Token": self.token},
|
||||
ping_interval=30,
|
||||
ping_timeout=10,
|
||||
close_timeout=10,
|
||||
)
|
||||
try:
|
||||
# 发送StartSynthesis请求
|
||||
start_message_id = str(uuid.uuid4().hex)
|
||||
start_request = {
|
||||
"header": {
|
||||
"message_id": start_message_id,
|
||||
"task_id": session_id,
|
||||
"namespace": "FlowingSpeechSynthesizer",
|
||||
"name": "StartSynthesis",
|
||||
"appkey": self.appkey,
|
||||
},
|
||||
"payload": {
|
||||
"voice": self.voice,
|
||||
"format": self.format,
|
||||
"sample_rate": self.sample_rate,
|
||||
"volume": self.volume,
|
||||
"speech_rate": self.speech_rate,
|
||||
"pitch_rate": self.pitch_rate,
|
||||
"enable_subtitle": True,
|
||||
},
|
||||
}
|
||||
await ws.send(json.dumps(start_request))
|
||||
|
||||
# 等待SynthesisStarted响应
|
||||
synthesis_started = False
|
||||
while not synthesis_started:
|
||||
msg = await ws.recv()
|
||||
if isinstance(msg, str):
|
||||
data = json.loads(msg)
|
||||
header = data.get("header", {})
|
||||
if header.get("name") == "SynthesisStarted":
|
||||
synthesis_started = True
|
||||
logger.bind(tag=TAG).debug("TTS合成已启动")
|
||||
elif header.get("name") == "TaskFailed":
|
||||
error_info = data.get("payload", {}).get(
|
||||
"error_info", {}
|
||||
)
|
||||
error_code = error_info.get("error_code")
|
||||
error_message = error_info.get(
|
||||
"error_message", "未知错误"
|
||||
)
|
||||
raise Exception(
|
||||
f"启动合成失败: {error_code} - {error_message}"
|
||||
)
|
||||
|
||||
# 发送文本合成请求
|
||||
filtered_text = MarkdownCleaner.clean_markdown(text)
|
||||
run_message_id = str(uuid.uuid4().hex)
|
||||
run_request = {
|
||||
"header": {
|
||||
"message_id": run_message_id,
|
||||
"task_id": session_id,
|
||||
"namespace": "FlowingSpeechSynthesizer",
|
||||
"name": "RunSynthesis",
|
||||
"appkey": self.appkey,
|
||||
},
|
||||
"payload": {"text": filtered_text},
|
||||
}
|
||||
await ws.send(json.dumps(run_request))
|
||||
|
||||
# 发送停止合成请求
|
||||
stop_message_id = str(uuid.uuid4().hex)
|
||||
stop_request = {
|
||||
"header": {
|
||||
"message_id": stop_message_id,
|
||||
"task_id": session_id,
|
||||
"namespace": "FlowingSpeechSynthesizer",
|
||||
"name": "StopSynthesis",
|
||||
"appkey": self.appkey,
|
||||
}
|
||||
}
|
||||
await ws.send(json.dumps(stop_request))
|
||||
|
||||
# 接收音频数据
|
||||
synthesis_completed = False
|
||||
while not synthesis_completed:
|
||||
msg = await ws.recv()
|
||||
if isinstance(msg, (bytes, bytearray)):
|
||||
# 编码为Opus并收集
|
||||
opus_frames = self.opus_encoder.encode_pcm_to_opus(
|
||||
msg, False
|
||||
)
|
||||
audio_data.extend(opus_frames)
|
||||
elif isinstance(msg, str):
|
||||
data = json.loads(msg)
|
||||
header = data.get("header", {})
|
||||
event_name = header.get("name")
|
||||
if event_name == "SynthesisCompleted":
|
||||
synthesis_completed = True
|
||||
logger.bind(tag=TAG).debug("TTS合成完成")
|
||||
elif event_name == "TaskFailed":
|
||||
error_info = data.get("payload", {}).get(
|
||||
"error_info", {}
|
||||
)
|
||||
error_code = error_info.get("error_code")
|
||||
error_message = error_info.get(
|
||||
"error_message", "未知错误"
|
||||
)
|
||||
raise Exception(
|
||||
f"合成失败: {error_code} - {error_message}"
|
||||
)
|
||||
finally:
|
||||
try:
|
||||
await ws.close()
|
||||
except:
|
||||
pass
|
||||
|
||||
loop.run_until_complete(_generate_audio())
|
||||
loop.close()
|
||||
|
||||
return audio_data
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"生成音频数据失败: {str(e)}")
|
||||
return []
|
||||
@@ -50,6 +50,7 @@ class TTSProviderBase(ABC):
|
||||
"；",
|
||||
";",
|
||||
"：",
|
||||
"~",
|
||||
)
|
||||
self.first_sentence_punctuations = (
|
||||
"，",
|
||||
@@ -159,7 +160,7 @@ class TTSProviderBase(ABC):
|
||||
if conn.sentence_id:
|
||||
sentence_id = conn.sentence_id
|
||||
else:
|
||||
sentence_id = str(uuid.uuid4()).replace("-", "")
|
||||
sentence_id = str(uuid.uuid4().hex)
|
||||
conn.sentence_id = sentence_id
|
||||
# 对于单句的文本，进行分段处理
|
||||
segments = re.split(r"([。！？!?；;\n])", content_detail)
|
||||
@@ -332,7 +333,6 @@ class TTSProviderBase(ABC):
|
||||
Returns:
|
||||
tuple: (sentence_type, audio_datas, content_detail)
|
||||
"""
|
||||
audio_datas = []
|
||||
if tts_file.endswith(".p3"):
|
||||
audio_datas, _ = p3.decode_opus_from_file(tts_file)
|
||||
elif self.conn.audio_format == "pcm":
|
||||
|
||||
@@ -52,10 +52,11 @@ EVENT_ConnectionFinished = 52 # 连接结束
|
||||
|
||||
# 上行Session事件
|
||||
EVENT_StartSession = 100
|
||||
|
||||
EVENT_CancelSession = 101
|
||||
EVENT_FinishSession = 102
|
||||
# 下行Session事件
|
||||
EVENT_SessionStarted = 150
|
||||
EVENT_SessionCanceled = 151
|
||||
EVENT_SessionFinished = 152
|
||||
|
||||
EVENT_SessionFailed = 153
|
||||
@@ -207,8 +208,16 @@ class TTSProvider(TTSProviderBase):
|
||||
self.conn.client_abort = False
|
||||
|
||||
if self.conn.client_abort:
|
||||
logger.bind(tag=TAG).info("收到打断信息，终止TTS文本处理线程")
|
||||
continue
|
||||
try:
|
||||
logger.bind(tag=TAG).info("收到打断信息，终止TTS文本处理线程")
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
self.cancel_session(self.conn.sentence_id),
|
||||
loop=self.conn.loop,
|
||||
)
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"取消TTS会话失败: {str(e)}")
|
||||
continue
|
||||
|
||||
if message.sentence_type == SentenceType.FIRST:
|
||||
# 初始化参数
|
||||
@@ -371,6 +380,27 @@ class TTSProvider(TTSProviderBase):
|
||||
await self.close()
|
||||
raise
|
||||
|
||||
async def cancel_session(self,session_id):
|
||||
logger.bind(tag=TAG).info(f"取消会话，释放服务端资源～～{session_id}")
|
||||
try:
|
||||
if self.ws:
|
||||
header = Header(
|
||||
message_type=FULL_CLIENT_REQUEST,
|
||||
message_type_specific_flags=MsgTypeFlagWithEvent,
|
||||
serial_method=JSON,
|
||||
).as_bytes()
|
||||
optional = Optional(
|
||||
event=EVENT_CancelSession, sessionId=session_id
|
||||
).as_bytes()
|
||||
payload = str.encode("{}")
|
||||
await self.send_event(self.ws, header, optional, payload)
|
||||
logger.bind(tag=TAG).info("会话取消请求已发送")
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"取消会话失败: {str(e)}")
|
||||
# 确保清理资源
|
||||
await self.close()
|
||||
raise
|
||||
|
||||
async def close(self):
|
||||
"""资源清理方法"""
|
||||
# 取消监听任务
|
||||
@@ -405,12 +435,11 @@ class TTSProvider(TTSProviderBase):
|
||||
res = self.parser_response(msg)
|
||||
self.print_response(res, "send_text res:")
|
||||
|
||||
# 检查客户端是否中止
|
||||
if self.conn.client_abort:
|
||||
logger.bind(tag=TAG).info("收到打断信息，终止监听TTS响应")
|
||||
if res.optional.event == EVENT_SessionCanceled:
|
||||
logger.bind(tag=TAG).debug(f"释放服务端资源成功～～")
|
||||
session_finished = True
|
||||
break
|
||||
|
||||
if res.optional.event == EVENT_TTSSentenceStart:
|
||||
elif res.optional.event == EVENT_TTSSentenceStart:
|
||||
json_data = json.loads(res.payload.decode("utf-8"))
|
||||
self.tts_text = json_data.get("text", "")
|
||||
logger.bind(tag=TAG).debug(f"句子语音生成开始: {self.tts_text}")
|
||||
@@ -435,10 +464,10 @@ class TTSProvider(TTSProviderBase):
|
||||
(SentenceType.MIDDLE, opus_datas, None)
|
||||
)
|
||||
else:
|
||||
opus_datas_cache = opus_datas_cache + opus_datas
|
||||
opus_datas_cache.extend(opus_datas)
|
||||
else:
|
||||
# 后续句子缓存
|
||||
opus_datas_cache = opus_datas_cache + opus_datas
|
||||
opus_datas_cache.extend(opus_datas)
|
||||
elif res.optional.event == EVENT_TTSSentenceEnd:
|
||||
logger.bind(tag=TAG).info(f"句子语音生成成功：{self.tts_text}")
|
||||
if not is_first_sentence or first_sentence_segment_count > 10:
|
||||
@@ -469,7 +498,7 @@ class TTSProvider(TTSProviderBase):
|
||||
except:
|
||||
pass
|
||||
self.ws = None
|
||||
# 监听任务退出时清理引用
|
||||
# 监听任务退出时清理引用
|
||||
finally:
|
||||
self._monitor_task = None
|
||||
|
||||
|
||||
@@ -21,6 +21,30 @@ WEEKDAY_MAP = {
|
||||
"Sunday": "星期日",
|
||||
}
|
||||
|
||||
EMOJI_List = [
|
||||
"😶",
|
||||
"🙂",
|
||||
"😆",
|
||||
"😂",
|
||||
"😔",
|
||||
"😠",
|
||||
"😭",
|
||||
"😍",
|
||||
"😳",
|
||||
"😲",
|
||||
"😱",
|
||||
"🤔",
|
||||
"😉",
|
||||
"😎",
|
||||
"😌",
|
||||
"🤤",
|
||||
"😘",
|
||||
"😏",
|
||||
"😴",
|
||||
"😜",
|
||||
"🙄",
|
||||
]
|
||||
|
||||
|
||||
class PromptManager:
|
||||
"""系统提示词管理器，负责管理和更新系统提示词"""
|
||||
@@ -206,6 +230,7 @@ class PromptManager:
|
||||
lunar_date=lunar_date,
|
||||
local_address=local_address,
|
||||
weather_info=weather_info,
|
||||
emojiList=EMOJI_List,
|
||||
)
|
||||
device_cache_key = f"device_prompt:{device_id}"
|
||||
self.cache_manager.set(
|
||||
|
||||
@@ -1,3 +1,31 @@
|
||||
import json
|
||||
|
||||
TAG = __name__
|
||||
EMOJI_MAP = {
|
||||
"😂": "laughing",
|
||||
"😭": "crying",
|
||||
"😠": "angry",
|
||||
"😔": "sad",
|
||||
"😍": "loving",
|
||||
"😲": "surprised",
|
||||
"😱": "shocked",
|
||||
"🤔": "thinking",
|
||||
"😌": "relaxed",
|
||||
"😴": "sleepy",
|
||||
"😜": "silly",
|
||||
"🙄": "confused",
|
||||
"😶": "neutral",
|
||||
"🙂": "happy",
|
||||
"😆": "laughing",
|
||||
"😳": "embarrassed",
|
||||
"😉": "winking",
|
||||
"😎": "cool",
|
||||
"🤤": "delicious",
|
||||
"😘": "kissy",
|
||||
"😏": "confident",
|
||||
}
|
||||
|
||||
|
||||
def get_string_no_punctuation_or_emoji(s):
|
||||
"""去除字符串首尾的空格、标点符号和表情符号"""
|
||||
chars = list(s)
|
||||
@@ -22,6 +50,11 @@ def is_punctuation_or_emoji(char):
|
||||
".", # 中文句号 + 英文句号
|
||||
"！",
|
||||
"!", # 中文感叹号 + 英文感叹号
|
||||
"“",
|
||||
"”",
|
||||
'"', # 中文双引号 + 英文引号
|
||||
"：",
|
||||
":", # 中文冒号 + 英文冒号
|
||||
"-",
|
||||
"－", # 英文连字符 + 中文全角横线
|
||||
"、", # 中文顿号
|
||||
@@ -44,3 +77,28 @@ def is_punctuation_or_emoji(char):
|
||||
(0x2700, 0x27BF),
|
||||
]
|
||||
return any(start <= code_point <= end for start, end in emoji_ranges)
|
||||
|
||||
|
||||
async def get_emotion(conn, text):
|
||||
"""获取文本内的情绪消息"""
|
||||
emoji = "🙂"
|
||||
emotion = "happy"
|
||||
for char in text:
|
||||
if char in EMOJI_MAP:
|
||||
emoji = char
|
||||
emotion = EMOJI_MAP[char]
|
||||
break
|
||||
try:
|
||||
await conn.websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "llm",
|
||||
"text": emoji,
|
||||
"emotion": emotion,
|
||||
"session_id": conn.session_id,
|
||||
}
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
conn.logger.bind(tag=TAG).warning(f"发送情绪表情失败，错误:{e}")
|
||||
return
|
||||
|
||||
@@ -125,51 +125,6 @@ def write_json_file(file_path, data):
|
||||
json.dump(data, file, ensure_ascii=False, indent=4)
|
||||
|
||||
|
||||
def is_punctuation_or_emoji(char):
|
||||
"""检查字符是否为空格、指定标点或表情符号"""
|
||||
# 定义需要去除的中英文标点（包括全角/半角）
|
||||
punctuation_set = {
|
||||
"，",
|
||||
",", # 中文逗号 + 英文逗号
|
||||
"-",
|
||||
"－", # 英文连字符 + 中文全角横线
|
||||
"、", # 中文顿号
|
||||
"“",
|
||||
"”",
|
||||
'"', # 中文双引号 + 英文引号
|
||||
"：",
|
||||
":", # 中文冒号 + 英文冒号
|
||||
}
|
||||
if char.isspace() or char in punctuation_set:
|
||||
return True
|
||||
# 检查表情符号（保留原有逻辑）
|
||||
code_point = ord(char)
|
||||
emoji_ranges = [
|
||||
(0x1F600, 0x1F64F),
|
||||
(0x1F300, 0x1F5FF),
|
||||
(0x1F680, 0x1F6FF),
|
||||
(0x1F900, 0x1F9FF),
|
||||
(0x1FA70, 0x1FAFF),
|
||||
(0x2600, 0x26FF),
|
||||
(0x2700, 0x27BF),
|
||||
]
|
||||
return any(start <= code_point <= end for start, end in emoji_ranges)
|
||||
|
||||
|
||||
def get_string_no_punctuation_or_emoji(s):
|
||||
"""去除字符串首尾的空格、标点符号和表情符号"""
|
||||
chars = list(s)
|
||||
# 处理开头的字符
|
||||
start = 0
|
||||
while start < len(chars) and is_punctuation_or_emoji(chars[start]):
|
||||
start += 1
|
||||
# 处理结尾的字符
|
||||
end = len(chars) - 1
|
||||
while end >= start and is_punctuation_or_emoji(chars[end]):
|
||||
end -= 1
|
||||
return "".join(chars[start : end + 1])
|
||||
|
||||
|
||||
def remove_punctuation_and_length(text):
|
||||
# 全角符号和半角符号的Unicode范围
|
||||
full_width_punctuations = (
|
||||
@@ -256,515 +211,6 @@ def extract_json_from_string(input_string):
|
||||
return None
|
||||
|
||||
|
||||
def analyze_emotion(text):
|
||||
"""
|
||||
分析文本情感并返回对应的emoji名称（支持中英文）
|
||||
"""
|
||||
if not text or not isinstance(text, str):
|
||||
return "neutral"
|
||||
|
||||
original_text = text
|
||||
text = text.lower().strip()
|
||||
|
||||
# 检查是否包含现有emoji
|
||||
for emotion, emoji in emoji_map.items():
|
||||
if emoji in original_text:
|
||||
return emotion
|
||||
|
||||
# 标点符号分析
|
||||
has_exclamation = "!" in original_text or "！" in original_text
|
||||
has_question = "?" in original_text or "？" in original_text
|
||||
has_ellipsis = "..." in original_text or "…" in original_text
|
||||
|
||||
# 定义情感关键词映射（中英文扩展版）
|
||||
emotion_keywords = {
|
||||
"happy": [
|
||||
"开心",
|
||||
"高兴",
|
||||
"快乐",
|
||||
"愉快",
|
||||
"幸福",
|
||||
"满意",
|
||||
"棒",
|
||||
"好",
|
||||
"不错",
|
||||
"完美",
|
||||
"棒极了",
|
||||
"太好了",
|
||||
"好呀",
|
||||
"好的",
|
||||
"happy",
|
||||
"joy",
|
||||
"great",
|
||||
"good",
|
||||
"nice",
|
||||
"awesome",
|
||||
"fantastic",
|
||||
"wonderful",
|
||||
],
|
||||
"laughing": [
|
||||
"哈哈",
|
||||
"哈哈哈",
|
||||
"呵呵",
|
||||
"嘿嘿",
|
||||
"嘻嘻",
|
||||
"笑死",
|
||||
"太好笑了",
|
||||
"笑死我了",
|
||||
"lol",
|
||||
"lmao",
|
||||
"haha",
|
||||
"hahaha",
|
||||
"hehe",
|
||||
"rofl",
|
||||
"funny",
|
||||
"laugh",
|
||||
],
|
||||
"funny": [
|
||||
"搞笑",
|
||||
"滑稽",
|
||||
"逗",
|
||||
"幽默",
|
||||
"笑点",
|
||||
"段子",
|
||||
"笑话",
|
||||
"太逗了",
|
||||
"hilarious",
|
||||
"joke",
|
||||
"comedy",
|
||||
],
|
||||
"sad": [
|
||||
"伤心",
|
||||
"难过",
|
||||
"悲哀",
|
||||
"悲伤",
|
||||
"忧郁",
|
||||
"郁闷",
|
||||
"沮丧",
|
||||
"失望",
|
||||
"想哭",
|
||||
"难受",
|
||||
"不开心",
|
||||
"唉",
|
||||
"呜呜",
|
||||
"sad",
|
||||
"upset",
|
||||
"unhappy",
|
||||
"depressed",
|
||||
"sorrow",
|
||||
"gloomy",
|
||||
],
|
||||
"angry": [
|
||||
"生气",
|
||||
"愤怒",
|
||||
"气死",
|
||||
"讨厌",
|
||||
"烦人",
|
||||
"可恶",
|
||||
"烦死了",
|
||||
"恼火",
|
||||
"暴躁",
|
||||
"火大",
|
||||
"愤怒",
|
||||
"气炸了",
|
||||
"angry",
|
||||
"mad",
|
||||
"annoyed",
|
||||
"furious",
|
||||
"pissed",
|
||||
"hate",
|
||||
],
|
||||
"crying": [
|
||||
"哭泣",
|
||||
"泪流",
|
||||
"大哭",
|
||||
"伤心欲绝",
|
||||
"泪目",
|
||||
"流泪",
|
||||
"哭死",
|
||||
"哭晕",
|
||||
"想哭",
|
||||
"泪崩",
|
||||
"cry",
|
||||
"crying",
|
||||
"tears",
|
||||
"sob",
|
||||
"weep",
|
||||
],
|
||||
"loving": [
|
||||
"爱你",
|
||||
"喜欢",
|
||||
"爱",
|
||||
"亲爱的",
|
||||
"宝贝",
|
||||
"么么哒",
|
||||
"抱抱",
|
||||
"想你",
|
||||
"思念",
|
||||
"最爱",
|
||||
"亲亲",
|
||||
"喜欢你",
|
||||
"love",
|
||||
"like",
|
||||
"adore",
|
||||
"darling",
|
||||
"sweetie",
|
||||
"honey",
|
||||
"miss you",
|
||||
"heart",
|
||||
],
|
||||
"embarrassed": [
|
||||
"尴尬",
|
||||
"不好意思",
|
||||
"害羞",
|
||||
"脸红",
|
||||
"难为情",
|
||||
"社死",
|
||||
"丢脸",
|
||||
"出丑",
|
||||
"embarrassed",
|
||||
"awkward",
|
||||
"shy",
|
||||
"blush",
|
||||
],
|
||||
"surprised": [
|
||||
"惊讶",
|
||||
"吃惊",
|
||||
"天啊",
|
||||
"哇塞",
|
||||
"哇",
|
||||
"居然",
|
||||
"竟然",
|
||||
"没想到",
|
||||
"出乎意料",
|
||||
"surprise",
|
||||
"wow",
|
||||
"omg",
|
||||
"oh my god",
|
||||
"amazing",
|
||||
"unbelievable",
|
||||
],
|
||||
"shocked": [
|
||||
"震惊",
|
||||
"吓到",
|
||||
"惊呆了",
|
||||
"不敢相信",
|
||||
"震撼",
|
||||
"吓死",
|
||||
"恐怖",
|
||||
"害怕",
|
||||
"吓人",
|
||||
"shocked",
|
||||
"shocking",
|
||||
"scared",
|
||||
"frightened",
|
||||
"terrified",
|
||||
"horror",
|
||||
],
|
||||
"thinking": [
|
||||
"思考",
|
||||
"考虑",
|
||||
"想一下",
|
||||
"琢磨",
|
||||
"沉思",
|
||||
"冥想",
|
||||
"想",
|
||||
"思考中",
|
||||
"在想",
|
||||
"think",
|
||||
"thinking",
|
||||
"consider",
|
||||
"ponder",
|
||||
"meditate",
|
||||
],
|
||||
"winking": [
|
||||
"调皮",
|
||||
"眨眼",
|
||||
"你懂的",
|
||||
"坏笑",
|
||||
"邪恶",
|
||||
"奸笑",
|
||||
"使眼色",
|
||||
"wink",
|
||||
"teasing",
|
||||
"naughty",
|
||||
"mischievous",
|
||||
],
|
||||
"cool": [
|
||||
"酷",
|
||||
"帅",
|
||||
"厉害",
|
||||
"棒极了",
|
||||
"真棒",
|
||||
"牛逼",
|
||||
"强",
|
||||
"优秀",
|
||||
"杰出",
|
||||
"出色",
|
||||
"完美",
|
||||
"cool",
|
||||
"awesome",
|
||||
"amazing",
|
||||
"great",
|
||||
"impressive",
|
||||
"perfect",
|
||||
],
|
||||
"relaxed": [
|
||||
"放松",
|
||||
"舒服",
|
||||
"惬意",
|
||||
"悠闲",
|
||||
"轻松",
|
||||
"舒适",
|
||||
"安逸",
|
||||
"自在",
|
||||
"relax",
|
||||
"relaxed",
|
||||
"comfortable",
|
||||
"cozy",
|
||||
"chill",
|
||||
"peaceful",
|
||||
],
|
||||
"delicious": [
|
||||
"好吃",
|
||||
"美味",
|
||||
"香",
|
||||
"馋",
|
||||
"可口",
|
||||
"香甜",
|
||||
"大餐",
|
||||
"大快朵颐",
|
||||
"流口水",
|
||||
"垂涎",
|
||||
"delicious",
|
||||
"yummy",
|
||||
"tasty",
|
||||
"yum",
|
||||
"appetizing",
|
||||
"mouthwatering",
|
||||
],
|
||||
"kissy": [
|
||||
"亲亲",
|
||||
"么么",
|
||||
"吻",
|
||||
"mua",
|
||||
"muah",
|
||||
"亲一下",
|
||||
"飞吻",
|
||||
"kiss",
|
||||
"xoxo",
|
||||
"hug",
|
||||
"muah",
|
||||
"smooch",
|
||||
],
|
||||
"confident": [
|
||||
"自信",
|
||||
"肯定",
|
||||
"确定",
|
||||
"毫无疑问",
|
||||
"当然",
|
||||
"必须的",
|
||||
"毫无疑问",
|
||||
"确信",
|
||||
"坚信",
|
||||
"confident",
|
||||
"sure",
|
||||
"certain",
|
||||
"definitely",
|
||||
"positive",
|
||||
],
|
||||
"sleepy": [
|
||||
"困",
|
||||
"睡觉",
|
||||
"晚安",
|
||||
"想睡",
|
||||
"好累",
|
||||
"疲惫",
|
||||
"疲倦",
|
||||
"困了",
|
||||
"想休息",
|
||||
"睡意",
|
||||
"sleep",
|
||||
"sleepy",
|
||||
"tired",
|
||||
"exhausted",
|
||||
"bedtime",
|
||||
"good night",
|
||||
],
|
||||
"silly": [
|
||||
"傻",
|
||||
"笨",
|
||||
"呆",
|
||||
"憨",
|
||||
"蠢",
|
||||
"二",
|
||||
"憨憨",
|
||||
"傻乎乎",
|
||||
"呆萌",
|
||||
"silly",
|
||||
"stupid",
|
||||
"dumb",
|
||||
"foolish",
|
||||
"goofy",
|
||||
"ridiculous",
|
||||
],
|
||||
"confused": [
|
||||
"疑惑",
|
||||
"不明白",
|
||||
"不懂",
|
||||
"困惑",
|
||||
"疑问",
|
||||
"为什么",
|
||||
"怎么回事",
|
||||
"啥意思",
|
||||
"不清楚",
|
||||
"confused",
|
||||
"puzzled",
|
||||
"doubt",
|
||||
"question",
|
||||
"what",
|
||||
"why",
|
||||
"how",
|
||||
],
|
||||
}
|
||||
|
||||
# 特殊句型判断（中英文）
|
||||
# 赞美他人
|
||||
if any(
|
||||
phrase in text
|
||||
for phrase in [
|
||||
"你真",
|
||||
"你好",
|
||||
"您真",
|
||||
"你真棒",
|
||||
"你好厉害",
|
||||
"你太强了",
|
||||
"你真好",
|
||||
"你真聪明",
|
||||
"you are",
|
||||
"you're",
|
||||
"you look",
|
||||
"you seem",
|
||||
"so smart",
|
||||
"so kind",
|
||||
]
|
||||
):
|
||||
return "loving"
|
||||
# 自我赞美
|
||||
if any(
|
||||
phrase in text
|
||||
for phrase in [
|
||||
"我真",
|
||||
"我最",
|
||||
"我太棒了",
|
||||
"我厉害",
|
||||
"我聪明",
|
||||
"我优秀",
|
||||
"i am",
|
||||
"i'm",
|
||||
"i feel",
|
||||
"so good",
|
||||
"so happy",
|
||||
]
|
||||
):
|
||||
return "cool"
|
||||
# 晚安/睡觉相关
|
||||
if any(
|
||||
phrase in text
|
||||
for phrase in [
|
||||
"睡觉",
|
||||
"晚安",
|
||||
"睡了",
|
||||
"好梦",
|
||||
"休息了",
|
||||
"去睡了",
|
||||
"sleep",
|
||||
"good night",
|
||||
"bedtime",
|
||||
"go to bed",
|
||||
]
|
||||
):
|
||||
return "sleepy"
|
||||
# 疑问句
|
||||
if has_question and not has_exclamation:
|
||||
return "thinking"
|
||||
# 强烈情感（感叹号）
|
||||
if has_exclamation and not has_question:
|
||||
# 检查是否是积极内容
|
||||
positive_words = (
|
||||
emotion_keywords["happy"]
|
||||
+ emotion_keywords["laughing"]
|
||||
+ emotion_keywords["cool"]
|
||||
)
|
||||
if any(word in text for word in positive_words):
|
||||
return "laughing"
|
||||
# 检查是否是消极内容
|
||||
negative_words = (
|
||||
emotion_keywords["angry"]
|
||||
+ emotion_keywords["sad"]
|
||||
+ emotion_keywords["crying"]
|
||||
)
|
||||
if any(word in text for word in negative_words):
|
||||
return "angry"
|
||||
return "surprised"
|
||||
# 省略号（表示犹豫或思考）
|
||||
if has_ellipsis:
|
||||
return "thinking"
|
||||
|
||||
# 关键词匹配（带权重）
|
||||
emotion_scores = {emotion: 0 for emotion in emoji_map.keys()}
|
||||
|
||||
# 给匹配到的关键词加分
|
||||
for emotion, keywords in emotion_keywords.items():
|
||||
for keyword in keywords:
|
||||
if keyword in text:
|
||||
emotion_scores[emotion] += 1
|
||||
|
||||
# 给长文本中的重复关键词额外加分
|
||||
if len(text) > 20: # 长文本
|
||||
for emotion, keywords in emotion_keywords.items():
|
||||
for keyword in keywords:
|
||||
emotion_scores[emotion] += text.count(keyword) * 0.5
|
||||
|
||||
# 根据分数选择最可能的情感
|
||||
max_score = max(emotion_scores.values())
|
||||
if max_score == 0:
|
||||
return "happy" # 默认
|
||||
|
||||
# 可能有多个情感同分，根据上下文选择最合适的
|
||||
top_emotions = [e for e, s in emotion_scores.items() if s == max_score]
|
||||
|
||||
# 如果多个情感同分，使用以下优先级
|
||||
priority_order = [
|
||||
"laughing",
|
||||
"crying",
|
||||
"angry",
|
||||
"surprised",
|
||||
"shocked", # 强烈情感优先
|
||||
"loving",
|
||||
"happy",
|
||||
"funny",
|
||||
"cool", # 积极情感
|
||||
"sad",
|
||||
"embarrassed",
|
||||
"confused", # 消极情感
|
||||
"thinking",
|
||||
"winking",
|
||||
"relaxed", # 中性情感
|
||||
"delicious",
|
||||
"kissy",
|
||||
"confident",
|
||||
"sleepy",
|
||||
"silly", # 特殊场景
|
||||
]
|
||||
|
||||
for emotion in priority_order:
|
||||
if emotion in top_emotions:
|
||||
return emotion
|
||||
|
||||
return top_emotions[0] # 如果都不在优先级列表里，返回第一个
|
||||
|
||||
|
||||
def audio_to_data(audio_file_path, is_opus=True):
|
||||
# 获取文件后缀名
|
||||
file_type = os.path.splitext(audio_file_path)[1]
|
||||
|
||||
Reference in New Issue
Block a user