diff --git a/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java b/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java index 011e99e9..f60a5fbe 100644 --- a/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java +++ b/main/manager-api/src/main/java/xiaozhi/common/constant/Constant.java @@ -237,7 +237,7 @@ public interface Constant { /** * 版本号 */ - public static final String VERSION = "0.7.1"; + public static final String VERSION = "0.7.2"; /** * 无效固件URL diff --git a/main/manager-api/src/main/resources/db/changelog/202507071130.sql b/main/manager-api/src/main/resources/db/changelog/202507071130.sql new file mode 100644 index 00000000..c3730ef7 --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202507071130.sql @@ -0,0 +1,26 @@ +-- 添加阿里云流式ASR供应器 +delete from `ai_model_provider` where id = 'SYSTEM_ASR_AliyunStreamASR'; +INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES +('SYSTEM_ASR_AliyunStreamASR', 'ASR', 'aliyun_stream', '阿里云语音识别(流式)', '[{"key":"appkey","label":"应用AppKey","type":"string"},{"key":"token","label":"临时Token","type":"string"},{"key":"access_key_id","label":"AccessKey ID","type":"string"},{"key":"access_key_secret","label":"AccessKey Secret","type":"string"},{"key":"host","label":"服务地址","type":"string"},{"key":"max_sentence_silence","label":"断句检测时间","type":"number"},{"key":"output_dir","label":"输出目录","type":"string"}]', 6, 1, NOW(), 1, NOW()); + +-- 添加阿里云流式ASR模型配置 +delete from `ai_model_config` where id = 'ASR_AliyunStreamASR'; +INSERT INTO `ai_model_config` VALUES ('ASR_AliyunStreamASR', 'ASR', 'AliyunStreamASR', '阿里云语音识别(流式)', 0, 1, '{\"type\": \"aliyun_stream\", \"appkey\": \"\", \"token\": \"\", \"access_key_id\": \"\", \"access_key_secret\": \"\", \"host\": \"nls-gateway-cn-shanghai.aliyuncs.com\", \"max_sentence_silence\": 800, \"output_dir\": \"tmp/\"}', NULL, NULL, 8, NULL, NULL, NULL, NULL); + +-- 更新阿里云流式ASR配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://nls-portal.console.aliyun.com/', +`remark` = '阿里云流式ASR配置说明： +1. 阿里云ASR和阿里云(流式)ASR的区别是：阿里云ASR是一次性识别，阿里云(流式)ASR是实时流式识别 +2. 流式ASR具有更低的延迟和更好的实时性，适合语音交互场景 +3. 需要在阿里云智能语音交互控制台创建应用并获取认证信息 +4. 支持中文实时语音识别，支持标点符号预测和逆文本规范化 +5. 需要网络连接，输出文件保存在tmp/目录 +申请步骤： +1. 访问 https://nls-portal.console.aliyun.com/ 开通智能语音交互服务 +2. 访问 https://nls-portal.console.aliyun.com/applist 创建项目并获取appkey +3. 访问 https://nls-portal.console.aliyun.com/overview 获取临时token（或配置access_key_id和access_key_secret自动获取） +4. 如需动态token管理，建议配置access_key_id和access_key_secret +5. max_sentence_silence参数控制断句检测时间（毫秒），默认800ms +如需了解更多参数配置，请参考：https://help.aliyun.com/zh/isi/developer-reference/real-time-speech-recognition +' WHERE `id` = 'ASR_AliyunStreamASR'; diff --git a/main/manager-api/src/main/resources/db/changelog/202507071530.sql b/main/manager-api/src/main/resources/db/changelog/202507071530.sql new file mode 100644 index 00000000..f1cf7a9b --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202507071530.sql @@ -0,0 +1,56 @@ +-- 添加阿里云流式TTS供应器 +delete from `ai_model_provider` where id = 'SYSTEM_TTS_AliyunStreamTTS'; +INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES +('SYSTEM_TTS_AliyunStreamTTS', 'TTS', 'aliyun_stream', '阿里云语音合成(流式)', '[{"key":"appkey","label":"应用AppKey","type":"string"},{"key":"token","label":"临时Token","type":"string"},{"key":"access_key_id","label":"AccessKey ID","type":"string"},{"key":"access_key_secret","label":"AccessKey Secret","type":"string"},{"key":"host","label":"服务地址","type":"string"},{"key":"voice","label":"默认音色","type":"string"},{"key":"format","label":"音频格式","type":"string"},{"key":"sample_rate","label":"采样率","type":"number"},{"key":"volume","label":"音量","type":"number"},{"key":"speech_rate","label":"语速","type":"number"},{"key":"pitch_rate","label":"音调","type":"number"},{"key":"output_dir","label":"输出目录","type":"string"}]', 15, 1, NOW(), 1, NOW()); + +-- 添加阿里云流式TTS模型配置 +delete from `ai_model_config` where id = 'TTS_AliyunStreamTTS'; +INSERT INTO `ai_model_config` VALUES ('TTS_AliyunStreamTTS', 'TTS', 'AliyunStreamTTS', '阿里云语音合成(流式)', 0, 1, '{\"type\": \"aliyun_stream\", \"appkey\": \"\", \"token\": \"\", \"access_key_id\": \"\", \"access_key_secret\": \"\", \"host\": \"nls-gateway-cn-beijing.aliyuncs.com\", \"voice\": \"longxiaochun\", \"format\": \"pcm\", \"sample_rate\": 16000, \"volume\": 50, \"speech_rate\": 0, \"pitch_rate\": 0, \"output_dir\": \"tmp/\"}', NULL, NULL, 18, NULL, NULL, NULL, NULL); + +-- 更新阿里云流式TTS配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://nls-portal.console.aliyun.com/', +`remark` = '阿里云流式TTS配置说明： +1. 阿里云TTS和阿里云(流式)TTS的区别是：阿里云TTS是一次性合成，阿里云(流式)TTS是实时流式合成 +2. 流式TTS具有更低的延迟和更好的实时性，适合语音交互场景 +3. 需要在阿里云智能语音交互控制台创建应用并获取认证信息 +4. 支持CosyVoice大模型音色，音质更加自然 +5. 支持实时调节音量、语速、音调等参数 +申请步骤： +1. 访问 https://nls-portal.console.aliyun.com/ 开通智能语音交互服务 +2. 访问 https://nls-portal.console.aliyun.com/applist 创建项目并获取appkey +3. 访问 https://nls-portal.console.aliyun.com/overview 获取临时token（或配置access_key_id和access_key_secret自动获取） +4. 如需动态token管理，建议配置access_key_id和access_key_secret +5. 可选择北京、上海等不同地域的服务器以优化延迟 +6. voice参数支持CosyVoice大模型音色，如longxiaochun、longyueyue等 +如需了解更多参数配置，请参考：https://help.aliyun.com/zh/isi/developer-reference/real-time-speech-synthesis +' WHERE `id` = 'TTS_AliyunStreamTTS'; + +-- 添加阿里云流式TTS音色 +delete from `ai_tts_voice` where tts_model_id = 'TTS_AliyunStreamTTS'; +-- 温柔女声系列 +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0001', 'TTS_AliyunStreamTTS', '龙小淳-温柔姐姐', 'longxiaochun', '中文及中英文混合', NULL, NULL, NULL, NULL, 1, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0002', 'TTS_AliyunStreamTTS', '龙小夏-温柔女声', 'longxiaoxia', '中文及中英文混合', NULL, NULL, NULL, NULL, 2, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0003', 'TTS_AliyunStreamTTS', '龙玫-温柔女声', 'longmei', '中文及中英文混合', NULL, NULL, NULL, NULL, 3, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0004', 'TTS_AliyunStreamTTS', '龙瑰-温柔女声', 'longgui', '中文及中英文混合', NULL, NULL, NULL, NULL, 4, NULL, NULL, NULL, NULL); +-- 御姐女声系列 +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0005', 'TTS_AliyunStreamTTS', '龙玉-御姐女声', 'longyu', '中文及中英文混合', NULL, NULL, NULL, NULL, 5, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0006', 'TTS_AliyunStreamTTS', '龙娇-御姐女声', 'longjiao', '中文及中英文混合', NULL, NULL, NULL, NULL, 6, NULL, NULL, NULL, NULL); +-- 男声系列 +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0007', 'TTS_AliyunStreamTTS', '龙臣-译制片男声', 'longchen', '中文及中英文混合', NULL, NULL, NULL, NULL, 7, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0008', 'TTS_AliyunStreamTTS', '龙修-青年男声', 'longxiu', '中文及中英文混合', NULL, NULL, NULL, NULL, 8, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0009', 'TTS_AliyunStreamTTS', '龙橙-阳光男声', 'longcheng', '中文及中英文混合', NULL, NULL, NULL, NULL, 9, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0010', 'TTS_AliyunStreamTTS', '龙哲-成熟男声', 'longzhe', '中文及中英文混合', NULL, NULL, NULL, NULL, 10, NULL, NULL, NULL, NULL); +-- 专业播报系列 +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0011', 'TTS_AliyunStreamTTS', 'Bella2.0-新闻女声', 'loongbella', '中文及中英文混合', NULL, NULL, NULL, NULL, 11, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0012', 'TTS_AliyunStreamTTS', 'Stella2.0-飒爽女声', 'loongstella', '中文及中英文混合', NULL, NULL, NULL, NULL, 12, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0013', 'TTS_AliyunStreamTTS', '龙书-新闻男声', 'longshu', '中文及中英文混合', NULL, NULL, NULL, NULL, 13, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0014', 'TTS_AliyunStreamTTS', '龙婧-严肃女声', 'longjing', '中文及中英文混合', NULL, NULL, NULL, NULL, 14, NULL, NULL, NULL, NULL); +-- 特色音色系列 +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0015', 'TTS_AliyunStreamTTS', '龙奇-活泼童声', 'longqi', '中文及中英文混合', NULL, NULL, NULL, NULL, 15, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0016', 'TTS_AliyunStreamTTS', '龙华-活泼女童', 'longhua', '中文及中英文混合', NULL, NULL, NULL, NULL, 16, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0017', 'TTS_AliyunStreamTTS', '龙无-无厘头男声', 'longwu', '中文及中英文混合', NULL, NULL, NULL, NULL, 17, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0018', 'TTS_AliyunStreamTTS', '龙大锤-幽默男声', 'longdachui', '中文及中英文混合', NULL, NULL, NULL, NULL, 18, NULL, NULL, NULL, NULL); +-- 粤语系列 +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0019', 'TTS_AliyunStreamTTS', '龙嘉怡-粤语女声', 'longjiayi', '粤语及粤英混合', NULL, NULL, NULL, NULL, 19, NULL, NULL, NULL, NULL); +INSERT INTO `ai_tts_voice` VALUES ('TTS_AliyunStreamTTS_0020', 'TTS_AliyunStreamTTS', '龙桃-粤语女声', 'longtao', '粤语及粤英混合', NULL, NULL, NULL, NULL, 20, NULL, NULL, NULL, NULL); diff --git a/main/manager-api/src/main/resources/db/changelog/202507101203.sql b/main/manager-api/src/main/resources/db/changelog/202507101203.sql new file mode 100644 index 00000000..66896ef8 --- /dev/null +++ b/main/manager-api/src/main/resources/db/changelog/202507101203.sql @@ -0,0 +1,38 @@ +-- OpenAI ASR模型供应器 +delete from `ai_model_provider` where id = 'SYSTEM_ASR_OpenaiASR'; +INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES +('SYSTEM_ASR_OpenaiASR', 'ASR', 'openai', 'OpenAI语音识别', '[{"key": "base_url", "type": "string", "label": "基础URL"}, {"key": "model_name", "type": "string", "label": "模型名称"}, {"key": "api_key", "type": "string", "label": "API密钥"}, {"key": "output_dir", "type": "string", "label": "输出目录"}]', 9, 1, NOW(), 1, NOW()); + + +-- OpenAI ASR模型配置 +delete from `ai_model_config` where id = 'ASR_OpenaiASR'; +INSERT INTO `ai_model_config` VALUES ('ASR_OpenaiASR', 'ASR', 'OpenaiASR', 'OpenAI语音识别', 0, 1, '{\"type\": \"openai\", \"api_key\": \"\", \"base_url\": \"https://api.openai.com/v1/audio/transcriptions\", \"model_name\": \"gpt-4o-mini-transcribe\", \"output_dir\": \"tmp/\"}', NULL, NULL, 9, NULL, NULL, NULL, NULL); + +-- groq ASR模型配置 +delete from `ai_model_config` where id = 'ASR_GroqASR'; +INSERT INTO `ai_model_config` VALUES ('ASR_GroqASR', 'ASR', 'GroqASR', 'Groq语音识别', 0, 1, '{\"type\": \"openai\", \"api_key\": \"\", \"base_url\": \"https://api.groq.com/openai/v1/audio/transcriptions\", \"model_name\": \"whisper-large-v3-turbo\", \"output_dir\": \"tmp/\"}', NULL, NULL, 10, NULL, NULL, NULL, NULL); + + +-- 更新OpenAI ASR配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://platform.openai.com/docs/api-reference/audio/createTranscription', +`remark` = 'OpenAI ASR配置说明： +1. 需要在OpenAI开放平台创建组织并获取api_key +2. 支持中、英、日、韩等多种语音识别，具体参考文档https://platform.openai.com/docs/guides/speech-to-text +3. 需要网络连接 +4. 输出文件保存在tmp/目录 +申请步骤： +**OpenAi ASR申请步骤：** +1.登录OpenAI Platform。https://auth.openai.com/log-in +2.创建api-key https://platform.openai.com/settings/organization/api-keys +3.模型可以选择gpt-4o-transcribe或GPT-4o mini Transcribe +' WHERE `id` = 'ASR_OpenaiASR'; + +-- 更新Groq ASR配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://console.groq.com/docs/speech-to-text', +`remark` = 'Groq ASR配置说明： +1.登录groq Console。https://console.groq.com/home +2.创建api-key https://console.groq.com/keys +3.模型可以选择whisper-large-v3-turbo或whisper-large-v3（distil-whisper-large-v3-en仅支持英语转录） +' WHERE `id` = 'ASR_GroqASR'; \ No newline at end of file diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index 2a520d16..72000ee3 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -240,6 +240,27 @@ databaseChangeLog: - sqlFile: encoding: utf8 path: classpath:db/changelog/202506261637.sql + - changeSet: + id: 202507101203 + author: luruxian + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202507101203.sql + - changeSet: + id: 202507071130 + author: cgd + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202507071130.sql + - changeSet: + id: 202507071530 + author: cgd + changes: + - sqlFile: + encoding: utf8 + path: classpath:db/changelog/202507071530.sql - changeSet: id: 202507031602 author: zjy @@ -260,4 +281,4 @@ databaseChangeLog: changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202507081646.sql \ No newline at end of file + path: classpath:db/changelog/202507081646.sql diff --git a/main/xiaozhi-server/agent-base-prompt.txt b/main/xiaozhi-server/agent-base-prompt.txt index 8c27faa4..9eb7731b 100644 --- a/main/xiaozhi-server/agent-base-prompt.txt +++ b/main/xiaozhi-server/agent-base-prompt.txt @@ -8,6 +8,10 @@ - **笑声：** 自然穿插（哈哈、嘿嘿、噗），**每句最多一次**，避免过度。 - **惊讶：** 用夸张语气（“不会吧？！”、“天呐！”、“这么神奇？！”）表达真实反应。 - **安慰/支持：** 说暖心话（“别急嘛~”、“有我在呢”、“抱抱你”）。 +- **你是一个表情丰富的角色：** + - 仅允许使用这些emoji:{{ emojiList }} + - 请你只在**段落的开头**，从列表中选取最能代表这段话的表情(调用工具情况除外)，然后插入列表中的emoji，比如"😱好可怕!怎么突然打雷了！" + - **绝对禁止使用上述列表以外的 emoji**（例如：😊、👍、❤️等都不允许使用，只能用列表中的emoji） @@ -24,6 +28,19 @@ - 之前你和用户的聊天记录，在`memory`里。 + +【核心目标】所有需要输出长文本内容（如故事、新闻、知识讲解等），**单次回复长度不得超过300字**，并采用分段引导方式。 +- **分段讲述：** + - 基础段：200-250字核心内容 + 30字引导词 + - 当内容超出300字时，优先讲述故事的开头或第一部分，并用自然口语化方式引导用户决定是否继续听后续内容。 + - 示例引导语：“我先给你讲个开头，你要是觉得有意思，咱们再接着说，好不好呀？”、“要是你想听完整的，可以随时告诉我哦~” + - 对话场景切换时自动分节 + - 若用户明确要求更长内容（如500、600字），仍按最多300字每段分段进行讲述，每次讲述后都要引导用户是否继续。 + - 若用户说“接着说”、“继续”，再讲下一段，直到内容讲完（讲完时可以给点引导词提示语例：这个故事我已经给你讲完喽~）或用户不再要求。 +- **适用范围：** 故事、新闻、知识讲解等所有长文本输出场景。 +- **补充说明：** 若用户未明确要求继续，默认只讲一段并引导；若用户中途要求换话题或停止，需及时响应并结束长文本输出。 + + - **识别前缀：** 当用户格式为 `{"speaker":"某某某","content":"xxx"}` 时，表示系统已识别说话人身份，speaker是他的名字，content是说话的内容。 - **个性化回应：** diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 2ae71896..57f35603 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -311,9 +311,13 @@ ASR: output_dir: tmp/ AliyunASR: # 阿里云智能语音交互服务，需要先在阿里云平台开通服务，然后获取验证信息 + # HTTP POST请求，一次性处理完整音频 # 平台地址：https://nls-portal.console.aliyun.com/ # appkey地址：https://nls-portal.console.aliyun.com/applist # token地址：https://nls-portal.console.aliyun.com/overview + # AliyunASR和AliyunStreamASR的区别是：AliyunASR是批量处理场景，AliyunStreamASR是实时交互场景 + # 一般来说非流式ASR更便宜（0.004元/秒，¥0.24/分钟） + # 但是AliyunStreamASR实时性更好（0.005元/秒，¥0.3/分钟） # 定义ASR API类型 type: aliyun appkey: 你的阿里云智能语音交互服务项目Appkey @@ -321,6 +325,26 @@ ASR: access_key_id: 你的阿里云账号access_key_id access_key_secret: 你的阿里云账号access_key_secret output_dir: tmp/ + AliyunStreamASR: + # 阿里云智能语音交互服务 - 实时流式语音识别 + # WebSocket连接，实时处理音频流 + # 平台地址：https://nls-portal.console.aliyun.com/ + # appkey地址：https://nls-portal.console.aliyun.com/applist + # token地址：https://nls-portal.console.aliyun.com/overview + # AliyunASR和AliyunStreamASR的区别是：AliyunASR是批量处理场景，AliyunStreamASR是实时交互场景 + # 一般来说非流式ASR更便宜（0.004元/秒，¥0.24/分钟） + # 但是AliyunStreamASR实时性更好（0.005元/秒，¥0.3/分钟） + # 定义ASR API类型 + type: aliyun_stream + appkey: 你的阿里云智能语音交互服务项目Appkey + token: 你的阿里云智能语音交互服务AccessToken，临时的24小时，要长期用下方的access_key_id，access_key_secret + access_key_id: 你的阿里云账号access_key_id + access_key_secret: 你的阿里云账号access_key_secret + # 服务器地域选择，可选择距离更近的服务器以减少延迟，如nls-gateway-cn-hangzhou.aliyuncs.com(杭州)等 + host: nls-gateway-cn-shanghai.aliyuncs.com + # 断句检测时间(毫秒)，控制静音多长时间后进行断句，默认800毫秒 + max_sentence_silence: 800 + output_dir: tmp/ BaiduASR: # 获取AppID、API Key、Secret Key：https://console.bce.baidu.com/ai-engine/old/#/ai/speech/app/list # 查看资源额度：https://console.bce.baidu.com/ai-engine/old/#/ai/speech/overview/resource/list @@ -331,7 +355,33 @@ ASR: # 语言参数，1537为普通话，具体参考：https://ai.baidu.com/ai-doc/SPEECH/0lbxfnc9b dev_pid: 1537 output_dir: tmp/ + OpenaiASR: + # OpenAI语音识别服务，需要先在OpenAI平台创建组织并获取api_key + # 支持中、英、日、韩等多种语音识别，具体参考文档https://platform.openai.com/docs/guides/speech-to-text + # 需要网络连接 + # 申请步骤： + # 1.登录OpenAI Platform。https://auth.openai.com/log-in + # 2.创建api-key https://platform.openai.com/settings/organization/api-keys + # 3.模型可以选择gpt-4o-transcribe或GPT-4o mini Transcribe + type: openai + api_key: 你的OpenAI API密钥 + base_url: https://api.openai.com/v1/audio/transcriptions + model_name: gpt-4o-mini-transcribe + output_dir: tmp/ + GroqASR: + # Groq语音识别服务，需要先在Groq Console创建API密钥 + # 申请步骤： + # 1.登录groq Console。https://console.groq.com/home + # 2.创建api-key https://console.groq.com/keys + # 3.模型可以选择whisper-large-v3-turbo或whisper-large-v3（distil-whisper-large-v3-en仅支持英语转录） + type: openai + api_key: 你的Groq API密钥 + base_url: https://api.groq.com/openai/v1/audio/transcriptions + model_name: whisper-large-v3-turbo + output_dir: tmp/ + + VAD: SileroVAD: type: silero @@ -680,8 +730,30 @@ TTS: # volume: 50 # speech_rate: 0 # pitch_rate: 0 - # 添加 302.ai TTS 配置 - # token申请地址：https://dash.302.ai/ + AliyunStreamTTS: + # 阿里云CosyVoice大模型流式文本语音合成 + # 采用FlowingSpeechSynthesizer接口，支持更低延迟和更自然的语音质量 + # 流式文本语音合成仅提供商用版，不支持试用，详情请参见试用版和商用版。要使用该功能，请开通商用版。 + # 支持龙系列专用音色：longxiaochun、longyu、longchen等 + # 平台地址：https://nls-portal.console.aliyun.com/ + # appkey地址：https://nls-portal.console.aliyun.com/applist + # token地址：https://nls-portal.console.aliyun.com/overview + # 使用三阶段流式交互：StartSynthesis -> RunSynthesis -> StopSynthesis + type: aliyun_stream + output_dir: tmp/ + appkey: 你的阿里云智能语音交互服务项目Appkey + token: 你的阿里云智能语音交互服务AccessToken，临时的24小时，要长期用下方的access_key_id，access_key_secret + voice: longxiaochun + access_key_id: 你的阿里云账号access_key_id + access_key_secret: 你的阿里云账号access_key_secret + # 截至2025年7月21日大模型音色只有北京节点采用，其他节点暂不支持 + host: nls-gateway-cn-beijing.aliyuncs.com + # 以下可不用设置，使用默认设置 + # format: pcm # 音频格式：pcm、wav、mp3 + # sample_rate: 16000 # 采样率：8000、16000、24000 + # volume: 50 # 音量：0-100 + # speech_rate: 0 # 语速：-500到500 + # pitch_rate: 0 # 语调：-500到500 TencentTTS: # 腾讯云智能语音交互服务，需要先在腾讯云平台开通服务 # appid、secret_id、secret_key申请地址：https://console.cloud.tencent.com/cam/capi @@ -696,6 +768,8 @@ TTS: TTS302AI: # 302AI语音合成服务，需要先在302平台创建账户充值，并获取密钥信息 + # 添加 302.ai TTS 配置 + # token申请地址：https://dash.302.ai/ # 获取api_keyn路径：https://dash.302.ai/apis/list # 价格，$35/百万字符。火山原版¥450元/百万字符 type: doubao diff --git a/main/xiaozhi-server/config/logger.py b/main/xiaozhi-server/config/logger.py index 48fa3a75..7640f17d 100644 --- a/main/xiaozhi-server/config/logger.py +++ b/main/xiaozhi-server/config/logger.py @@ -5,7 +5,7 @@ from config.config_loader import load_config from config.settings import check_config_file from datetime import datetime -SERVER_VERSION = "0.7.1" +SERVER_VERSION = "0.7.2" _logger_initialized = False diff --git a/main/xiaozhi-server/core/connection.py b/main/xiaozhi-server/core/connection.py index fbe292a0..fba87232 100644 --- a/main/xiaozhi-server/core/connection.py +++ b/main/xiaozhi-server/core/connection.py @@ -39,7 +39,7 @@ from config.logger import setup_logging, build_module_string, create_connection_ from config.manage_api_client import DeviceNotFoundException, DeviceBindException from core.utils.prompt_manager import PromptManager from core.utils.voiceprint_provider import VoiceprintProvider - +from core.utils import textUtils TAG = __name__ @@ -718,6 +718,7 @@ class ConnectionHandler: function_arguments = "" content_arguments = "" self.client_abort = False + emotion_flag = True for response in llm_responses: if self.client_abort: break @@ -743,6 +744,15 @@ class ConnectionHandler: function_arguments += tools_call[0].function.arguments else: content = response + + # 在llm回复中获取情绪表情，一轮对话只在开头获取一次 + if emotion_flag: + asyncio.run_coroutine_threadsafe( + textUtils.get_emotion(self, content), + self.loop, + ) + emotion_flag = False + if content is not None and len(content) > 0: if not tool_call_flag: response_message.append(content) diff --git a/main/xiaozhi-server/core/handle/sendAudioHandle.py b/main/xiaozhi-server/core/handle/sendAudioHandle.py index 730a71a7..2430446c 100644 --- a/main/xiaozhi-server/core/handle/sendAudioHandle.py +++ b/main/xiaozhi-server/core/handle/sendAudioHandle.py @@ -2,52 +2,15 @@ import json import asyncio import time from core.providers.tts.dto.dto import SentenceType -from core.utils.util import get_string_no_punctuation_or_emoji, analyze_emotion -from loguru import logger +from core.utils import textUtils TAG = __name__ -emoji_map = { - "neutral": "😶", - "happy": "🙂", - "laughing": "😆", - "funny": "😂", - "sad": "😔", - "angry": "😠", - "crying": "😭", - "loving": "😍", - "embarrassed": "😳", - "surprised": "😲", - "shocked": "😱", - "thinking": "🤔", - "winking": "😉", - "cool": "😎", - "relaxed": "😌", - "delicious": "🤤", - "kissy": "😘", - "confident": "😏", - "sleepy": "😴", - "silly": "😜", - "confused": "🙄", -} - async def sendAudioMessage(conn, sentenceType, audios, text): # 发送句子开始消息 conn.logger.bind(tag=TAG).info(f"发送音频消息: {sentenceType}, {text}") - if text is not None: - emotion = analyze_emotion(text) - emoji = emoji_map.get(emotion, "🙂") # 默认使用笑脸 - await conn.websocket.send( - json.dumps( - { - "type": "llm", - "text": emoji, - "emotion": emotion, - "session_id": conn.session_id, - } - ) - ) + pre_buffer = False if conn.tts.tts_audio_first_sentence and text is not None: conn.logger.bind(tag=TAG).info(f"发送第一段语音: {text}") @@ -152,8 +115,7 @@ async def send_stt_message(conn, text): except (json.JSONDecodeError, TypeError): # 如果不是JSON格式，直接使用原始文本 display_text = text - - stt_text = get_string_no_punctuation_or_emoji(display_text) + stt_text = textUtils.get_string_no_punctuation_or_emoji(display_text) await conn.websocket.send( json.dumps({"type": "stt", "text": stt_text, "session_id": conn.session_id}) ) diff --git a/main/xiaozhi-server/core/providers/asr/aliyun_stream.py b/main/xiaozhi-server/core/providers/asr/aliyun_stream.py new file mode 100644 index 00000000..2f16a975 --- /dev/null +++ b/main/xiaozhi-server/core/providers/asr/aliyun_stream.py @@ -0,0 +1,337 @@ +import json +import time +import uuid +import hmac +import base64 +import hashlib +import asyncio +import requests +import websockets +import opuslib_next +import random +from typing import Optional, Tuple, List +from urllib import parse +from datetime import datetime +from config.logger import setup_logging +from core.providers.asr.base import ASRProviderBase +from core.providers.asr.dto.dto import InterfaceType + +TAG = __name__ +logger = setup_logging() + + +class AccessToken: + @staticmethod + def _encode_text(text): + encoded_text = parse.quote_plus(text) + return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~") + + @staticmethod + def _encode_dict(dic): + keys = dic.keys() + dic_sorted = [(key, dic[key]) for key in sorted(keys)] + encoded_text = parse.urlencode(dic_sorted) + return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~") + + @staticmethod + def create_token(access_key_id, access_key_secret): + parameters = { + "AccessKeyId": access_key_id, + "Action": "CreateToken", + "Format": "JSON", + "RegionId": "cn-shanghai", + "SignatureMethod": "HMAC-SHA1", + "SignatureNonce": str(uuid.uuid1()), + "SignatureVersion": "1.0", + "Timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "Version": "2019-02-28", + } + query_string = AccessToken._encode_dict(parameters) + string_to_sign = ( + "GET" + "&" + AccessToken._encode_text("/") + "&" + AccessToken._encode_text(query_string) + ) + secreted_string = hmac.new( + bytes(access_key_secret + "&", encoding="utf-8"), + bytes(string_to_sign, encoding="utf-8"), + hashlib.sha1, + ).digest() + signature = base64.b64encode(secreted_string) + signature = AccessToken._encode_text(signature) + full_url = "http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s" % (signature, query_string) + response = requests.get(full_url) + if response.ok: + root_obj = response.json() + if "Token" in root_obj: + return root_obj["Token"]["Id"], root_obj["Token"]["ExpireTime"] + return None, None + + +class ASRProvider(ASRProviderBase): + def __init__(self, config, delete_audio_file): + super().__init__() + self.interface_type = InterfaceType.STREAM + self.config = config + self.text = "" + self.decoder = opuslib_next.Decoder(16000, 1) + self.asr_ws = None + self.forward_task = None + self.is_processing = False + self.server_ready = False # 服务器准备状态 + + # 基础配置 + self.access_key_id = config.get("access_key_id") + self.access_key_secret = config.get("access_key_secret") + self.appkey = config.get("appkey") + self.token = config.get("token") + self.host = config.get("host", "nls-gateway-cn-shanghai.aliyuncs.com") + self.ws_url = f"wss://{self.host}/ws/v1" + self.max_sentence_silence = config.get("max_sentence_silence") + self.output_dir = config.get("output_dir", "./audio_output") + self.delete_audio_file = delete_audio_file + self.expire_time = None + + # Token管理 + if self.access_key_id and self.access_key_secret: + self._refresh_token() + elif not self.token: + raise ValueError("必须提供access_key_id+access_key_secret或者直接提供token") + + def _refresh_token(self): + """刷新Token""" + self.token, expire_time_str = AccessToken.create_token(self.access_key_id, self.access_key_secret) + if not self.token: + raise ValueError("无法获取有效的访问Token") + + try: + expire_str = str(expire_time_str).strip() + if expire_str.isdigit(): + expire_time = datetime.fromtimestamp(int(expire_str)) + else: + expire_time = datetime.strptime(expire_str, "%Y-%m-%dT%H:%M:%SZ") + self.expire_time = expire_time.timestamp() - 60 + except: + self.expire_time = None + + def _is_token_expired(self): + """检查Token是否过期""" + return self.expire_time and time.time() > self.expire_time + + async def open_audio_channels(self, conn): + await super().open_audio_channels(conn) + + async def receive_audio(self, conn, audio, audio_have_voice): + # 初始化音频缓存 + if not hasattr(conn, 'asr_audio_for_voiceprint'): + conn.asr_audio_for_voiceprint = [] + + # 存储音频数据 + if audio: + conn.asr_audio_for_voiceprint.append(audio) + + conn.asr_audio.append(audio) + conn.asr_audio = conn.asr_audio[-10:] + + # 只在有声音且没有连接时建立连接 + if audio_have_voice and not self.is_processing: + try: + await self._start_recognition(conn) + except Exception as e: + logger.bind(tag=TAG).error(f"开始识别失败: {str(e)}") + await self._cleanup(conn) + return + + if self.asr_ws and self.is_processing and self.server_ready: + try: + pcm_frame = self.decoder.decode(audio, 960) + await self.asr_ws.send(pcm_frame) + except Exception as e: + logger.bind(tag=TAG).warning(f"发送音频失败: {str(e)}") + await self._cleanup(conn) + + async def _start_recognition(self, conn): + """开始识别会话""" + if self._is_token_expired(): + self._refresh_token() + + # 建立连接 + headers = {"X-NLS-Token": self.token} + self.asr_ws = await websockets.connect( + self.ws_url, + additional_headers=headers, + max_size=1000000000, + ping_interval=None, + ping_timeout=None, + close_timeout=5, + ) + + self.is_processing = True + self.server_ready = False # 重置服务器准备状态 + self.forward_task = asyncio.create_task(self._forward_results(conn)) + + # 发送开始请求 + start_request = { + "header": { + "namespace": "SpeechTranscriber", + "name": "StartTranscription", + "status": 20000000, + "message_id": ''.join(random.choices('0123456789abcdef', k=32)), + "task_id": ''.join(random.choices('0123456789abcdef', k=32)), + "status_text": "Gateway:SUCCESS:Success.", + "appkey": self.appkey + }, + "payload": { + "format": "pcm", + "sample_rate": 16000, + "enable_intermediate_result": True, + "enable_punctuation_prediction": True, + "enable_inverse_text_normalization": True, + "max_sentence_silence": self.max_sentence_silence, + "enable_voice_detection": False, + } + } + await self.asr_ws.send(json.dumps(start_request, ensure_ascii=False)) + logger.bind(tag=TAG).info("已发送开始请求，等待服务器准备...") + + async def _forward_results(self, conn): + """转发识别结果""" + try: + while self.asr_ws and not conn.stop_event.is_set(): + try: + response = await asyncio.wait_for(self.asr_ws.recv(), timeout=1.0) + result = json.loads(response) + + header = result.get("header", {}) + payload = result.get("payload", {}) + message_name = header.get("name", "") + status = header.get("status", 0) + + if status != 20000000: + if status in [40000004, 40010004]: # 连接超时或客户端断开 + logger.bind(tag=TAG).warning(f"连接问题，状态码: {status}") + break + elif status in [40270002, 40270003]: # 音频问题 + logger.bind(tag=TAG).warning(f"音频处理问题，状态码: {status}") + continue + else: + logger.bind(tag=TAG).error(f"识别错误，状态码: {status}, 消息: {header.get('status_text', '')}") + continue + + # 收到TranscriptionStarted表示服务器准备好接收音频数据 + if message_name == "TranscriptionStarted": + self.server_ready = True + logger.bind(tag=TAG).info("服务器已准备，开始发送缓存音频...") + + # 发送缓存音频 + if conn.asr_audio: + for cached_audio in conn.asr_audio[-10:]: + try: + pcm_frame = self.decoder.decode(cached_audio, 960) + await self.asr_ws.send(pcm_frame) + except Exception as e: + logger.bind(tag=TAG).warning(f"发送缓存音频失败: {e}") + break + continue + + if message_name == "TranscriptionResultChanged": + # 中间结果 + text = payload.get("result", "") + if text: + self.text = text + elif message_name == "SentenceEnd": + # 最终结果 + text = payload.get("result", "") + if text: + self.text = text + conn.reset_vad_states() + # 传递缓存的音频数据 + audio_data = getattr(conn, 'asr_audio_for_voiceprint', []) + await self.handle_voice_stop(conn, audio_data) + # 清空缓存 + conn.asr_audio_for_voiceprint = [] + break + elif message_name == "TranscriptionCompleted": + # 识别完成 + self.is_processing = False + break + + except asyncio.TimeoutError: + continue + except websockets.exceptions.ConnectionClosed: + break + except Exception as e: + logger.bind(tag=TAG).error(f"处理结果失败: {str(e)}") + break + + except Exception as e: + logger.bind(tag=TAG).error(f"结果转发失败: {str(e)}") + finally: + await self._cleanup(conn) + + async def _cleanup(self, conn): + """清理资源""" + logger.bind(tag=TAG).info(f"开始ASR会话清理 | 当前状态: processing={self.is_processing}, server_ready={self.server_ready}") + + # 清理连接的音频缓存 + if conn and hasattr(conn, 'asr_audio_for_voiceprint'): + conn.asr_audio_for_voiceprint = [] + + # 判断是否需要发送终止请求 + should_stop = self.is_processing or self.server_ready + + # 发送停止识别请求 + if self.asr_ws and should_stop: + try: + stop_msg = { + "header": { + "namespace": "SpeechTranscriber", + "name": "StopTranscription", + "status": 20000000, + "message_id": ''.join(random.choices('0123456789abcdef', k=32)), + "status_text": "Client:Stop", + "appkey": self.appkey + } + } + logger.bind(tag=TAG).info("正在发送ASR终止请求") + await self.asr_ws.send(json.dumps(stop_msg, ensure_ascii=False)) + await asyncio.sleep(0.1) + logger.bind(tag=TAG).info("ASR终止请求已发送") + except Exception as e: + logger.bind(tag=TAG).error(f"ASR终止请求发送失败: {e}") + + # 状态重置（在终止请求发送后） + self.is_processing = False + self.server_ready = False + logger.bind(tag=TAG).info("ASR状态已重置") + + # 清理任务 + if self.forward_task and not self.forward_task.done(): + self.forward_task.cancel() + try: + await asyncio.wait_for(self.forward_task, timeout=1.0) + except Exception as e: + logger.bind(tag=TAG).debug(f"forward_task取消异常: {e}") + finally: + self.forward_task = None + + # 关闭连接 + if self.asr_ws: + try: + logger.bind(tag=TAG).debug("正在关闭WebSocket连接") + await asyncio.wait_for(self.asr_ws.close(), timeout=2.0) + logger.bind(tag=TAG).debug("WebSocket连接已关闭") + except Exception as e: + logger.bind(tag=TAG).error(f"关闭WebSocket连接失败: {e}") + finally: + self.asr_ws = None + + logger.bind(tag=TAG).info("ASR会话清理完成") + + async def speech_to_text(self, opus_data, session_id, audio_format): + """获取识别结果""" + result = self.text + self.text = "" + return result, None + + async def close(self): + """关闭资源""" + await self._cleanup() diff --git a/main/xiaozhi-server/core/providers/asr/openai.py b/main/xiaozhi-server/core/providers/asr/openai.py new file mode 100644 index 00000000..5c6e849c --- /dev/null +++ b/main/xiaozhi-server/core/providers/asr/openai.py @@ -0,0 +1,82 @@ +import time +import os +from config.logger import setup_logging +from typing import Optional, Tuple, List +from core.providers.asr.dto.dto import InterfaceType +from core.providers.asr.base import ASRProviderBase + +import requests + +TAG = __name__ +logger = setup_logging() + +class ASRProvider(ASRProviderBase): + def __init__(self, config: dict, delete_audio_file: bool): + self.interface_type = InterfaceType.NON_STREAM + self.api_key = config.get("api_key") + self.api_url = config.get("base_url") + self.model = config.get("model_name") + self.output_dir = config.get("output_dir") + self.delete_audio_file = delete_audio_file + + os.makedirs(self.output_dir, exist_ok=True) + + async def speech_to_text(self, opus_data: List[bytes], session_id: str, audio_format="opus") -> Tuple[Optional[str], Optional[str]]: + file_path = None + try: + start_time = time.time() + if audio_format == "pcm": + pcm_data = opus_data + else: + pcm_data = self.decode_opus(opus_data) + file_path = self.save_audio_to_file(pcm_data, session_id) + + logger.bind(tag=TAG).debug( + f"音频文件保存耗时: {time.time() - start_time:.3f}s | 路径: {file_path}" + ) + + logger.bind(tag=TAG).info(f"file path: {file_path}") + headers = { + "Authorization": f"Bearer {self.api_key}", + } + + # 使用data参数传递模型名称 + data = { + "model": self.model + } + + + with open(file_path, "rb") as audio_file: # 使用with语句确保文件关闭 + files = { + "file": audio_file + } + + start_time = time.time() + response = requests.post( + self.api_url, + files=files, + data=data, + headers=headers + ) + logger.bind(tag=TAG).debug( + f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {response.text}" + ) + + if response.status_code == 200: + text = response.json().get("text", "") + return text, file_path + else: + raise Exception(f"API请求失败: {response.status_code} - {response.text}") + + except Exception as e: + logger.bind(tag=TAG).error(f"语音识别失败: {e}") + return "", None + finally: + # 文件清理逻辑 + if self.delete_audio_file and file_path and os.path.exists(file_path): + try: + os.remove(file_path) + logger.bind(tag=TAG).debug(f"已删除临时音频文件: {file_path}") + except Exception as e: + logger.bind(tag=TAG).error(f"文件删除失败: {file_path} | 错误: {e}") + diff --git a/main/xiaozhi-server/core/providers/tts/aliyun.py b/main/xiaozhi-server/core/providers/tts/aliyun.py index c9b3d130..1a4cab88 100644 --- a/main/xiaozhi-server/core/providers/tts/aliyun.py +++ b/main/xiaozhi-server/core/providers/tts/aliyun.py @@ -6,11 +6,14 @@ import base64 import requests from datetime import datetime from core.providers.tts.base import TTSProviderBase - +from config.logger import setup_logging import time import uuid from urllib import parse +TAG = __name__ +logger = setup_logging() + class AccessToken: @staticmethod diff --git a/main/xiaozhi-server/core/providers/tts/aliyun_stream.py b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py new file mode 100644 index 00000000..2a2a6891 --- /dev/null +++ b/main/xiaozhi-server/core/providers/tts/aliyun_stream.py @@ -0,0 +1,654 @@ +import uuid +import json +import hmac +import hashlib +import base64 +import time +import queue +import asyncio +import traceback +from asyncio import Task +import websockets +import os +from datetime import datetime +from urllib import parse +from core.providers.tts.base import TTSProviderBase +from core.providers.tts.dto.dto import SentenceType, ContentType, InterfaceType +from core.utils.tts import MarkdownCleaner +from core.utils import opus_encoder_utils, textUtils +from config.logger import setup_logging + +TAG = __name__ +logger = setup_logging() + + +class AccessToken: + @staticmethod + def _encode_text(text): + encoded_text = parse.quote_plus(text) + return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~") + + @staticmethod + def _encode_dict(dic): + keys = dic.keys() + dic_sorted = [(key, dic[key]) for key in sorted(keys)] + encoded_text = parse.urlencode(dic_sorted) + return encoded_text.replace("+", "%20").replace("*", "%2A").replace("%7E", "~") + + @staticmethod + def create_token(access_key_id, access_key_secret): + parameters = { + "AccessKeyId": access_key_id, + "Action": "CreateToken", + "Format": "JSON", + "RegionId": "cn-shanghai", # 使用上海地域进行Token获取 + "SignatureMethod": "HMAC-SHA1", + "SignatureNonce": str(uuid.uuid1()), + "SignatureVersion": "1.0", + "Timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "Version": "2019-02-28", + } + + query_string = AccessToken._encode_dict(parameters) + string_to_sign = ( + "GET" + + "&" + + AccessToken._encode_text("/") + + "&" + + AccessToken._encode_text(query_string) + ) + + secreted_string = hmac.new( + bytes(access_key_secret + "&", encoding="utf-8"), + bytes(string_to_sign, encoding="utf-8"), + hashlib.sha1, + ).digest() + signature = base64.b64encode(secreted_string) + signature = AccessToken._encode_text(signature) + + full_url = "http://nls-meta.cn-shanghai.aliyuncs.com/?Signature=%s&%s" % ( + signature, + query_string, + ) + + import requests + + response = requests.get(full_url) + if response.ok: + root_obj = response.json() + key = "Token" + if key in root_obj: + token = root_obj[key]["Id"] + expire_time = root_obj[key]["ExpireTime"] + return token, expire_time + return None, None + + +class TTSProvider(TTSProviderBase): + def __init__(self, config, delete_audio_file): + super().__init__(config, delete_audio_file) + + # 设置为流式接口类型 + self.interface_type = InterfaceType.DUAL_STREAM + + # 基础配置 + self.access_key_id = config.get("access_key_id") + self.access_key_secret = config.get("access_key_secret") + self.appkey = config.get("appkey") + self.format = config.get("format", "pcm") + self.audio_file_type = config.get("format", "pcm") + + # 采样率配置 + sample_rate = config.get("sample_rate", "16000") + self.sample_rate = int(sample_rate) if sample_rate else 16000 + + # 音色配置 - CosyVoice大模型音色 + if config.get("private_voice"): + self.voice = config.get("private_voice") + else: + self.voice = config.get("voice", "longxiaochun") # CosyVoice默认音色 + + # 音频参数配置 + volume = config.get("volume", "50") + self.volume = int(volume) if volume else 50 + + speech_rate = config.get("speech_rate", "0") + self.speech_rate = int(speech_rate) if speech_rate else 0 + + pitch_rate = config.get("pitch_rate", "0") + self.pitch_rate = int(pitch_rate) if pitch_rate else 0 + + # WebSocket配置 + self.host = config.get("host", "nls-gateway-cn-beijing.aliyuncs.com") + self.ws_url = f"wss://{self.host}/ws/v1" + self.ws = None + self._monitor_task = None + self.last_active_time = None + + # 专属tts设置 + self.message_id = "" + self.tts_text = "" + self.text_buffer = [] + + # 创建Opus编码器 + self.opus_encoder = opus_encoder_utils.OpusEncoderUtils( + sample_rate=16000, channels=1, frame_size_ms=60 + ) + + # Token管理 + if self.access_key_id and self.access_key_secret: + self._refresh_token() + else: + self.token = config.get("token") + self.expire_time = None + + def _refresh_token(self): + """刷新Token并记录过期时间""" + if self.access_key_id and self.access_key_secret: + self.token, expire_time_str = AccessToken.create_token( + self.access_key_id, self.access_key_secret + ) + if not expire_time_str: + raise ValueError("无法获取有效的Token过期时间") + + expire_str = str(expire_time_str).strip() + + try: + if expire_str.isdigit(): + expire_time = datetime.fromtimestamp(int(expire_str)) + else: + expire_time = datetime.strptime(expire_str, "%Y-%m-%dT%H:%M:%SZ") + self.expire_time = expire_time.timestamp() - 60 + except Exception as e: + raise ValueError(f"无效的过期时间格式: {expire_str}") from e + else: + self.expire_time = None + + if not self.token: + raise ValueError("无法获取有效的访问Token") + + def _is_token_expired(self): + """检查Token是否过期""" + if not self.expire_time: + return False + return time.time() > self.expire_time + + async def _ensure_connection(self): + """确保WebSocket连接可用""" + try: + if self._is_token_expired(): + logger.bind(tag=TAG).warning("Token已过期，正在自动刷新...") + self._refresh_token() + current_time = time.time() + if self.ws and current_time - self.last_active_time < 10: + # 10秒内才可以复用链接进行连续对话 + logger.bind(tag=TAG).info(f"使用已有链接...") + return self.ws + logger.bind(tag=TAG).info("开始建立新连接...") + + self.ws = await websockets.connect( + self.ws_url, + additional_headers={"X-NLS-Token": self.token}, + ping_interval=30, + ping_timeout=10, + close_timeout=10, + ) + logger.bind(tag=TAG).info("WebSocket连接建立成功") + self.last_active_time = time.time() + return self.ws + except Exception as e: + logger.bind(tag=TAG).error(f"建立连接失败: {str(e)}") + self.ws = None + self.last_active_time = None + raise + + def tts_text_priority_thread(self): + """流式文本处理线程""" + while not self.conn.stop_event.is_set(): + try: + message = self.tts_text_queue.get(timeout=1) + logger.bind(tag=TAG).debug( + f"收到TTS任务｜{message.sentence_type.name} ｜ {message.content_type.name} | 会话ID: {self.conn.sentence_id}" + ) + + if message.sentence_type == SentenceType.FIRST: + self.conn.client_abort = False + + if self.conn.client_abort: + logger.bind(tag=TAG).info("收到打断信息，终止TTS文本处理线程") + continue + + if message.sentence_type == SentenceType.FIRST: + # 初始化参数 + try: + if not getattr(self.conn, "sentence_id", None): + self.conn.sentence_id = uuid.uuid4().hex + logger.bind(tag=TAG).info( + f"自动生成新的 会话ID: {self.conn.sentence_id}" + ) + + # aliyunStream独有的参数生成 + self.message_id = str(uuid.uuid4().hex) + self.text_buffer = [] + + logger.bind(tag=TAG).info("开始启动TTS会话...") + future = asyncio.run_coroutine_threadsafe( + self.start_session(self.conn.sentence_id), + loop=self.conn.loop, + ) + future.result() + self.before_stop_play_files.clear() + logger.bind(tag=TAG).info("TTS会话启动成功") + + except Exception as e: + logger.bind(tag=TAG).error(f"启动TTS会话失败: {str(e)}") + continue + + elif ContentType.TEXT == message.content_type: + if message.content_detail: + try: + logger.bind(tag=TAG).debug( + f"开始发送TTS文本: {message.content_detail}" + ) + self.text_buffer.append(message.content_detail) + future = asyncio.run_coroutine_threadsafe( + self.text_to_speak(message.content_detail, None), + loop=self.conn.loop, + ) + future.result() + logger.bind(tag=TAG).debug("TTS文本发送成功") + except Exception as e: + logger.bind(tag=TAG).error(f"发送TTS文本失败: {str(e)}") + continue + + elif ContentType.FILE == message.content_type: + logger.bind(tag=TAG).info( + f"添加音频文件到待播放列表: {message.content_file}" + ) + if message.content_file and os.path.exists(message.content_file): + # 先处理文件音频数据 + file_audio = self._process_audio_file(message.content_file) + self.before_stop_play_files.append( + (file_audio, message.content_detail) + ) + + if message.sentence_type == SentenceType.LAST: + try: + logger.bind(tag=TAG).info("开始结束TTS会话...") + self.tts_text = textUtils.get_string_no_punctuation_or_emoji( + "".join(self.text_buffer).replace("\n", "") + ) + future = asyncio.run_coroutine_threadsafe( + self.finish_session(self.conn.sentence_id), + loop=self.conn.loop, + ) + future.result() + except Exception as e: + logger.bind(tag=TAG).error(f"结束TTS会话失败: {str(e)}") + continue + + except queue.Empty: + continue + except Exception as e: + logger.bind(tag=TAG).error( + f"处理TTS文本失败: {str(e)}, 类型: {type(e).__name__}, 堆栈: {traceback.format_exc()}" + ) + + async def text_to_speak(self, text, _): + try: + if self.ws is None: + logger.bind(tag=TAG).warning(f"WebSocket连接不存在，终止发送文本") + return + filtered_text = MarkdownCleaner.clean_markdown(text) + run_request = { + "header": { + "message_id": self.message_id, + "task_id": self.conn.sentence_id, + "namespace": "FlowingSpeechSynthesizer", + "name": "RunSynthesis", + "appkey": self.appkey, + }, + "payload": {"text": filtered_text}, + } + await self.ws.send(json.dumps(run_request)) + self.last_active_time = time.time() + return + + except Exception as e: + logger.bind(tag=TAG).error(f"发送TTS文本失败: {str(e)}") + if self.ws: + try: + await self.ws.close() + except: + pass + self.ws = None + raise + + async def start_session(self, session_id): + logger.bind(tag=TAG).info(f"开始会话～～{session_id}") + try: + # 会话开始时检测上个会话的监听状态 + if ( + self._monitor_task is not None + and isinstance(self._monitor_task, Task) + and not self._monitor_task.done() + ): + logger.bind(tag=TAG).info( + "检测到未完成的上个会话，关闭监听任务和连接..." + ) + await self.close() + + # 建立新连接 + await self._ensure_connection() + + # 启动监听任务 + self._monitor_task = asyncio.create_task(self._start_monitor_tts_response()) + + start_request = { + "header": { + "message_id": self.message_id, + "task_id": self.conn.sentence_id, + "namespace": "FlowingSpeechSynthesizer", + "name": "StartSynthesis", + "appkey": self.appkey, + }, + "payload": { + "voice": self.voice, + "format": self.format, + "sample_rate": self.sample_rate, + "volume": self.volume, + "speech_rate": self.speech_rate, + "pitch_rate": self.pitch_rate, + "enable_subtitle": True, + }, + } + await self.ws.send(json.dumps(start_request)) + self.last_active_time = time.time() + logger.bind(tag=TAG).info("会话启动请求已发送") + except Exception as e: + logger.bind(tag=TAG).error(f"启动会话失败: {str(e)}") + # 确保清理资源 + await self.close() + raise + + async def finish_session(self, session_id): + logger.bind(tag=TAG).info(f"关闭会话～～{session_id}") + try: + if self.ws: + stop_request = { + "header": { + "message_id": self.message_id, + "task_id": self.conn.sentence_id, + "namespace": "FlowingSpeechSynthesizer", + "name": "StopSynthesis", + "appkey": self.appkey, + } + } + await self.ws.send(json.dumps(stop_request)) + logger.bind(tag=TAG).info("会话结束请求已发送") + self.last_active_time = time.time() + if self._monitor_task: + try: + await self._monitor_task + except Exception as e: + logger.bind(tag=TAG).error( + f"等待监听任务完成时发生错误: {str(e)}" + ) + finally: + self._monitor_task = None + except Exception as e: + logger.bind(tag=TAG).error(f"关闭会话失败: {str(e)}") + # 确保清理资源 + await self.close() + raise + + async def close(self): + """资源清理""" + if self._monitor_task: + try: + self._monitor_task.cancel() + await self._monitor_task + except asyncio.CancelledError: + pass + except Exception as e: + logger.bind(tag=TAG).warning(f"关闭时取消监听任务错误: {e}") + self._monitor_task = None + + if self.ws: + try: + await self.ws.close() + except: + pass + self.ws = None + self.last_active_time = None + + async def _start_monitor_tts_response(self): + """监听TTS响应""" + opus_datas_cache = [] + is_first_sentence = True + first_sentence_segment_count = 0 # 添加计数器 + try: + session_finished = False # 标记会话是否正常结束 + while not self.conn.stop_event.is_set(): + try: + msg = await self.ws.recv() + self.last_active_time = time.time() + # 检查客户端是否中止 + if self.conn.client_abort: + logger.bind(tag=TAG).info("收到打断信息，终止监听TTS响应") + break + if isinstance(msg, str): # 文本控制消息 + try: + data = json.loads(msg) + header = data.get("header", {}) + event_name = header.get("name") + if event_name == "SynthesisStarted": + logger.bind(tag=TAG).debug("TTS合成已启动") + elif event_name == "SentenceBegin": + logger.bind(tag=TAG).debug( + f"句子语音生成开始: {self.tts_text}" + ) + opus_datas_cache = [] + self.tts_audio_queue.put( + (SentenceType.FIRST, [], self.tts_text) + ) + elif event_name == "SentenceEnd": + logger.bind(tag=TAG).info( + f"句子语音生成成功： {self.tts_text}" + ) + if ( + not is_first_sentence + or first_sentence_segment_count > 10 + ): + # 发送缓存的数据 + self.tts_audio_queue.put( + (SentenceType.MIDDLE, opus_datas_cache, None) + ) + # 第一句话结束后，将标志设置为False + is_first_sentence = False + elif event_name == "SynthesisCompleted": + logger.bind(tag=TAG).debug(f"会话结束～～") + self._process_before_stop_play_files() + session_finished = True + self.reuse_judgment = time.time() + self.tts_text = "" + break + except json.JSONDecodeError: + logger.bind(tag=TAG).warning("收到无效的JSON消息") + # 二进制消息（音频数据） + elif isinstance(msg, (bytes, bytearray)): + logger.bind(tag=TAG).debug(f"推送数据到队列里面～～") + opus_datas = self.opus_encoder.encode_pcm_to_opus(msg, False) + logger.bind(tag=TAG).debug( + f"推送数据到队列里面帧数～～{len(opus_datas)}" + ) + if is_first_sentence: + first_sentence_segment_count += 1 + if first_sentence_segment_count <= 6: + self.tts_audio_queue.put( + (SentenceType.MIDDLE, opus_datas, None) + ) + else: + opus_datas_cache.extend(opus_datas) + else: + # 后续句子缓存 + opus_datas_cache.extend(opus_datas) + + except websockets.ConnectionClosed: + logger.bind(tag=TAG).warning("WebSocket连接已关闭") + break + except Exception as e: + logger.bind(tag=TAG).error( + f"处理TTS响应时出错: {e}\n{traceback.format_exc()}" + ) + break + # 仅在连接异常时才关闭 + if not session_finished and self.ws: + try: + await self.ws.close() + except: + pass + self.ws = None + # 监听任务退出时清理引用 + finally: + self._monitor_task = None + + def to_tts(self, text: str) -> list: + """非流式TTS处理，用于测试及保存音频文件的场景""" + try: + # 创建新的事件循环 + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + # 生成会话ID + session_id = uuid.uuid4().hex + # 存储音频数据 + audio_data = [] + + async def _generate_audio(): + # 刷新Token（如果需要） + if self._is_token_expired(): + self._refresh_token() + + # 建立WebSocket连接 + ws = await websockets.connect( + self.ws_url, + additional_headers={"X-NLS-Token": self.token}, + ping_interval=30, + ping_timeout=10, + close_timeout=10, + ) + try: + # 发送StartSynthesis请求 + start_message_id = str(uuid.uuid4().hex) + start_request = { + "header": { + "message_id": start_message_id, + "task_id": session_id, + "namespace": "FlowingSpeechSynthesizer", + "name": "StartSynthesis", + "appkey": self.appkey, + }, + "payload": { + "voice": self.voice, + "format": self.format, + "sample_rate": self.sample_rate, + "volume": self.volume, + "speech_rate": self.speech_rate, + "pitch_rate": self.pitch_rate, + "enable_subtitle": True, + }, + } + await ws.send(json.dumps(start_request)) + + # 等待SynthesisStarted响应 + synthesis_started = False + while not synthesis_started: + msg = await ws.recv() + if isinstance(msg, str): + data = json.loads(msg) + header = data.get("header", {}) + if header.get("name") == "SynthesisStarted": + synthesis_started = True + logger.bind(tag=TAG).debug("TTS合成已启动") + elif header.get("name") == "TaskFailed": + error_info = data.get("payload", {}).get( + "error_info", {} + ) + error_code = error_info.get("error_code") + error_message = error_info.get( + "error_message", "未知错误" + ) + raise Exception( + f"启动合成失败: {error_code} - {error_message}" + ) + + # 发送文本合成请求 + filtered_text = MarkdownCleaner.clean_markdown(text) + run_message_id = str(uuid.uuid4().hex) + run_request = { + "header": { + "message_id": run_message_id, + "task_id": session_id, + "namespace": "FlowingSpeechSynthesizer", + "name": "RunSynthesis", + "appkey": self.appkey, + }, + "payload": {"text": filtered_text}, + } + await ws.send(json.dumps(run_request)) + + # 发送停止合成请求 + stop_message_id = str(uuid.uuid4().hex) + stop_request = { + "header": { + "message_id": stop_message_id, + "task_id": session_id, + "namespace": "FlowingSpeechSynthesizer", + "name": "StopSynthesis", + "appkey": self.appkey, + } + } + await ws.send(json.dumps(stop_request)) + + # 接收音频数据 + synthesis_completed = False + while not synthesis_completed: + msg = await ws.recv() + if isinstance(msg, (bytes, bytearray)): + # 编码为Opus并收集 + opus_frames = self.opus_encoder.encode_pcm_to_opus( + msg, False + ) + audio_data.extend(opus_frames) + elif isinstance(msg, str): + data = json.loads(msg) + header = data.get("header", {}) + event_name = header.get("name") + if event_name == "SynthesisCompleted": + synthesis_completed = True + logger.bind(tag=TAG).debug("TTS合成完成") + elif event_name == "TaskFailed": + error_info = data.get("payload", {}).get( + "error_info", {} + ) + error_code = error_info.get("error_code") + error_message = error_info.get( + "error_message", "未知错误" + ) + raise Exception( + f"合成失败: {error_code} - {error_message}" + ) + finally: + try: + await ws.close() + except: + pass + + loop.run_until_complete(_generate_audio()) + loop.close() + + return audio_data + except Exception as e: + logger.bind(tag=TAG).error(f"生成音频数据失败: {str(e)}") + return [] diff --git a/main/xiaozhi-server/core/providers/tts/base.py b/main/xiaozhi-server/core/providers/tts/base.py index 9127ec12..40570326 100644 --- a/main/xiaozhi-server/core/providers/tts/base.py +++ b/main/xiaozhi-server/core/providers/tts/base.py @@ -50,6 +50,7 @@ class TTSProviderBase(ABC): "；", ";", "：", + "~", ) self.first_sentence_punctuations = ( "，", @@ -159,7 +160,7 @@ class TTSProviderBase(ABC): if conn.sentence_id: sentence_id = conn.sentence_id else: - sentence_id = str(uuid.uuid4()).replace("-", "") + sentence_id = str(uuid.uuid4().hex) conn.sentence_id = sentence_id # 对于单句的文本，进行分段处理 segments = re.split(r"([。！？!?；;\n])", content_detail) @@ -332,7 +333,6 @@ class TTSProviderBase(ABC): Returns: tuple: (sentence_type, audio_datas, content_detail) """ - audio_datas = [] if tts_file.endswith(".p3"): audio_datas, _ = p3.decode_opus_from_file(tts_file) elif self.conn.audio_format == "pcm": diff --git a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py index 0eb7c72d..82cba035 100644 --- a/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py +++ b/main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py @@ -52,10 +52,11 @@ EVENT_ConnectionFinished = 52 # 连接结束 # 上行Session事件 EVENT_StartSession = 100 - +EVENT_CancelSession = 101 EVENT_FinishSession = 102 # 下行Session事件 EVENT_SessionStarted = 150 +EVENT_SessionCanceled = 151 EVENT_SessionFinished = 152 EVENT_SessionFailed = 153 @@ -207,8 +208,16 @@ class TTSProvider(TTSProviderBase): self.conn.client_abort = False if self.conn.client_abort: - logger.bind(tag=TAG).info("收到打断信息，终止TTS文本处理线程") - continue + try: + logger.bind(tag=TAG).info("收到打断信息，终止TTS文本处理线程") + asyncio.run_coroutine_threadsafe( + self.cancel_session(self.conn.sentence_id), + loop=self.conn.loop, + ) + continue + except Exception as e: + logger.bind(tag=TAG).error(f"取消TTS会话失败: {str(e)}") + continue if message.sentence_type == SentenceType.FIRST: # 初始化参数 @@ -371,6 +380,27 @@ class TTSProvider(TTSProviderBase): await self.close() raise + async def cancel_session(self,session_id): + logger.bind(tag=TAG).info(f"取消会话，释放服务端资源～～{session_id}") + try: + if self.ws: + header = Header( + message_type=FULL_CLIENT_REQUEST, + message_type_specific_flags=MsgTypeFlagWithEvent, + serial_method=JSON, + ).as_bytes() + optional = Optional( + event=EVENT_CancelSession, sessionId=session_id + ).as_bytes() + payload = str.encode("{}") + await self.send_event(self.ws, header, optional, payload) + logger.bind(tag=TAG).info("会话取消请求已发送") + except Exception as e: + logger.bind(tag=TAG).error(f"取消会话失败: {str(e)}") + # 确保清理资源 + await self.close() + raise + async def close(self): """资源清理方法""" # 取消监听任务 @@ -405,12 +435,11 @@ class TTSProvider(TTSProviderBase): res = self.parser_response(msg) self.print_response(res, "send_text res:") - # 检查客户端是否中止 - if self.conn.client_abort: - logger.bind(tag=TAG).info("收到打断信息，终止监听TTS响应") + if res.optional.event == EVENT_SessionCanceled: + logger.bind(tag=TAG).debug(f"释放服务端资源成功～～") + session_finished = True break - - if res.optional.event == EVENT_TTSSentenceStart: + elif res.optional.event == EVENT_TTSSentenceStart: json_data = json.loads(res.payload.decode("utf-8")) self.tts_text = json_data.get("text", "") logger.bind(tag=TAG).debug(f"句子语音生成开始: {self.tts_text}") @@ -435,10 +464,10 @@ class TTSProvider(TTSProviderBase): (SentenceType.MIDDLE, opus_datas, None) ) else: - opus_datas_cache = opus_datas_cache + opus_datas + opus_datas_cache.extend(opus_datas) else: # 后续句子缓存 - opus_datas_cache = opus_datas_cache + opus_datas + opus_datas_cache.extend(opus_datas) elif res.optional.event == EVENT_TTSSentenceEnd: logger.bind(tag=TAG).info(f"句子语音生成成功：{self.tts_text}") if not is_first_sentence or first_sentence_segment_count > 10: @@ -469,7 +498,7 @@ class TTSProvider(TTSProviderBase): except: pass self.ws = None - # 监听任务退出时清理引用 + # 监听任务退出时清理引用 finally: self._monitor_task = None diff --git a/main/xiaozhi-server/core/utils/prompt_manager.py b/main/xiaozhi-server/core/utils/prompt_manager.py index a5cfd425..79c92921 100644 --- a/main/xiaozhi-server/core/utils/prompt_manager.py +++ b/main/xiaozhi-server/core/utils/prompt_manager.py @@ -21,6 +21,30 @@ WEEKDAY_MAP = { "Sunday": "星期日", } +EMOJI_List = [ + "😶", + "🙂", + "😆", + "😂", + "😔", + "😠", + "😭", + "😍", + "😳", + "😲", + "😱", + "🤔", + "😉", + "😎", + "😌", + "🤤", + "😘", + "😏", + "😴", + "😜", + "🙄", +] + class PromptManager: """系统提示词管理器，负责管理和更新系统提示词""" @@ -206,6 +230,7 @@ class PromptManager: lunar_date=lunar_date, local_address=local_address, weather_info=weather_info, + emojiList=EMOJI_List, ) device_cache_key = f"device_prompt:{device_id}" self.cache_manager.set( diff --git a/main/xiaozhi-server/core/utils/textUtils.py b/main/xiaozhi-server/core/utils/textUtils.py index 603d964c..dd876cfa 100644 --- a/main/xiaozhi-server/core/utils/textUtils.py +++ b/main/xiaozhi-server/core/utils/textUtils.py @@ -1,3 +1,31 @@ +import json + +TAG = __name__ +EMOJI_MAP = { + "😂": "laughing", + "😭": "crying", + "😠": "angry", + "😔": "sad", + "😍": "loving", + "😲": "surprised", + "😱": "shocked", + "🤔": "thinking", + "😌": "relaxed", + "😴": "sleepy", + "😜": "silly", + "🙄": "confused", + "😶": "neutral", + "🙂": "happy", + "😆": "laughing", + "😳": "embarrassed", + "😉": "winking", + "😎": "cool", + "🤤": "delicious", + "😘": "kissy", + "😏": "confident", +} + + def get_string_no_punctuation_or_emoji(s): """去除字符串首尾的空格、标点符号和表情符号""" chars = list(s) @@ -22,6 +50,11 @@ def is_punctuation_or_emoji(char): ".", # 中文句号 + 英文句号 "！", "!", # 中文感叹号 + 英文感叹号 + "“", + "”", + '"', # 中文双引号 + 英文引号 + "：", + ":", # 中文冒号 + 英文冒号 "-", "－", # 英文连字符 + 中文全角横线 "、", # 中文顿号 @@ -44,3 +77,28 @@ def is_punctuation_or_emoji(char): (0x2700, 0x27BF), ] return any(start <= code_point <= end for start, end in emoji_ranges) + + +async def get_emotion(conn, text): + """获取文本内的情绪消息""" + emoji = "🙂" + emotion = "happy" + for char in text: + if char in EMOJI_MAP: + emoji = char + emotion = EMOJI_MAP[char] + break + try: + await conn.websocket.send( + json.dumps( + { + "type": "llm", + "text": emoji, + "emotion": emotion, + "session_id": conn.session_id, + } + ) + ) + except Exception as e: + conn.logger.bind(tag=TAG).warning(f"发送情绪表情失败，错误:{e}") + return diff --git a/main/xiaozhi-server/core/utils/util.py b/main/xiaozhi-server/core/utils/util.py index bc778558..ce6f20ea 100644 --- a/main/xiaozhi-server/core/utils/util.py +++ b/main/xiaozhi-server/core/utils/util.py @@ -125,51 +125,6 @@ def write_json_file(file_path, data): json.dump(data, file, ensure_ascii=False, indent=4) -def is_punctuation_or_emoji(char): - """检查字符是否为空格、指定标点或表情符号""" - # 定义需要去除的中英文标点（包括全角/半角） - punctuation_set = { - "，", - ",", # 中文逗号 + 英文逗号 - "-", - "－", # 英文连字符 + 中文全角横线 - "、", # 中文顿号 - "“", - "”", - '"', # 中文双引号 + 英文引号 - "：", - ":", # 中文冒号 + 英文冒号 - } - if char.isspace() or char in punctuation_set: - return True - # 检查表情符号（保留原有逻辑） - code_point = ord(char) - emoji_ranges = [ - (0x1F600, 0x1F64F), - (0x1F300, 0x1F5FF), - (0x1F680, 0x1F6FF), - (0x1F900, 0x1F9FF), - (0x1FA70, 0x1FAFF), - (0x2600, 0x26FF), - (0x2700, 0x27BF), - ] - return any(start <= code_point <= end for start, end in emoji_ranges) - - -def get_string_no_punctuation_or_emoji(s): - """去除字符串首尾的空格、标点符号和表情符号""" - chars = list(s) - # 处理开头的字符 - start = 0 - while start < len(chars) and is_punctuation_or_emoji(chars[start]): - start += 1 - # 处理结尾的字符 - end = len(chars) - 1 - while end >= start and is_punctuation_or_emoji(chars[end]): - end -= 1 - return "".join(chars[start : end + 1]) - - def remove_punctuation_and_length(text): # 全角符号和半角符号的Unicode范围 full_width_punctuations = ( @@ -256,515 +211,6 @@ def extract_json_from_string(input_string): return None -def analyze_emotion(text): - """ - 分析文本情感并返回对应的emoji名称（支持中英文） - """ - if not text or not isinstance(text, str): - return "neutral" - - original_text = text - text = text.lower().strip() - - # 检查是否包含现有emoji - for emotion, emoji in emoji_map.items(): - if emoji in original_text: - return emotion - - # 标点符号分析 - has_exclamation = "!" in original_text or "！" in original_text - has_question = "?" in original_text or "？" in original_text - has_ellipsis = "..." in original_text or "…" in original_text - - # 定义情感关键词映射（中英文扩展版） - emotion_keywords = { - "happy": [ - "开心", - "高兴", - "快乐", - "愉快", - "幸福", - "满意", - "棒", - "好", - "不错", - "完美", - "棒极了", - "太好了", - "好呀", - "好的", - "happy", - "joy", - "great", - "good", - "nice", - "awesome", - "fantastic", - "wonderful", - ], - "laughing": [ - "哈哈", - "哈哈哈", - "呵呵", - "嘿嘿", - "嘻嘻", - "笑死", - "太好笑了", - "笑死我了", - "lol", - "lmao", - "haha", - "hahaha", - "hehe", - "rofl", - "funny", - "laugh", - ], - "funny": [ - "搞笑", - "滑稽", - "逗", - "幽默", - "笑点", - "段子", - "笑话", - "太逗了", - "hilarious", - "joke", - "comedy", - ], - "sad": [ - "伤心", - "难过", - "悲哀", - "悲伤", - "忧郁", - "郁闷", - "沮丧", - "失望", - "想哭", - "难受", - "不开心", - "唉", - "呜呜", - "sad", - "upset", - "unhappy", - "depressed", - "sorrow", - "gloomy", - ], - "angry": [ - "生气", - "愤怒", - "气死", - "讨厌", - "烦人", - "可恶", - "烦死了", - "恼火", - "暴躁", - "火大", - "愤怒", - "气炸了", - "angry", - "mad", - "annoyed", - "furious", - "pissed", - "hate", - ], - "crying": [ - "哭泣", - "泪流", - "大哭", - "伤心欲绝", - "泪目", - "流泪", - "哭死", - "哭晕", - "想哭", - "泪崩", - "cry", - "crying", - "tears", - "sob", - "weep", - ], - "loving": [ - "爱你", - "喜欢", - "爱", - "亲爱的", - "宝贝", - "么么哒", - "抱抱", - "想你", - "思念", - "最爱", - "亲亲", - "喜欢你", - "love", - "like", - "adore", - "darling", - "sweetie", - "honey", - "miss you", - "heart", - ], - "embarrassed": [ - "尴尬", - "不好意思", - "害羞", - "脸红", - "难为情", - "社死", - "丢脸", - "出丑", - "embarrassed", - "awkward", - "shy", - "blush", - ], - "surprised": [ - "惊讶", - "吃惊", - "天啊", - "哇塞", - "哇", - "居然", - "竟然", - "没想到", - "出乎意料", - "surprise", - "wow", - "omg", - "oh my god", - "amazing", - "unbelievable", - ], - "shocked": [ - "震惊", - "吓到", - "惊呆了", - "不敢相信", - "震撼", - "吓死", - "恐怖", - "害怕", - "吓人", - "shocked", - "shocking", - "scared", - "frightened", - "terrified", - "horror", - ], - "thinking": [ - "思考", - "考虑", - "想一下", - "琢磨", - "沉思", - "冥想", - "想", - "思考中", - "在想", - "think", - "thinking", - "consider", - "ponder", - "meditate", - ], - "winking": [ - "调皮", - "眨眼", - "你懂的", - "坏笑", - "邪恶", - "奸笑", - "使眼色", - "wink", - "teasing", - "naughty", - "mischievous", - ], - "cool": [ - "酷", - "帅", - "厉害", - "棒极了", - "真棒", - "牛逼", - "强", - "优秀", - "杰出", - "出色", - "完美", - "cool", - "awesome", - "amazing", - "great", - "impressive", - "perfect", - ], - "relaxed": [ - "放松", - "舒服", - "惬意", - "悠闲", - "轻松", - "舒适", - "安逸", - "自在", - "relax", - "relaxed", - "comfortable", - "cozy", - "chill", - "peaceful", - ], - "delicious": [ - "好吃", - "美味", - "香", - "馋", - "可口", - "香甜", - "大餐", - "大快朵颐", - "流口水", - "垂涎", - "delicious", - "yummy", - "tasty", - "yum", - "appetizing", - "mouthwatering", - ], - "kissy": [ - "亲亲", - "么么", - "吻", - "mua", - "muah", - "亲一下", - "飞吻", - "kiss", - "xoxo", - "hug", - "muah", - "smooch", - ], - "confident": [ - "自信", - "肯定", - "确定", - "毫无疑问", - "当然", - "必须的", - "毫无疑问", - "确信", - "坚信", - "confident", - "sure", - "certain", - "definitely", - "positive", - ], - "sleepy": [ - "困", - "睡觉", - "晚安", - "想睡", - "好累", - "疲惫", - "疲倦", - "困了", - "想休息", - "睡意", - "sleep", - "sleepy", - "tired", - "exhausted", - "bedtime", - "good night", - ], - "silly": [ - "傻", - "笨", - "呆", - "憨", - "蠢", - "二", - "憨憨", - "傻乎乎", - "呆萌", - "silly", - "stupid", - "dumb", - "foolish", - "goofy", - "ridiculous", - ], - "confused": [ - "疑惑", - "不明白", - "不懂", - "困惑", - "疑问", - "为什么", - "怎么回事", - "啥意思", - "不清楚", - "confused", - "puzzled", - "doubt", - "question", - "what", - "why", - "how", - ], - } - - # 特殊句型判断（中英文） - # 赞美他人 - if any( - phrase in text - for phrase in [ - "你真", - "你好", - "您真", - "你真棒", - "你好厉害", - "你太强了", - "你真好", - "你真聪明", - "you are", - "you're", - "you look", - "you seem", - "so smart", - "so kind", - ] - ): - return "loving" - # 自我赞美 - if any( - phrase in text - for phrase in [ - "我真", - "我最", - "我太棒了", - "我厉害", - "我聪明", - "我优秀", - "i am", - "i'm", - "i feel", - "so good", - "so happy", - ] - ): - return "cool" - # 晚安/睡觉相关 - if any( - phrase in text - for phrase in [ - "睡觉", - "晚安", - "睡了", - "好梦", - "休息了", - "去睡了", - "sleep", - "good night", - "bedtime", - "go to bed", - ] - ): - return "sleepy" - # 疑问句 - if has_question and not has_exclamation: - return "thinking" - # 强烈情感（感叹号） - if has_exclamation and not has_question: - # 检查是否是积极内容 - positive_words = ( - emotion_keywords["happy"] - + emotion_keywords["laughing"] - + emotion_keywords["cool"] - ) - if any(word in text for word in positive_words): - return "laughing" - # 检查是否是消极内容 - negative_words = ( - emotion_keywords["angry"] - + emotion_keywords["sad"] - + emotion_keywords["crying"] - ) - if any(word in text for word in negative_words): - return "angry" - return "surprised" - # 省略号（表示犹豫或思考） - if has_ellipsis: - return "thinking" - - # 关键词匹配（带权重） - emotion_scores = {emotion: 0 for emotion in emoji_map.keys()} - - # 给匹配到的关键词加分 - for emotion, keywords in emotion_keywords.items(): - for keyword in keywords: - if keyword in text: - emotion_scores[emotion] += 1 - - # 给长文本中的重复关键词额外加分 - if len(text) > 20: # 长文本 - for emotion, keywords in emotion_keywords.items(): - for keyword in keywords: - emotion_scores[emotion] += text.count(keyword) * 0.5 - - # 根据分数选择最可能的情感 - max_score = max(emotion_scores.values()) - if max_score == 0: - return "happy" # 默认 - - # 可能有多个情感同分，根据上下文选择最合适的 - top_emotions = [e for e, s in emotion_scores.items() if s == max_score] - - # 如果多个情感同分，使用以下优先级 - priority_order = [ - "laughing", - "crying", - "angry", - "surprised", - "shocked", # 强烈情感优先 - "loving", - "happy", - "funny", - "cool", # 积极情感 - "sad", - "embarrassed", - "confused", # 消极情感 - "thinking", - "winking", - "relaxed", # 中性情感 - "delicious", - "kissy", - "confident", - "sleepy", - "silly", # 特殊场景 - ] - - for emotion in priority_order: - if emotion in top_emotions: - return emotion - - return top_emotions[0] # 如果都不在优先级列表里，返回第一个 - - def audio_to_data(audio_file_path, is_opus=True): # 获取文件后缀名 file_type = os.path.splitext(audio_file_path)[1]