调整相关逻辑
This commit is contained in:
1 parent
1120be2f60
commit
3bf619e2dd
17 files changed
+105
-58
No files matched your search
@@ -42,7 +42,7 @@ from core.providers.tts.dto.dto import ContentType, TTSMessageDTO, SentenceType
|
||||
from config.logger import setup_logging, build_module_string, create_connection_logger
|
||||
from config.manage_api_client import DeviceNotFoundException, DeviceBindException, generate_and_save_chat_title
|
||||
from core.utils.prompt_manager import PromptManager
|
||||
from core.utils.voiceprint_provider import VoiceprintProvider
|
||||
from core.utils.voiceprint_provider import VoiceprintProvider, UNKNOWN_SPEAKER_NAME
|
||||
from core.utils.util import get_system_error_response
|
||||
from core.utils import textUtils
|
||||
|
||||
@@ -159,7 +159,6 @@ class ConnectionHandler:
|
||||
self.asr_audio = [] # 存储PCM帧列表,供VAD和ASR共享
|
||||
self.asr_audio_queue = queue.Queue()
|
||||
self.current_speaker = None # 当前说话人展示名
|
||||
self.current_speaker_id = None # 声纹识别得到的 speaker_id
|
||||
self.introduced_speakers = set() # 已"首次引入"的说话人,控制只在首轮带名字
|
||||
self.system_introduced_speakers = set() # 已在 system 注入过身份的说话人,控制 system 身份只首轮出现
|
||||
|
||||
@@ -1053,7 +1052,8 @@ class ConnectionHandler:
|
||||
# 更新系统prompt至上下文
|
||||
self.dialogue.update_system_message(self.prompt)
|
||||
|
||||
def chat(self, query, depth=0):
|
||||
def chat(self, query, depth=0, speaker=None):
|
||||
# speaker: 本轮声纹身份快照,随工具调用下传
|
||||
# 保存当前任务的sentence_id到局部变量,避免被新任务覆盖
|
||||
current_sentence_id = None
|
||||
|
||||
@@ -1123,7 +1123,7 @@ class ConnectionHandler:
|
||||
# 避免每轮在 system 重复出现名字诱导模型反复称呼
|
||||
speaker_for_system = None
|
||||
cs = (self.current_speaker or "").strip()
|
||||
if cs and cs != "未知说话人" and cs not in self.system_introduced_speakers:
|
||||
if cs and cs != UNKNOWN_SPEAKER_NAME and cs not in self.system_introduced_speakers:
|
||||
self.system_introduced_speakers.add(cs)
|
||||
speaker_for_system = cs
|
||||
|
||||
@@ -1338,7 +1338,7 @@ class ConnectionHandler:
|
||||
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
self.func_handler.handle_llm_function_call(
|
||||
self, tool_call_data
|
||||
self, tool_call_data, speaker=speaker
|
||||
),
|
||||
self.loop,
|
||||
)
|
||||
@@ -1501,7 +1501,7 @@ class ConnectionHandler:
|
||||
)
|
||||
)
|
||||
|
||||
self.chat(None, depth=depth + 1)
|
||||
self.chat(None, depth=depth + 1, speaker=speaker)
|
||||
|
||||
def _report_worker(self):
|
||||
"""聊天记录上报工作线程"""
|
||||
|
||||
@@ -17,7 +17,7 @@ from core.providers.tts.dto.dto import TTSMessageDTO, SentenceType
|
||||
TAG = __name__
|
||||
|
||||
|
||||
async def handle_user_intent(conn: "ConnectionHandler", text):
|
||||
async def handle_user_intent(conn: "ConnectionHandler", text, speaker=None):
|
||||
# 预处理输入文本,处理可能的JSON格式
|
||||
try:
|
||||
if text.strip().startswith("{") and text.strip().endswith("}"):
|
||||
@@ -172,7 +172,7 @@ async def process_intent_result(
|
||||
try:
|
||||
result = asyncio.run_coroutine_threadsafe(
|
||||
conn.func_handler.handle_llm_function_call(
|
||||
conn, function_call_data
|
||||
conn, function_call_data, speaker=speaker
|
||||
),
|
||||
conn.loop,
|
||||
).result(timeout=tool_call_timeout)
|
||||
|
||||
@@ -40,7 +40,8 @@ async def resume_vad_detection(conn: "ConnectionHandler"):
|
||||
conn.just_woken_up = False
|
||||
|
||||
|
||||
async def startToChat(conn: "ConnectionHandler", text):
|
||||
async def startToChat(conn: "ConnectionHandler", text, speaker=None):
|
||||
"""speaker: 本轮声纹身份快照,随调用链传递。"""
|
||||
# 检查输入是否是JSON格式(包含说话人信息)
|
||||
speaker_name = None
|
||||
actual_text = text
|
||||
@@ -69,13 +70,8 @@ async def startToChat(conn: "ConnectionHandler", text):
|
||||
# 如果解析失败,继续使用原始文本
|
||||
pass
|
||||
|
||||
# 保存说话人信息到连接对象。speaker_id 由声纹识别写入,这里只在没有说话人时清掉,
|
||||
# 避免上一轮身份落到没有声纹的文本消息上。
|
||||
if speaker_name:
|
||||
conn.current_speaker = speaker_name
|
||||
else:
|
||||
conn.current_speaker = None
|
||||
conn.current_speaker_id = None
|
||||
# 说话人名供提示词注入;声纹身份走 speaker 快照,不挂连接
|
||||
conn.current_speaker = speaker_name
|
||||
|
||||
if conn.need_bind:
|
||||
await check_bind_device(conn)
|
||||
@@ -94,7 +90,7 @@ async def startToChat(conn: "ConnectionHandler", text):
|
||||
await handleAbortMessage(conn)
|
||||
|
||||
# 首先进行意图分析,使用实际文本内容
|
||||
intent_handled = await handle_user_intent(conn, actual_text)
|
||||
intent_handled = await handle_user_intent(conn, actual_text, speaker=speaker)
|
||||
|
||||
if intent_handled:
|
||||
# 如果意图已被处理,不再进行聊天
|
||||
@@ -106,7 +102,7 @@ async def startToChat(conn: "ConnectionHandler", text):
|
||||
# 准备开始新会话
|
||||
conn.client_abort = False
|
||||
|
||||
conn.executor.submit(conn.chat, actual_text)
|
||||
conn.executor.submit(conn.chat, actual_text, depth=0, speaker=speaker)
|
||||
|
||||
|
||||
async def no_voice_close_connect(conn: "ConnectionHandler", have_voice):
|
||||
|
||||
@@ -17,6 +17,7 @@ from core.providers.asr.dto.dto import InterfaceType
|
||||
from core.handle.receiveAudioHandle import startToChat
|
||||
from core.handle.reportHandle import enqueue_asr_report
|
||||
from core.utils.util import remove_punctuation_and_length
|
||||
from core.utils.voiceprint_provider import SpeakerIdentification
|
||||
from core.handle.receiveAudioHandle import handleAudioMessage
|
||||
from typing import Optional, Tuple, List, NamedTuple, TYPE_CHECKING
|
||||
|
||||
@@ -119,13 +120,11 @@ class ASRProviderBase(ABC):
|
||||
else:
|
||||
raw_text, _ = asr_result
|
||||
|
||||
speaker_id = None
|
||||
speaker_name = ""
|
||||
if isinstance(voiceprint_result, Exception):
|
||||
logger.bind(tag=TAG).error(f"声纹识别失败: {voiceprint_result}")
|
||||
elif voiceprint_result is not None:
|
||||
speaker_name = voiceprint_result.name or ""
|
||||
speaker_id = voiceprint_result.speaker_id
|
||||
|
||||
# 判断 ASR 结果类型
|
||||
if isinstance(raw_text, dict):
|
||||
@@ -166,15 +165,16 @@ class ASRProviderBase(ABC):
|
||||
self.stop_ws_connection()
|
||||
|
||||
if text_len > 0:
|
||||
# 声纹 id 不进入发给 LLM 的文本,留在连接上供本轮 MCP 接入点调用读取。
|
||||
if isinstance(speaker_id, str) and speaker_id.strip():
|
||||
conn.current_speaker_id = speaker_id.strip()
|
||||
else:
|
||||
conn.current_speaker_id = None
|
||||
# 本轮声纹身份快照,随调用链供 MCP 接入点工具使用
|
||||
speaker = (
|
||||
voiceprint_result
|
||||
if isinstance(voiceprint_result, SpeakerIdentification)
|
||||
else None
|
||||
)
|
||||
audio_snapshot = asr_audio_task.copy()
|
||||
enqueue_asr_report(conn, enhanced_text, audio_snapshot)
|
||||
# 使用自定义模块进行上报
|
||||
await startToChat(conn, enhanced_text)
|
||||
await startToChat(conn, enhanced_text, speaker=speaker)
|
||||
except Exception as e:
|
||||
logger.bind(tag=TAG).error(f"处理语音停止失败: {e}")
|
||||
import traceback
|
||||
|
||||
@@ -11,9 +11,9 @@ class ToolExecutor(ABC):
|
||||
|
||||
@abstractmethod
|
||||
async def execute(
|
||||
self, conn, tool_name: str, arguments: Dict[str, Any]
|
||||
self, conn, tool_name: str, arguments: Dict[str, Any], speaker=None
|
||||
) -> ActionResponse:
|
||||
"""执行工具调用"""
|
||||
"""执行工具调用。speaker: 可选的说话人身份快照。"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
|
||||
@@ -15,7 +15,7 @@ class DeviceIoTExecutor(ToolExecutor):
|
||||
self.iot_tools: Dict[str, ToolDefinition] = {}
|
||||
|
||||
async def execute(
|
||||
self, conn, tool_name: str, arguments: Dict[str, Any]
|
||||
self, conn, tool_name: str, arguments: Dict[str, Any], speaker=None
|
||||
) -> ActionResponse:
|
||||
"""执行设备端IoT工具"""
|
||||
if not self.has_tool(tool_name):
|
||||
|
||||
@@ -16,7 +16,7 @@ class DeviceMCPExecutor(ToolExecutor):
|
||||
self.conn = conn
|
||||
|
||||
async def execute(
|
||||
self, conn: "ConnectionHandler", tool_name: str, arguments: Dict[str, Any]
|
||||
self, conn: "ConnectionHandler", tool_name: str, arguments: Dict[str, Any], speaker=None
|
||||
) -> ActionResponse:
|
||||
"""执行设备端MCP工具"""
|
||||
if not hasattr(conn, "mcp_client") or not conn.mcp_client:
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
"""MCP接入点工具执行器"""
|
||||
|
||||
from typing import Dict, Any
|
||||
from typing import Dict, Any, Optional
|
||||
from ..base import ToolType, ToolDefinition, ToolExecutor
|
||||
from plugins_func.register import Action, ActionResponse
|
||||
from core.utils.voiceprint_provider import SpeakerIdentification
|
||||
from .mcp_endpoint_handler import call_mcp_endpoint_tool
|
||||
from .mcp_speaker import should_forward_speaker, speaker_call_meta
|
||||
|
||||
@@ -14,9 +15,9 @@ class MCPEndpointExecutor(ToolExecutor):
|
||||
self.conn = conn
|
||||
|
||||
async def execute(
|
||||
self, conn, tool_name: str, arguments: Dict[str, Any]
|
||||
self, conn, tool_name: str, arguments: Dict[str, Any], speaker: Optional[SpeakerIdentification] = None
|
||||
) -> ActionResponse:
|
||||
"""执行MCP接入点工具"""
|
||||
"""执行MCP接入点工具。speaker: 本轮说话人快照。"""
|
||||
if not hasattr(conn, "mcp_endpoint_client") or not conn.mcp_endpoint_client:
|
||||
return ActionResponse(
|
||||
action=Action.ERROR,
|
||||
@@ -34,11 +35,10 @@ class MCPEndpointExecutor(ToolExecutor):
|
||||
import json
|
||||
|
||||
meta = None
|
||||
if should_forward_speaker(getattr(conn, "config", None)):
|
||||
meta = speaker_call_meta(
|
||||
getattr(conn, "current_speaker_id", None),
|
||||
getattr(conn, "current_speaker", None),
|
||||
)
|
||||
if speaker is not None and should_forward_speaker(
|
||||
getattr(conn, "config", None)
|
||||
):
|
||||
meta = speaker_call_meta(speaker.speaker_id, speaker.name)
|
||||
|
||||
args_str = json.dumps(arguments) if arguments else "{}"
|
||||
|
||||
|
||||
@@ -13,7 +13,10 @@ def should_forward_speaker(config: Optional[dict]) -> bool:
|
||||
"""mcp_forward_speaker 缺省为开启。显式 false 时不附带身份。"""
|
||||
if not isinstance(config, dict):
|
||||
return True
|
||||
return bool(config.get("mcp_forward_speaker", True))
|
||||
value = config.get("mcp_forward_speaker", True)
|
||||
if value is None:
|
||||
return True
|
||||
return str(value).strip().lower() in ("true", "1", "yes")
|
||||
|
||||
|
||||
def speaker_call_meta(speaker_id: Any, speaker_name: Any) -> Optional[Dict[str, str]]:
|
||||
|
||||
@@ -22,7 +22,7 @@ class ServerMCPExecutor(ToolExecutor):
|
||||
await self.mcp_manager.initialize_servers()
|
||||
|
||||
async def execute(
|
||||
self, conn, tool_name: str, arguments: Dict[str, Any]
|
||||
self, conn, tool_name: str, arguments: Dict[str, Any], speaker=None
|
||||
) -> ActionResponse:
|
||||
"""执行服务端MCP工具"""
|
||||
if not self._initialized or not self.mcp_manager:
|
||||
|
||||
@@ -17,7 +17,7 @@ class ServerPluginExecutor(ToolExecutor):
|
||||
self.config = conn.config
|
||||
|
||||
async def execute(
|
||||
self, conn: "ConnectionHandler", tool_name: str, arguments: Dict[str, Any]
|
||||
self, conn: "ConnectionHandler", tool_name: str, arguments: Dict[str, Any], speaker=None
|
||||
) -> ActionResponse:
|
||||
"""执行服务端插件工具"""
|
||||
func_item = all_function_registry.get(tool_name)
|
||||
|
||||
@@ -137,16 +137,16 @@ class UnifiedToolHandler:
|
||||
return self.tool_manager.has_tool(tool_name)
|
||||
|
||||
async def handle_llm_function_call(
|
||||
self, conn, function_call_data: Dict[str, Any]
|
||||
self, conn, function_call_data: Dict[str, Any], speaker=None
|
||||
) -> Optional[ActionResponse]:
|
||||
"""处理LLM函数调用"""
|
||||
"""处理LLM函数调用。speaker: 本轮说话人快照。"""
|
||||
try:
|
||||
# 处理多函数调用
|
||||
if "function_calls" in function_call_data:
|
||||
responses = []
|
||||
for call in function_call_data["function_calls"]:
|
||||
result = await self.tool_manager.execute_tool(
|
||||
call["name"], call.get("arguments", {})
|
||||
call["name"], call.get("arguments", {}), speaker=speaker
|
||||
)
|
||||
responses.append(result)
|
||||
return self._combine_responses(responses)
|
||||
@@ -175,7 +175,9 @@ class UnifiedToolHandler:
|
||||
self.logger.warning(f"发送工具调用显示消息失败: {e}")
|
||||
|
||||
# 执行工具调用
|
||||
result = await self.tool_manager.execute_tool(function_name, arguments)
|
||||
result = await self.tool_manager.execute_tool(
|
||||
function_name, arguments, speaker=speaker
|
||||
)
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
|
||||
@@ -71,9 +71,9 @@ class ToolManager:
|
||||
return tool_def.tool_type if tool_def else None
|
||||
|
||||
async def execute_tool(
|
||||
self, tool_name: str, arguments: Dict[str, Any]
|
||||
self, tool_name: str, arguments: Dict[str, Any], speaker=None
|
||||
) -> ActionResponse:
|
||||
"""执行工具调用"""
|
||||
"""执行工具调用。speaker: 本轮说话人快照。"""
|
||||
try:
|
||||
# 查找工具类型
|
||||
tool_type = self.get_tool_type(tool_name)
|
||||
@@ -93,7 +93,7 @@ class ToolManager:
|
||||
|
||||
# 执行工具
|
||||
self.logger.info(f"执行工具: {tool_name},参数: {arguments}")
|
||||
result = await executor.execute(self.conn, tool_name, arguments)
|
||||
result = await executor.execute(self.conn, tool_name, arguments, speaker=speaker)
|
||||
self.logger.debug(f"工具执行结果: {result}")
|
||||
return result
|
||||
|
||||
|
||||
@@ -3,6 +3,8 @@ import re
|
||||
from typing import List, Dict
|
||||
from datetime import datetime
|
||||
|
||||
from core.utils.voiceprint_provider import UNKNOWN_SPEAKER_NAME
|
||||
|
||||
|
||||
class Message:
|
||||
def __init__(
|
||||
@@ -125,7 +127,7 @@ class Dialogue:
|
||||
current_speaker_name = (current_speaker or "").strip()
|
||||
# 仅在本轮注入了有效身份时才输出 speakers_info,避免列表里的名字每轮
|
||||
# 重复出现诱导模型反复称呼;后续轮不再注入身份,靠对话历史首轮保留
|
||||
if current_speaker_name and current_speaker_name != "未知说话人":
|
||||
if current_speaker_name and current_speaker_name != UNKNOWN_SPEAKER_NAME:
|
||||
speakers = voiceprint_config.get("speakers", [])
|
||||
speakers_info = "\n<speakers_info>"
|
||||
speakers_info += f"\n当前说话人:{current_speaker_name}"
|
||||
|
||||
@@ -23,7 +23,10 @@ class SpeakerIdentification(NamedTuple):
|
||||
|
||||
|
||||
def resolve_speaker_identification(
|
||||
speaker_id, score, speaker_map, similarity_threshold
|
||||
speaker_id: Optional[str],
|
||||
score: float,
|
||||
speaker_map: Dict[str, Dict[str, str]],
|
||||
similarity_threshold: float,
|
||||
) -> SpeakerIdentification:
|
||||
"""把声纹接口返回的 speaker_id / score 解析成展示名,并保留命中的 id。"""
|
||||
if score < similarity_threshold:
|
||||
|
||||
Whitespace-only changes.
@@ -8,7 +8,7 @@ from core.providers.tools.mcp_endpoint.mcp_speaker import (
|
||||
should_forward_speaker,
|
||||
speaker_call_meta,
|
||||
)
|
||||
from core.utils.voiceprint_provider import UNKNOWN_SPEAKER_NAME
|
||||
from core.utils.voiceprint_provider import UNKNOWN_SPEAKER_NAME, SpeakerIdentification
|
||||
|
||||
|
||||
def test_meta_includes_id_and_name():
|
||||
@@ -29,6 +29,14 @@ def test_forwarding_defaults_on_and_can_be_disabled():
|
||||
assert should_forward_speaker({}) is True
|
||||
assert should_forward_speaker({"mcp_forward_speaker": True}) is True
|
||||
assert should_forward_speaker({"mcp_forward_speaker": False}) is False
|
||||
# 字符串 false 可关闭;空值视为缺省开启
|
||||
assert should_forward_speaker({"mcp_forward_speaker": "false"}) is False
|
||||
assert should_forward_speaker({"mcp_forward_speaker": "False"}) is False
|
||||
assert should_forward_speaker({"mcp_forward_speaker": "0"}) is False
|
||||
assert should_forward_speaker({"mcp_forward_speaker": "true"}) is True
|
||||
assert should_forward_speaker({"mcp_forward_speaker": None}) is True
|
||||
# 空串按关闭处理,与 manager-api 的 Boolean.parseBoolean("") 一致
|
||||
assert should_forward_speaker({"mcp_forward_speaker": ""}) is False
|
||||
|
||||
|
||||
def test_payload_adds_meta_without_touching_arguments():
|
||||
@@ -57,11 +65,9 @@ class _Client:
|
||||
|
||||
|
||||
class _Conn:
|
||||
def __init__(self, config=None, speaker_id="test1", speaker_name="张三"):
|
||||
def __init__(self, config=None):
|
||||
self.mcp_endpoint_client = _Client()
|
||||
self.config = {} if config is None else config
|
||||
self.current_speaker_id = speaker_id
|
||||
self.current_speaker = speaker_name
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -80,7 +86,10 @@ async def test_executor_forwards_speaker_in_meta_only(monkeypatch):
|
||||
arguments = {"area": "客厅"}
|
||||
conn = _Conn()
|
||||
result = await executor_mod.MCPEndpointExecutor(conn).execute(
|
||||
conn, "HassTurnOn", arguments
|
||||
conn,
|
||||
"HassTurnOn",
|
||||
arguments,
|
||||
speaker=SpeakerIdentification("test1", "张三"),
|
||||
)
|
||||
|
||||
assert json.loads(captured["args"]) == {"area": "客厅"}
|
||||
@@ -101,7 +110,12 @@ async def test_executor_skips_meta_when_disabled(monkeypatch):
|
||||
|
||||
monkeypatch.setattr(executor_mod, "call_mcp_endpoint_tool", fake_call)
|
||||
conn = _Conn(config={"mcp_forward_speaker": False})
|
||||
await executor_mod.MCPEndpointExecutor(conn).execute(conn, "HassTurnOn", {"area": "客厅"})
|
||||
await executor_mod.MCPEndpointExecutor(conn).execute(
|
||||
conn,
|
||||
"HassTurnOn",
|
||||
{"area": "客厅"},
|
||||
speaker=SpeakerIdentification("test1", "张三"),
|
||||
)
|
||||
assert captured["meta"] is None
|
||||
|
||||
|
||||
@@ -116,6 +130,33 @@ async def test_executor_omits_meta_for_unknown_speaker(monkeypatch):
|
||||
return "done"
|
||||
|
||||
monkeypatch.setattr(executor_mod, "call_mcp_endpoint_tool", fake_call)
|
||||
conn = _Conn(speaker_id=None, speaker_name=UNKNOWN_SPEAKER_NAME)
|
||||
await executor_mod.MCPEndpointExecutor(conn).execute(conn, "HassTurnOn", {})
|
||||
conn = _Conn()
|
||||
await executor_mod.MCPEndpointExecutor(conn).execute(
|
||||
conn,
|
||||
"HassTurnOn",
|
||||
{},
|
||||
speaker=SpeakerIdentification(None, UNKNOWN_SPEAKER_NAME),
|
||||
)
|
||||
assert captured["meta"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_executor_reads_snapshot_not_conn_state(monkeypatch):
|
||||
"""meta 取自快照,不受连接态影响。"""
|
||||
from core.providers.tools.mcp_endpoint import mcp_endpoint_executor as executor_mod
|
||||
|
||||
captured = {}
|
||||
|
||||
async def fake_call(client, tool_name, args_str="{}", timeout=30, meta=None):
|
||||
captured["meta"] = meta
|
||||
return "done"
|
||||
|
||||
monkeypatch.setattr(executor_mod, "call_mcp_endpoint_tool", fake_call)
|
||||
conn = _Conn()
|
||||
await executor_mod.MCPEndpointExecutor(conn).execute(
|
||||
conn,
|
||||
"HassTurnOn",
|
||||
{},
|
||||
speaker=SpeakerIdentification("test1", "张三"),
|
||||
)
|
||||
assert captured["meta"] == {"speakerId": "test1", "speakerName": "张三"}
|
||||
Reference in new issue
Block a user