From 0be634c627e6cb831d0351e67e1201db3ff9732e Mon Sep 17 00:00:00 2001 From: korexil <289361616+korexil@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:32:56 +0900 Subject: [PATCH] =?UTF-8?q?fix:=20SenseVoice=20=E6=97=A5=E8=AF=AD=E8=AF=86?= =?UTF-8?q?=E5=88=AB=E7=BB=93=E6=9E=9C=E6=8C=89=E8=AF=8D=E5=B8=A6=E7=A9=BA?= =?UTF-8?q?=E6=A0=BC=20(#3412)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SenseVoice 识别日语时偶尔按词输出空格（如「今日 の こと を 少し 長め に」）， 这些空格会原样进入对话历史并交给 LLM。lang_tag_filter 现在去掉两个 中日文字符之间的空格，韩文、英文的词间空格保持不变。 --- .../core/providers/asr/utils.py | 8 +++++++ .../tests/core/providers/__init__.py | 0 .../tests/core/providers/asr/__init__.py | 0 .../providers/asr/test_lang_tag_filter.py | 23 +++++++++++++++++++ 4 files changed, 31 insertions(+) create mode 100644 main/xiaozhi-server/tests/core/providers/__init__.py create mode 100644 main/xiaozhi-server/tests/core/providers/asr/__init__.py create mode 100644 main/xiaozhi-server/tests/core/providers/asr/test_lang_tag_filter.py diff --git a/main/xiaozhi-server/core/providers/asr/utils.py b/main/xiaozhi-server/core/providers/asr/utils.py index 780d3cc06..eafab8463 100644 --- a/main/xiaozhi-server/core/providers/asr/utils.py +++ b/main/xiaozhi-server/core/providers/asr/utils.py @@ -51,6 +51,14 @@ def lang_tag_filter(text: str) -> dict: # 移除所有 <|...|> 格式的标签，获取纯文本 clean_text = re.sub(tag_pattern, "", text).strip() + # SenseVoice 对日语等偶尔按词输出空格（如「今日 の こと を」），去掉中日文字符之间的空格； + # 韩文、英文的词间空格不受影响 + clean_text = re.sub( + r"(?<=[\u3000-\u303f\u3040-\u30ff\u4e00-\u9fff\uff01-\uff60])\s+" + r"(?=[\u3000-\u303f\u3040-\u30ff\u4e00-\u9fff\uff01-\uff60])", + "", + clean_text, + ) # 保持返回结构一致，避免调用方把纯文本误当成字典访问。 if not all_tags: diff --git a/main/xiaozhi-server/tests/core/providers/__init__.py b/main/xiaozhi-server/tests/core/providers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/main/xiaozhi-server/tests/core/providers/asr/__init__.py b/main/xiaozhi-server/tests/core/providers/asr/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/main/xiaozhi-server/tests/core/providers/asr/test_lang_tag_filter.py b/main/xiaozhi-server/tests/core/providers/asr/test_lang_tag_filter.py new file mode 100644 index 000000000..48b34eb3c --- /dev/null +++ b/main/xiaozhi-server/tests/core/providers/asr/test_lang_tag_filter.py @@ -0,0 +1,23 @@ +"""lang_tag_filter: SenseVoice tag parsing and CJK word-spacing cleanup.""" +from core.providers.asr.utils import lang_tag_filter + + +def test_japanese_word_spaces_are_removed(): + result = lang_tag_filter("<|ja|><|NEUTRAL|><|Speech|><|withitn|>今日 の こと を 少し 長め に 話し て ください。") + assert result["content"] == "今日のことを少し長めに話してください。" + assert result["language"] == "ja" + + +def test_chinese_text_is_unchanged(): + result = lang_tag_filter("<|zh|><|HAPPY|><|Speech|><|withitn|>你好啊，测试测试。") + assert result["content"] == "你好啊，测试测试。" + + +def test_spaces_next_to_latin_and_korean_are_kept(): + assert lang_tag_filter("<|ja|><|NEUTRAL|><|Speech|><|withitn|>OK Google と 言って")["content"] == "OK Google と言って" + assert lang_tag_filter("<|ko|><|NEUTRAL|><|Speech|><|withitn|>안녕 하세요 반가워요")["content"] == "안녕 하세요 반가워요" + assert lang_tag_filter("<|en|><|NEUTRAL|><|Speech|><|withitn|>Hello world.")["content"] == "Hello world." + + +def test_plain_text_without_tags(): + assert lang_tag_filter("plain text") == {"content": "plain text"}