Merge pull request #3434 from xinnan-tech/py-test-asr

fix: SenseVoice 日语识别结果按词带空格 (#3412)

main/xiaozhi-server/core/providers/asr/utils.py

@@ -51,6 +51,14 @@ def lang_tag_filter(text: str) -> dict:
 
     # 移除所有 <|...|> 格式的标签，获取纯文本
     clean_text = re.sub(tag_pattern, "", text).strip()
+    # SenseVoice 对日语等偶尔按词输出空格（如「今日 の こと を」），去掉中日文字符之间的空格；
+    # 韩文、英文的词间空格不受影响
+    clean_text = re.sub(
+        r"(?<=[\u3000-\u303f\u3040-\u30ff\u4e00-\u9fff\uff01-\uff60])\s+"
+        r"(?=[\u3000-\u303f\u3040-\u30ff\u4e00-\u9fff\uff01-\uff60])",
+        "",
+        clean_text,
+    )
 
     # 保持返回结构一致，避免调用方把纯文本误当成字典访问。
     if not all_tags:

main/xiaozhi-server/tests/core/providers/asr/test_lang_tag_filter.py

@@ -0,0 +1,23 @@
+"""lang_tag_filter: SenseVoice tag parsing and CJK word-spacing cleanup."""
+from core.providers.asr.utils import lang_tag_filter
+
+
+def test_japanese_word_spaces_are_removed():
+    result = lang_tag_filter("<|ja|><|NEUTRAL|><|Speech|><|withitn|>今日 の こと を 少し 長め に 話し て ください。")
+    assert result["content"] == "今日のことを少し長めに話してください。"
+    assert result["language"] == "ja"
+
+
+def test_chinese_text_is_unchanged():
+    result = lang_tag_filter("<|zh|><|HAPPY|><|Speech|><|withitn|>你好啊，测试测试。")
+    assert result["content"] == "你好啊，测试测试。"
+
+
+def test_spaces_next_to_latin_and_korean_are_kept():
+    assert lang_tag_filter("<|ja|><|NEUTRAL|><|Speech|><|withitn|>OK Google と 言って")["content"] == "OK Google と言って"
+    assert lang_tag_filter("<|ko|><|NEUTRAL|><|Speech|><|withitn|>안녕 하세요 반가워요")["content"] == "안녕 하세요 반가워요"
+    assert lang_tag_filter("<|en|><|NEUTRAL|><|Speech|><|withitn|>Hello world.")["content"] == "Hello world."
+
+
+def test_plain_text_without_tags():
+    assert lang_tag_filter("plain text") == {"content": "plain text"}