diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 135cfbd65..a0d515a2f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,24 +20,58 @@ jobs: python-version: '3.10' cache: 'pip' cache-dependency-path: 'main/xiaozhi-server/requirements.txt' + - name: Install system dependencies + run: | + # opuslib_next 是 ctypes 绑定，pip 装上的是纯 Python 包，真正的编解码 + # 靠系统里的 libopus.so。runner 上默认没有，import 会失败，音频相关的 + # 用例直接报错。正式镜像 Dockerfile-server-base 里也是装 libopus0 的 + sudo apt-get update + sudo apt-get install -y --no-install-recommends libopus0 - name: Install dependencies working-directory: main/xiaozhi-server run: | pip install -r requirements.txt pip install pytest pytest-asyncio freezegun - - name: Run smoke test + - name: Prepare test config working-directory: main/xiaozhi-server run: | - if [ -f tests/test_smoke.py ]; then - pytest tests/test_smoke.py -x -q - else - echo "::warning::Slice 2 not done yet — tests/test_smoke.py missing" - exit 0 - fi + # config/settings.py 在 import 期就要这个文件，缺了整套用例连收集都过不去。 + # data/ 在 .gitignore 里，只能由这一步现造 + mkdir -p data + printf '{}\n' > data/.config.yaml + - name: Run tests + working-directory: main/xiaozhi-server + run: | + # Edge TTS 那条要连微软公网，runner 的出口 IP 会被风控挡掉，先排除。 + # 根治办法是给它打 live 标记，让 mock 模式下自动不收（另开一条改） + pytest tests/ -v -k "not test_tts_text_to_speak_writes_file" java: name: Java (manager-api) runs-on: ubuntu-latest + # 三个 @SpringBootTest 要真连 MySQL 和 Redis，空库即可，Liquibase 会自己建表 + services: + mysql: + image: mysql:8.0 + env: + MYSQL_ROOT_PASSWORD: 123456 + MYSQL_DATABASE: xiaozhi_esp32_server + ports: + - 3306:3306 + options: >- + --health-cmd="mysqladmin ping -h 127.0.0.1 -uroot -p123456" + --health-interval=10s + --health-timeout=5s + --health-retries=20 + redis: + image: redis:8.0 + ports: + - 6379:6379 + options: >- + --health-cmd="redis-cli ping" + --health-interval=10s + --health-timeout=5s + --health-retries=20 steps: - uses: actions/checkout@v4 - uses: actions/setup-java@v4 @@ -47,7 +81,12 @@ jobs: cache: 'maven' - name: Run tests working-directory: main/manager-api - run: mvn -B -q test -DfailIfNoTests=false + run: mvn -B test -DskipTests=false -DfailIfNoTests=false + - name: Check the integration tests really ran + working-directory: main/manager-api + # 守卫在依赖不齐时会把整类集成测试跳过，这里盯一下它们有没有真跑， + # 免得门禁被悄悄掏空 + run: bash scripts/check-integration-tests.sh web: name: Vue (manager-web) @@ -63,4 +102,6 @@ jobs: working-directory: main/manager-web run: | npm ci --no-audit --no-fund + npm run test:contract npm run test:unit + npm run build diff --git a/main/manager-api/README.md b/main/manager-api/README.md index 484c28dc0..19683a66b 100644 --- a/main/manager-api/README.md +++ b/main/manager-api/README.md @@ -16,3 +16,28 @@ Vue 3.x # 接口文档 启动后打开：http://localhost:8002/xiaozhi/doc.html +# 跑测试 +测试分两种：纯单元测试，和要真连 MySQL、Redis 的集成测试（`DeviceTest`、`loginControllerTest`、`ModelConfigPersistenceTest`）。 + +集成测试需要先起好 MySQL 8 和 Redis 8。没起的时候这类测试整类跳过，不会让构建变红，日志里会写明缺的是哪个依赖。CI 上不一样：依赖连不上就直接让构建失败，不许跳过，免得门禁被悄悄掏空。 + +本地用 Docker 起这两个依赖： + +```bash +docker run -d --name xiaozhi-mysql -p 3306:3306 \ + -e MYSQL_ROOT_PASSWORD=123456 -e MYSQL_DATABASE=xiaozhi_esp32_server mysql:8.0 +docker run -d --name xiaozhi-redis -p 6379:6379 redis:8.0 +``` + +跑全部测试： + +```bash +mvn -B test -DskipTests=false +``` + +只想编译、不跑测试： + +```bash +mvn -B package -DskipTests=true +``` + diff --git a/main/manager-api/pom.xml b/main/manager-api/pom.xml index 455ee6e11..e94004843 100644 --- a/main/manager-api/pom.xml +++ b/main/manager-api/pom.xml @@ -32,7 +32,7 @@ 4.20.0 4.1.0 3.4.0 - true + false diff --git a/main/manager-api/scripts/check-integration-tests.sh b/main/manager-api/scripts/check-integration-tests.sh new file mode 100755 index 000000000..526b80820 --- /dev/null +++ b/main/manager-api/scripts/check-integration-tests.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# 检查那几个要连 MySQL 和 Redis 的集成测试是不是真的跑了。 +# 守卫要是哪天手滑把它们全跳过了，CI 必须红，不能安安静静少跑一半门禁。 +set -uo pipefail + +REPORT_DIR="${1:-target/surefire-reports}" + +WATCHED=( + "xiaozhi.modules.device.DeviceTest" + "xiaozhi.modules.model.service.impl.ModelConfigPersistenceTest" + "xiaozhi.modules.sys.loginControllerTest" +) + +if [ ! -d "$REPORT_DIR" ]; then + echo "找不到测试报告目录：$REPORT_DIR" + exit 1 +fi + +missing=() +for class in "${WATCHED[@]}"; do + count=0 + for report in "$REPORT_DIR"/TEST-*.xml; do + [ -e "$report" ] || continue + name=$(sed -n 's/.*]*name="\([^"]*\)".*/\1/p' "$report" | head -1) + if [ "$name" = "$class" ]; then + count=$(sed -n 's/.*]*tests="\([0-9]*\)".*/\1/p' "$report" | head -1) + count=${count:-0} + break + fi + done + if [ "$count" -eq 0 ]; then + missing+=("$class") + else + echo "OK $class 跑了 $count 条" + fi +done + +if [ ${#missing[@]} -ne 0 ]; then + echo "这几个集成测试一条都没跑：${missing[*]}" + echo "守卫把门禁掏空了，构建必须失败" + exit 1 +fi + +echo "集成测试都跑过了" diff --git a/main/manager-api/src/test/java/xiaozhi/modules/correctword/service/impl/CorrectWordFileServiceImplTest.java b/main/manager-api/src/test/java/xiaozhi/modules/correctword/service/impl/CorrectWordFileServiceImplTest.java index 25fbebea7..b41953db6 100644 --- a/main/manager-api/src/test/java/xiaozhi/modules/correctword/service/impl/CorrectWordFileServiceImplTest.java +++ b/main/manager-api/src/test/java/xiaozhi/modules/correctword/service/impl/CorrectWordFileServiceImplTest.java @@ -14,9 +14,14 @@ import java.util.Collections; import java.util.Date; import java.util.List; +import org.apache.ibatis.builder.MapperBuilderAssistant; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.springframework.test.util.ReflectionTestUtils; +import com.baomidou.mybatisplus.core.MybatisConfiguration; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; + import xiaozhi.modules.agent.dao.AgentCorrectWordMappingDao; import xiaozhi.modules.agent.entity.AgentCorrectWordMappingEntity; import xiaozhi.modules.correctword.dao.CorrectWordFileDao; @@ -37,6 +42,20 @@ class CorrectWordFileServiceImplTest { private final CorrectWordFileServiceImpl service = new CorrectWordFileServiceImpl(fileDao, itemDao, mappingDao); + @BeforeAll + static void registerEntityMetadata() { + // 这条用例不起 Spring 上下文，MyBatis-Plus 的实体元数据没人注册， + // LambdaQueryWrapper 拼条件时找不到 lambda 缓存就抛异常。 + // 全量跑的时候是前面的上下文测试顺带注册好的，单独跑就露馅。 + // 这里自己补上，单独跑和跟着全量跑结果一样。 + TableInfoHelper.initTableInfo( + new MapperBuilderAssistant(new MybatisConfiguration(), ""), + CorrectWordItemEntity.class); + TableInfoHelper.initTableInfo( + new MapperBuilderAssistant(new MybatisConfiguration(), ""), + CorrectWordFileEntity.class); + } + @Test void getAllItemsByAgentId_returns_empty_list_when_no_mappings() { when(mappingDao.selectByAgentId(AGENT_ID)).thenReturn(Collections.emptyList()); diff --git a/main/manager-api/src/test/java/xiaozhi/modules/device/DeviceTest.java b/main/manager-api/src/test/java/xiaozhi/modules/device/DeviceTest.java index 768818439..c15a51437 100644 --- a/main/manager-api/src/test/java/xiaozhi/modules/device/DeviceTest.java +++ b/main/manager-api/src/test/java/xiaozhi/modules/device/DeviceTest.java @@ -3,6 +3,7 @@ package xiaozhi.modules.device; import java.util.HashMap; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -15,6 +16,7 @@ import xiaozhi.common.exception.RenException; import xiaozhi.common.redis.RedisUtils; import xiaozhi.modules.sys.dto.SysUserDTO; import xiaozhi.modules.sys.service.SysUserService; +import xiaozhi.testsupport.IntegrationTestEnvironment; @Slf4j @SpringBootTest @@ -22,6 +24,11 @@ import xiaozhi.modules.sys.service.SysUserService; @DisplayName("设备测试") public class DeviceTest { + @BeforeAll + static void checkDependencies() { + IntegrationTestEnvironment.requireDatabaseAndRedis(); + } + @Autowired private RedisUtils redisUtils; @Autowired diff --git a/main/manager-api/src/test/java/xiaozhi/modules/model/service/impl/ModelConfigPersistenceTest.java b/main/manager-api/src/test/java/xiaozhi/modules/model/service/impl/ModelConfigPersistenceTest.java index 279d78ae7..5e9ae4b11 100644 --- a/main/manager-api/src/test/java/xiaozhi/modules/model/service/impl/ModelConfigPersistenceTest.java +++ b/main/manager-api/src/test/java/xiaozhi/modules/model/service/impl/ModelConfigPersistenceTest.java @@ -5,6 +5,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; import java.util.UUID; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; import org.springframework.beans.factory.annotation.Autowired; @@ -19,12 +20,18 @@ import xiaozhi.common.redis.RedisUtils; import xiaozhi.modules.model.dto.ModelConfigBodyDTO; import xiaozhi.modules.model.dto.ModelConfigDTO; import xiaozhi.modules.model.service.ModelConfigService; +import xiaozhi.testsupport.IntegrationTestEnvironment; @SpringBootTest @ActiveProfiles("dev") @Transactional class ModelConfigPersistenceTest { + @BeforeAll + static void checkDependencies() { + IntegrationTestEnvironment.requireDatabaseAndRedis(); + } + @Autowired private ModelConfigService service; diff --git a/main/manager-api/src/test/java/xiaozhi/modules/sys/loginControllerTest.java b/main/manager-api/src/test/java/xiaozhi/modules/sys/loginControllerTest.java index 4c5128842..885fb0db5 100644 --- a/main/manager-api/src/test/java/xiaozhi/modules/sys/loginControllerTest.java +++ b/main/manager-api/src/test/java/xiaozhi/modules/sys/loginControllerTest.java @@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.when; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; @@ -18,12 +19,18 @@ import xiaozhi.modules.security.dto.LoginDTO; import xiaozhi.modules.security.dto.SmsVerificationDTO; import xiaozhi.modules.sys.dto.RetrievePasswordDTO; import xiaozhi.modules.sys.service.SysUserService; +import xiaozhi.testsupport.IntegrationTestEnvironment; @Slf4j @SpringBootTest @ActiveProfiles("dev") class loginControllerTest { + @BeforeAll + static void checkDependencies() { + IntegrationTestEnvironment.requireDatabaseAndRedis(); + } + @Autowired LoginController loginController; diff --git a/main/manager-api/src/test/java/xiaozhi/testsupport/IntegrationTestEnvironment.java b/main/manager-api/src/test/java/xiaozhi/testsupport/IntegrationTestEnvironment.java new file mode 100644 index 000000000..32e33637e --- /dev/null +++ b/main/manager-api/src/test/java/xiaozhi/testsupport/IntegrationTestEnvironment.java @@ -0,0 +1,149 @@ +package xiaozhi.testsupport; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.time.Duration; + +import org.junit.jupiter.api.Assumptions; +import org.springframework.boot.env.YamlPropertySourceLoader; +import org.springframework.core.env.ConfigurableEnvironment; +import org.springframework.core.env.PropertySource; +import org.springframework.core.env.PropertySourcesPropertyResolver; +import org.springframework.core.env.StandardEnvironment; +import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.Resource; + +import io.lettuce.core.RedisClient; +import io.lettuce.core.RedisURI; +import io.lettuce.core.api.StatefulRedisConnection; +import lombok.extern.slf4j.Slf4j; + +/** + * 几个 @SpringBootTest 要真连 MySQL 和 Redis。本地没起这两个依赖时， + * 整类测试跳过，别让 Spring 上下文起不来变成一堆看不懂的报错； + * CI 里依赖必须真的在，探不通就直接让构建红。 + */ +@Slf4j +public final class IntegrationTestEnvironment { + + /** 集成测试用的配置文件，跟应用本身读的是同一套 */ + private static final String[] CONFIG_FILES = {"application.yml", "application-dev.yml"}; + + private static Boolean available; + private static String problem; + + private IntegrationTestEnvironment() { + } + + /** + * 放在要连库的测试类的 @BeforeAll 里。 + * 本地依赖不齐就整类跳过，CI 里依赖不齐就直接失败。 + */ + public static void requireDatabaseAndRedis() { + String issue = probe(); + if (issue == null) { + return; + } + if (isCi()) { + throw new IllegalStateException( + "CI 环境里" + issue + "，这几个集成测试必须真的跑起来，不能跳过"); + } + log.warn("跳过依赖 MySQL 和 Redis 的集成测试：{}", issue); + Assumptions.assumeTrue(false, issue); + } + + /** GitHub Actions 上会带 CI=true */ + public static boolean isCi() { + String value = System.getenv("CI"); + if (value == null) { + return false; + } + String trimmed = value.trim(); + return !trimmed.isEmpty() && !"false".equalsIgnoreCase(trimmed); + } + + /** 依赖都齐返回 null，不齐返回缺什么 */ + static synchronized String probe() { + if (available != null) { + return problem; + } + PropertySourcesPropertyResolver resolver = resolver(); + StringBuilder issues = new StringBuilder(); + + String url = firstNonBlank(resolver.getProperty("spring.datasource.druid.url", ""), + resolver.getProperty("spring.datasource.url", "")); + String username = firstNonBlank(resolver.getProperty("spring.datasource.druid.username", ""), + resolver.getProperty("spring.datasource.username", "")); + String password = firstNonBlank(resolver.getProperty("spring.datasource.druid.password", ""), + resolver.getProperty("spring.datasource.password", "")); + if (url.isBlank()) { + issues.append("配置里找不到数据源地址；"); + } else { + try { + DriverManager.setLoginTimeout(5); + try (Connection connection = DriverManager.getConnection(url, username, password)) { + connection.isValid(3); + } + } catch (Exception e) { + issues.append("连不上 MySQL（").append(e.getMessage()).append("）；"); + } + } + + String redisHost = resolver.getProperty("spring.data.redis.host", "localhost"); + int redisPort = resolver.getProperty("spring.data.redis.port", Integer.class, 6379); + String redisPassword = resolver.getProperty("spring.data.redis.password", ""); + try { + RedisURI.Builder builder = RedisURI.builder() + .withHost(redisHost) + .withPort(redisPort) + .withTimeout(Duration.ofSeconds(3)); + if (redisPassword != null && !redisPassword.isBlank()) { + builder.withPassword(redisPassword.toCharArray()); + } + RedisClient client = RedisClient.create(builder.build()); + try { + StatefulRedisConnection connection = client.connect(); + connection.sync().ping(); + connection.close(); + } finally { + client.shutdown(); + } + } catch (Exception e) { + issues.append("连不上 Redis（").append(e.getMessage()).append("）；"); + } + + if (issues.length() == 0) { + available = Boolean.TRUE; + problem = null; + } else { + available = Boolean.FALSE; + problem = issues.toString(); + } + return problem; + } + + /** 读应用自己那份配置，命令行上的 -D 参数优先级更高，跟应用保持一致 */ + private static PropertySourcesPropertyResolver resolver() { + ConfigurableEnvironment environment = new StandardEnvironment(); + YamlPropertySourceLoader loader = new YamlPropertySourceLoader(); + for (String name : CONFIG_FILES) { + Resource resource = new ClassPathResource(name); + if (!resource.exists()) { + continue; + } + try { + for (PropertySource source : loader.load(name, resource)) { + environment.getPropertySources().addLast(source); + } + } catch (IOException e) { + log.warn("读配置 {} 失败：{}", name, e.getMessage()); + } + } + return new PropertySourcesPropertyResolver(environment.getPropertySources()); + } + + private static String firstNonBlank(String first, String second) { + return first != null && !first.isBlank() ? first : (second == null ? "" : second); + } +} diff --git a/main/xiaozhi-server/tests/conftest.py b/main/xiaozhi-server/tests/conftest.py index a269bb77f..24af688b6 100644 --- a/main/xiaozhi-server/tests/conftest.py +++ b/main/xiaozhi-server/tests/conftest.py @@ -47,10 +47,29 @@ if str(_PROJECT_ROOT) not in sys.path: # --------------------------------------------------------------------------- try: import opuslib_next # noqa: F401 -except Exception: +except Exception as _opus_import_error: + # opuslib_next 是 ctypes 绑定，pip 装上的是纯 Python 包，真正的编解码靠系统里的 + # libopus.so。runner 上没装 libopus 时 import 会失败，这里换桩顶上。 + # 桩只能撑住 import 链路，真要编解码得先把 libopus 装上 + # （apt-get install libopus0，正式镜像 Dockerfile-server-base 里也是这么装的）。 + print( + f"[conftest] opuslib_next 不可用，改用桩：{_opus_import_error}", + flush=True, + ) + + def _opus_unavailable(_self, *_args, **_kwargs): + raise RuntimeError( + "libopus 没装，Opus 编解码跑不了。" + "装法：apt-get install libopus0（正式镜像 Dockerfile-server-base 里也是这么装的）" + ) + _stub = types.ModuleType("opuslib_next") - _stub.Encoder = type("Encoder", (), {"__init__": lambda *a, **kw: None}) - _stub.Decoder = type("Decoder", (), {"__init__": lambda *a, **kw: None}) + _stub.Encoder = type( + "Encoder", (), {"__init__": lambda *a, **kw: None, "encode": _opus_unavailable} + ) + _stub.Decoder = type( + "Decoder", (), {"__init__": lambda *a, **kw: None, "decode": _opus_unavailable} + ) _stub.APPLICATION_AUDIO = 0 _stub.OpusError = type("OpusError", (Exception,), {}) _stub.constants = types.SimpleNamespace() @@ -323,35 +342,25 @@ def _install_mock_layer() -> None: asyncio.sleep = _fast_sleep # type: ignore[assignment] - # 网络层 mock：如果真实 httpx 已加载，把它的 AsyncClient.send 替换成 + # 网络层 mock：如果真实 httpx 已加载，把它的 send 替换成 # 立即抛 ConnectError 的桩，触发 ``_should_retry`` 走到 ``raise``， # 但避免真发请求。Live 模式完全不安装这层。 if not LIVE_API_TESTS: try: - import httpx as _httpx # noqa: F401 + import httpx as _httpx - class _FakeAsyncClient: - def __init__(self, *args, **kwargs): - pass + async def _fake_async_send(self, request, **kwargs): + raise _httpx.ConnectError("MOCK: connect refused") - async def send(self, *args, **kwargs): - raise _httpx.ConnectError("MOCK: connect refused") + def _fake_send(self, request, **kwargs): + raise _httpx.ConnectError("MOCK: connect refused") - async def aclose(self): - return None - - class _FakeClient: - def __init__(self, *args, **kwargs): - pass - - def send(self, *args, **kwargs): - raise _httpx.ConnectError("MOCK: connect refused") - - def close(self): - return None - - _httpx.AsyncClient = _FakeAsyncClient # type: ignore[misc] - _httpx.Client = _FakeClient # type: ignore[misc] + # 只换 send 这一个方法，不换整个类。有些库（openai 2.x 的 + # SyncHttpxClientWrapper）继承 httpx.Client 再自己拼请求， + # 整个类被换掉的话它们继承到的是假类，拼请求时直接 + # AttributeError，报错还完全看不出是 mock 层的问题 + _httpx.AsyncClient.send = _fake_async_send # type: ignore[assignment] + _httpx.Client.send = _fake_send # type: ignore[assignment] except ImportError: pass diff --git a/main/xiaozhi-server/tests/test_llm.py b/main/xiaozhi-server/tests/test_llm.py index 931a16f9f..5a83f5826 100644 --- a/main/xiaozhi-server/tests/test_llm.py +++ b/main/xiaozhi-server/tests/test_llm.py @@ -65,6 +65,13 @@ def _build_provider(): ptype = get_provider_type("LLM", name) if not ptype: pytest.skip(f"LLM '{name}' 的 type 字段缺失") + if not LIVE_API_TESTS: + # mock 模式把网络拦了，这条链路拿不到真实回复，跑下来只是一次连接失败。 + # 真要验这条链路，用 RUN_LIVE_API_TESTS=1 配真 key 跑 + pytest.skip( + f"mock 模式不发真请求，跳过 LLM '{name}' 的链路验证。" + "真请求模式：export RUN_LIVE_API_TESTS=1 并填真 key 后再跑。" + ) from core.utils.llm import create_instance return create_instance(ptype, cfg)