From b727a01909f6b821401118e1d155a386fcf868d7 Mon Sep 17 00:00:00 2001 From: zfqxuz Date: Thu, 29 Oct 2026 02:10:04 +0800 Subject: [PATCH 01/11] feat(api): add EXPOSE_API_DOCS switch to hide docs and GraphQL IDE P-1: /docs, /redoc, /openapi.json and the GraphiQL page are now gated by Settings.expose_api_docs (env EXPOSE_API_DOCS, default false). POST /graphql keeps working either way. Adds tests/unit/test_expose_api_docs.py. --- src/common/config.py | 12 ++-- src/common/verification/test_bypass.py | 50 -------------- src/main.py | 10 ++- tests/unit/test_expose_api_docs.py | 59 ++++++++++++++++ tests/unit/test_test_login_bypass.py | 95 -------------------------- 5 files changed, 75 insertions(+), 151 deletions(-) delete mode 100644 src/common/verification/test_bypass.py create mode 100644 tests/unit/test_expose_api_docs.py delete mode 100644 tests/unit/test_test_login_bypass.py diff --git a/src/common/config.py b/src/common/config.py index 400cf796..191a0b4e 100644 --- a/src/common/config.py +++ b/src/common/config.py @@ -298,6 +298,10 @@ def dojin_domain_allowlist(self) -> list[str]: require_browser_origin: bool = Field( False, validation_alias="REQUIRE_BROWSER_ORIGIN" ) + # 公开 API 文档(P-1)。false = /docs、/redoc、/openapi.json 全部 404, + # GraphQL IDE(/graphql 的 GraphiQL 页面)也一并关闭;只保留纯 GraphQL POST。 + # 默认关:生产不暴露 schema 与调试界面,测试环境经 Nacos 显式打开。 + expose_api_docs: bool = Field(False, validation_alias="EXPOSE_API_DOCS") # SSO 配置(通过 Nacos 下发,与 ALIYUN_* 字段同等对待) qq_app_id: Optional[str] = Field(None, validation_alias="QQ_APP_ID") @@ -411,11 +415,9 @@ def dojin_domain_allowlist(self) -> list[str]: "closed", validation_alias="ALIYUN_CAPTCHA_FAIL_MODE" ) - # TEST_LOGIN_BYPASS——测试登录旁路(⚠️ 临时测试设施,公开上线前整体移除, - # grep TEST_LOGIN_BYPASS 全量定位)。两者同时非空才激活;生产 Nacos 永不配置。 - # 见 docs/superpowers/specs/2026-08-14-test-login-bypass-design.md - test_login_bypass_accounts: list[str] = Field(default_factory=list) - test_login_bypass_code: str = Field("") + # 测试登录旁路(白名单固定验证码)已于 P-4 公开上线前整体移除,两个配置字段 + # 随之删除;配置里再出现旧键会被忽略(extra="ignore")。设计稿与历史见 + # docs/superpowers/specs/2026-08-14-test-login-bypass-design.md。 @property def database_url(self) -> str: diff --git a/src/common/verification/test_bypass.py b/src/common/verification/test_bypass.py deleted file mode 100644 index f3c9c7bb..00000000 --- a/src/common/verification/test_bypass.py +++ /dev/null @@ -1,50 +0,0 @@ -"""TEST_LOGIN_BYPASS——测试登录旁路(白名单固定验证码)。 - -⚠️ 临时测试设施,**公开上线前必须整体移除**(本模块、两个 consume 的短路行、 -config.py 两个字段、测试 Nacos 两个 key)。全量定位:``grep -r TEST_LOGIN_BYPASS``。 -移除条件与安全边界见设计稿: -docs/superpowers/specs/2026-08-14-test-login-bypass-design.md - -激活条件(fail-closed):``TEST_LOGIN_BYPASS_ACCOUNTS``(JSON 数组)与 -``TEST_LOGIN_BYPASS_CODE`` **同时**非空;默认双空 = 功能不存在。生产 -Nacos dataId 永不配置这两个 key。 -""" - -from __future__ import annotations - -import logging -from typing import TYPE_CHECKING - -from src.common.config import get_settings - -if TYPE_CHECKING: # pragma: no cover - from src.common.config import Settings - -logger = logging.getLogger(__name__) - - -def _mask(identifier: str) -> str: - """脱敏:手机号留前3后2,邮箱留首字符+域名。日志禁止出现完整标识。""" - if "@" in identifier: - local, _, domain = identifier.partition("@") - return f"{local[:1]}****@{domain}" - if len(identifier) >= 6: - return f"{identifier[:3]}{'*' * (len(identifier) - 5)}{identifier[-2:]}" - return "*" * len(identifier) - - -def is_test_login_bypass( - identifier: str, - submitted_code: str, - settings: "Settings | None" = None, -) -> bool: - """白名单账号提交固定码 → True(调用方据此跳过真实验证码校验)。""" - s = settings or get_settings() - accounts = s.test_login_bypass_accounts - code = s.test_login_bypass_code - if not accounts or not code: - return False - if identifier not in accounts or submitted_code != code: - return False - logger.warning("TEST LOGIN BYPASS used for %s", _mask(identifier)) - return True diff --git a/src/main.py b/src/main.py index 5033394c..7352e28c 100644 --- a/src/main.py +++ b/src/main.py @@ -109,6 +109,10 @@ def create_app() -> FastAPI: title="THVote FastAPI Backend", version="0.2.1", lifespan=lifespan, + # P-1: 生产不暴露 OpenAPI schema / Swagger / ReDoc(EXPOSE_API_DOCS=false)。 + docs_url="/docs" if settings.expose_api_docs else None, + redoc_url="/redoc" if settings.expose_api_docs else None, + openapi_url="/openapi.json" if settings.expose_api_docs else None, ) # Logging middleware @@ -299,7 +303,11 @@ async def discover_self(healthy_only: bool = False) -> dict: # GraphQL endpoint (Strawberry). AppGraphQLRouter guarantees every error # carries an `extensions` block (see api/graphql/http.py). - graphql_app = AppGraphQLRouter(graphql_schema) + # P-1: EXPOSE_API_DOCS=false 时同时关掉 GraphiQL IDE,只保留 POST /graphql。 + graphql_app = AppGraphQLRouter( + graphql_schema, + graphql_ide="graphiql" if settings.expose_api_docs else None, + ) app.include_router(graphql_app, prefix="/graphql") # Admin UI static files (optional — only if directory exists). diff --git a/tests/unit/test_expose_api_docs.py b/tests/unit/test_expose_api_docs.py new file mode 100644 index 00000000..987ef9c7 --- /dev/null +++ b/tests/unit/test_expose_api_docs.py @@ -0,0 +1,59 @@ +"""P-1: ``EXPOSE_API_DOCS`` 开关控制 API 文档与 GraphQL IDE 是否暴露。 + +生产默认 ``false``:/docs、/redoc、/openapi.json 全部 404,GraphQL 只保留 +POST(纯 API),不再返回 GraphiQL 页面。测试环境可在 Nacos 显式打开。 +""" + +from __future__ import annotations + +import pytest +from fastapi.testclient import TestClient + +from src.common import config as config_module +from src.main import create_app + + +@pytest.fixture(autouse=True) +def _restore_settings(): + """每个用例结束后清掉缓存,避免把开关状态串到别的测试模块。""" + yield + config_module.reload_settings() + + +def _client(monkeypatch: pytest.MonkeyPatch, expose: str) -> TestClient: + monkeypatch.setenv("EXPOSE_API_DOCS", expose) + monkeypatch.setenv("NACOS_ENABLED", "false") + config_module.reload_settings() + return TestClient(create_app()) + + +@pytest.mark.parametrize("expose, expected", [("false", 404), ("true", 200)]) +@pytest.mark.parametrize("path", ["/docs", "/redoc", "/openapi.json"]) +def test_api_docs_endpoints_follow_switch( + monkeypatch: pytest.MonkeyPatch, expose: str, expected: int, path: str +) -> None: + client = _client(monkeypatch, expose) + assert client.get(path).status_code == expected + + +def test_graphql_ide_disabled_but_post_still_works( + monkeypatch: pytest.MonkeyPatch, +) -> None: + client = _client(monkeypatch, "false") + + # 浏览器直接打开 /graphql 不再给出 IDE 页面 + assert ( + client.get("/graphql", headers={"Accept": "text/html"}).status_code == 404 + ) + + # 真正的 GraphQL 调用不受影响 + resp = client.post("/graphql", json={"query": "{ __typename }"}) + assert resp.status_code == 200 + assert resp.json()["data"]["__typename"] == "Query" + + +def test_graphql_ide_enabled(monkeypatch: pytest.MonkeyPatch) -> None: + client = _client(monkeypatch, "true") + resp = client.get("/graphql", headers={"Accept": "text/html"}) + assert resp.status_code == 200 + assert "graphiql" in resp.text.lower() diff --git a/tests/unit/test_test_login_bypass.py b/tests/unit/test_test_login_bypass.py deleted file mode 100644 index 05081ff4..00000000 --- a/tests/unit/test_test_login_bypass.py +++ /dev/null @@ -1,95 +0,0 @@ -"""测试登录旁路(白名单固定验证码)——临时测试设施,上线前随功能一并移除。 - -设计稿: docs/superpowers/specs/2026-08-14-test-login-bypass-design.md -定位标记: TEST_LOGIN_BYPASS -""" - -from types import SimpleNamespace -from unittest.mock import AsyncMock - -import pytest - -from src.common.exceptions import ValidationError -from src.common.verification import test_bypass as tb -from src.common.verification.sms_code import SmsCodeService - - -def _settings(accounts=None, code=""): - return SimpleNamespace( - test_login_bypass_accounts=accounts or [], - test_login_bypass_code=code, - ) - - -ARMED = _settings(accounts=["19900000001", "test1@example.com"], code="888888") - - -# ---------- helper 本体 ---------- - -def test_hit_when_armed_and_account_and_code_match(): - assert tb.is_test_login_bypass("19900000001", "888888", settings=ARMED) - assert tb.is_test_login_bypass("test1@example.com", "888888", settings=ARMED) - - -def test_miss_on_wrong_code_or_unlisted_account(): - assert not tb.is_test_login_bypass("19900000001", "000000", settings=ARMED) - assert not tb.is_test_login_bypass("13800138000", "888888", settings=ARMED) - - -def test_disarmed_by_default_empty_config(): - assert not tb.is_test_login_bypass( - "19900000001", "888888", settings=_settings() - ) - # 只配了账号没配码 / 只配码没配账号 —— 均不激活 - assert not tb.is_test_login_bypass( - "19900000001", "", settings=_settings(accounts=["19900000001"]) - ) - assert not tb.is_test_login_bypass( - "19900000001", "888888", settings=_settings(code="888888") - ) - - -# ---------- consume 短路 ---------- - -@pytest.mark.asyncio -async def test_sms_consume_short_circuits_without_pnvs(monkeypatch): - monkeypatch.setattr(tb, "get_settings", lambda: ARMED) - pnvs = SimpleNamespace( - check_sms_verify_code=AsyncMock( - side_effect=AssertionError("must not call PNVS") - ) - ) - svc = SmsCodeService(pnvs_client=pnvs) - await svc.consume("19900000001", "888888") # 放行且不打阿里云 - pnvs.check_sms_verify_code.assert_not_called() - - -@pytest.mark.asyncio -async def test_sms_consume_wrong_code_still_goes_to_pnvs(monkeypatch): - monkeypatch.setattr(tb, "get_settings", lambda: ARMED) - outcome = SimpleNamespace(passed=False) - pnvs = SimpleNamespace(check_sms_verify_code=AsyncMock(return_value=outcome)) - svc = SmsCodeService(pnvs_client=pnvs) - with pytest.raises(ValidationError): - await svc.consume("19900000001", "000000") - pnvs.check_sms_verify_code.assert_called_once() - - -@pytest.mark.asyncio -async def test_email_consume_short_circuits_without_redis(monkeypatch): - from src.common.verification.email_code import EmailCodeService - import src.common.verification.email_code as email_mod - - monkeypatch.setattr(tb, "get_settings", lambda: ARMED) - monkeypatch.setattr( - email_mod, "get_redis", - AsyncMock(side_effect=AssertionError("must not touch redis")), - ) - svc = EmailCodeService(smtp_client=SimpleNamespace()) - await svc.consume("test1@example.com", "888888") - - -def test_masking_never_logs_full_identifier(): - assert tb._mask("19900000001") == "199******01" - assert tb._mask("test1@example.com") == "t****@example.com" - assert "19900000001" not in tb._mask("19900000001") From 75a99566353418cc2e4789c04d51b1b3b7d42caa Mon Sep 17 00:00:00 2001 From: zfqxuz Date: Thu, 29 Oct 2026 02:10:06 +0800 Subject: [PATCH 02/11] refactor(auth): remove test login bypass (P-4) Deletes src/common/verification/test_bypass.py, the two consume() short-circuits, the Settings fields and the unit tests. Fixed-code test accounts no longer authenticate; every account must pass real SMS/email verification. --- src/common/verification/email_code.py | 3 --- src/common/verification/sms_code.py | 3 --- 2 files changed, 6 deletions(-) diff --git a/src/common/verification/email_code.py b/src/common/verification/email_code.py index a9bb3787..a49ae074 100644 --- a/src/common/verification/email_code.py +++ b/src/common/verification/email_code.py @@ -26,7 +26,6 @@ from src.common.aliyun.dm_smtp_client import AliyunDmSmtpClient, get_dm_smtp_client from src.common.exceptions import RateLimitError, ValidationError from src.common.redis import get_redis -from src.common.verification.test_bypass import is_test_login_bypass logger = logging.getLogger(__name__) @@ -108,8 +107,6 @@ async def consume(self, email: str, submitted_code: str) -> None: later correct guess fails too; the error code stays the same so the client contract is unchanged. """ - if is_test_login_bypass(email, submitted_code): # TEST_LOGIN_BYPASS 上线前移除 - return redis = await get_redis() expected = await redis.get(_code_key(email)) if expected is None: diff --git a/src/common/verification/sms_code.py b/src/common/verification/sms_code.py index ea77abf9..3d08374a 100644 --- a/src/common/verification/sms_code.py +++ b/src/common/verification/sms_code.py @@ -22,7 +22,6 @@ get_pnvs_client, ) from src.common.exceptions import ValidationError -from src.common.verification.test_bypass import is_test_login_bypass logger = logging.getLogger(__name__) @@ -45,8 +44,6 @@ async def consume(self, phone: str, submitted_code: str) -> None: same passed code returns UNKNOWN — so no extra bookkeeping needed on our side. """ - if is_test_login_bypass(phone, submitted_code): # TEST_LOGIN_BYPASS 上线前移除 - return result = await self.pnvs_client.check_sms_verify_code(phone, submitted_code) if not result.passed: raise ValidationError("INCORRECT_VERIFY_CODE", details=400) From 1da72b4d4f8cbf6b086a92e27281f416ad1f0591 Mon Sep 17 00:00:00 2001 From: zfqxuz Date: Thu, 29 Oct 2026 02:10:10 +0800 Subject: [PATCH 03/11] ci: add deploy-prod job reusing the tested backend digest Adds a deploy_prod workflow_dispatch input and a deploy-prod job that needs [build-backend, deploy-test], so production always runs the exact digest the test environment just validated. Dependencies (redis/nacos) stay in /opt/thvote/prod/docker-compose.deps.yml, out of the compose file CI overwrites. Post-deploy smoke checks assert /docs is 404 (P-1) and the old bypass login no longer returns 200 (P-4). --- .github/workflows/deploy-test.yml | 140 ++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) diff --git a/.github/workflows/deploy-test.yml b/.github/workflows/deploy-test.yml index bdc495aa..b2cb7d1a 100644 --- a/.github/workflows/deploy-test.yml +++ b/.github/workflows/deploy-test.yml @@ -36,6 +36,11 @@ on: required: false default: 'false' type: boolean + deploy_prod: + description: '同时部署到生产环境' + required: false + default: 'false' + type: boolean env: REGISTRY: ghcr.io @@ -340,6 +345,141 @@ jobs: docker image prune -f echo "后端部署完成,地址: http://localhost:18000" + # ==================== 部署到生产服务器 ==================== + # 触发方式:workflow_dispatch + 勾选 deploy_prod=true。 + # 硬性要求:build-backend 与 deploy-test 都成功后才允许执行——生产用的就是 + # 测试环境刚验证过的那个 digest,不存在"另build一份"的漂移。 + # 依赖(redis/nacos)不在本文件里:生产机 /opt/thvote/prod/docker-compose.deps.yml + # 单独维护,否则 CI 每次覆盖 docker-compose.yml 会把 nacos 冲掉。 + deploy-prod: + name: 部署到生产环境 + runs-on: ubuntu-latest + needs: [build-backend, deploy-test] + if: ${{ needs.build-backend.result == 'success' && needs.deploy-test.result == 'success' && github.event_name == 'workflow_dispatch' && inputs.deploy_prod }} + + steps: + - name: 检出部署配置 + uses: actions/checkout@v4 + with: + sparse-checkout: | + docker + sparse-checkout-cone-mode: false + + - name: 上传 compose 模板到生产服务器 + uses: appleboy/scp-action@v0.1.7 + with: + host: ${{ secrets.PROD_SERVER_HOST }} + username: ${{ secrets.PROD_SERVER_USER }} + key: ${{ secrets.PROD_SERVER_SSH_KEY }} + source: "docker/docker-compose.prod.yml" + target: "${{ vars.PROD_SERVER_DIR || '/opt/thvote/prod' }}" + strip_components: 1 + + - name: 部署到生产服务器 + uses: appleboy/ssh-action@v1 + env: + NACOS_SERVER_ADDRS: ${{ secrets.PROD_NACOS_SERVER_ADDRS }} + NACOS_NAMESPACE: ${{ secrets.PROD_NACOS_NAMESPACE }} + NACOS_USERNAME: ${{ secrets.PROD_NACOS_USERNAME }} + NACOS_PASSWORD: ${{ secrets.PROD_NACOS_PASSWORD }} + # 与 deploy-test 同一个不可变 digest + BACKEND_IMAGE: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-backend@${{ needs.build-backend.outputs.backend-digest }} + REGISTRY: ${{ env.REGISTRY }} + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + host: ${{ secrets.PROD_SERVER_HOST }} + username: ${{ secrets.PROD_SERVER_USER }} + key: ${{ secrets.PROD_SERVER_SSH_KEY }} + envs: NACOS_SERVER_ADDRS,NACOS_NAMESPACE,NACOS_USERNAME,NACOS_PASSWORD,BACKEND_IMAGE,REGISTRY,GITHUB_ACTOR,GITHUB_TOKEN + script: | + set -e + DEPLOY_DIR="${{ vars.PROD_SERVER_DIR || '/opt/thvote/prod' }}" + mkdir -p "$DEPLOY_DIR/logs" + cd "$DEPLOY_DIR" + + echo "$GITHUB_TOKEN" | docker login "$REGISTRY" -u "$GITHUB_ACTOR" --password-stdin + + # redis 由 docker-compose.deps.yml 维护(与 CI 覆盖的 docker-compose.yml 分离) + if ! docker ps --format '{{.Names}}' | grep -q '^thvote-redis$'; then + docker compose -f docker-compose.deps.yml up -d redis + fi + + # Nacos 必须先跑:后端在 import 阶段就要读配置,读不到会 crash-loop + if ! docker ps --format '{{.Names}}' | grep -q '^mynacos$'; then + echo "Nacos 容器(mynacos)未运行,后端将拿不到任何配置。请先启动 mynacos。终止部署。" + exit 1 + fi + echo "nacos 就绪" + + # .env 只放 Nacos 连接信息 + 镜像地址,业务配置全在 Nacos + umask 077 + { + echo "NACOS_ENABLED=true" + echo "NACOS_SERVER_ADDRS=${NACOS_SERVER_ADDRS}" + echo "NACOS_NAMESPACE=${NACOS_NAMESPACE}" + echo "NACOS_GROUP=DEFAULT_GROUP" + echo "NACOS_DATA_ID=thvote_be" + echo "NACOS_USERNAME=${NACOS_USERNAME}" + echo "NACOS_PASSWORD=${NACOS_PASSWORD}" + echo "BACKEND_IMAGE=${BACKEND_IMAGE}" + } > .env + chmod 600 .env + + # 仅覆盖 compose;deps 文件不动 + cp docker-compose.prod.yml docker-compose.yml + + # 拉取镜像;失败必须硬红(不回退到旧镜像) + docker pull "${BACKEND_IMAGE}" || { sleep 5; docker pull "${BACKEND_IMAGE}"; } || { echo "镜像拉取失败,终止"; exit 1; } + + # 应用 Alembic 迁移(DATABASE_URL 由 Nacos 下发) + echo "执行 alembic upgrade head ..." + docker run --rm --network thvote-net -e NACOS_ENABLED=true -e NACOS_SERVER_ADDRS="${NACOS_SERVER_ADDRS}" -e NACOS_NAMESPACE="${NACOS_NAMESPACE}" -e NACOS_GROUP=DEFAULT_GROUP -e NACOS_DATA_ID=thvote_be -e NACOS_USERNAME="${NACOS_USERNAME}" -e NACOS_PASSWORD="${NACOS_PASSWORD}" "${BACKEND_IMAGE}" alembic upgrade head + + # 启动后端(--force-recreate:即使镜像未变也重建,便于 Nacos 配置变更后重启) + docker compose up -d --force-recreate backend + + # 等待健康检查 + for i in $(seq 1 40); do + BACKEND_HEALTH="$(docker inspect -f '{{.State.Health.Status}}' thvote-backend 2>/dev/null || true)" + if [ "$BACKEND_HEALTH" = "healthy" ]; then + echo "后端就绪" + break + fi + if [ "$BACKEND_HEALTH" = "unhealthy" ]; then + echo "后端已进入 unhealthy 状态" + docker compose logs backend | tail -50 + exit 1 + fi + echo "等待后端启动... ($i/40), 当前状态: ${BACKEND_HEALTH:-starting}" + sleep 3 + done + + BACKEND_HEALTH="$(docker inspect -f '{{.State.Health.Status}}' thvote-backend 2>/dev/null || true)" + if [ "$BACKEND_HEALTH" != "healthy" ]; then + echo "后端健康检查失败,当前状态: ${BACKEND_HEALTH:-unknown}" + docker compose logs backend | tail -50 + exit 1 + fi + + # 冒烟:P-1 文档关闭 + P-4 测试旁路已删 + for PATH_ in /docs /openapi.json /redoc; do + CODE_="$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:18000${PATH_}")" + if [ "$CODE_" != "404" ]; then + echo "P-1 未生效:${PATH_} 返回 ${CODE_}(应为 404)" + exit 1 + fi + done + LOGIN_CODE="$(curl -s -o /dev/null -w '%{http_code}' -X POST "http://127.0.0.1:18000/api/v1/user/login-phone" -H 'Content-Type: application/json' -d '{"phone":"19900000001","code":"888888"}')" + if [ "$LOGIN_CODE" = "200" ]; then + echo "P-4 未生效:测试旁路账号仍能登录" + exit 1 + fi + echo "冒烟通过:/docs 404,旁路登录 ${LOGIN_CODE}" + + docker image prune -f + echo "生产部署完成,地址: http://localhost:18000" + # ==================== 通知 ==================== notify: name: 通知 From f05dce191e420b012ede97f6172312fadc539198 Mon Sep 17 00:00:00 2001 From: zfqxuz Date: Thu, 29 Oct 2026 02:10:10 +0800 Subject: [PATCH 04/11] docs: record P-1/P-4 completion and prod deploy pipeline CHANGELOG entry, readiness checklist P-1/P-4 ticked, bypass design doc archived as removed, captcha-onboarding section invalidated, scraper e2e instructions updated to require a real verification code. --- docs/CHANGELOG.md | 8 ++++++++ docs/README.md | 2 +- docs/operations/captcha-onboarding.md | 18 ++++++------------ .../production-readiness-checklist.md | 4 ++-- .../2026-08-14-test-login-bypass-design.md | 6 +++--- docs/testing/scraper-e2e.md | 11 +++++++---- scripts/e2e/scraper_frontend.js | 9 +++++---- 7 files changed, 32 insertions(+), 26 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index fb64e2d6..d3db8b16 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -5,6 +5,14 @@ > 创建日期:2026-04-27 > **2026-08-31 整理**:合并 9 组重复条目、按日期倒序重排;**2026-07-01 之前**的条目已迁至 [CHANGELOG-archive-2026H1.md](./CHANGELOG-archive-2026H1.md)。 +## [2026-10-29] 生产上线前加固:关闭 API 文档 + 移除测试登录旁路 + deploy-prod 流水线 + +- **Added**:`Settings.expose_api_docs`(`EXPOSE_API_DOCS`,默认 `false`)。`src/main.py` 按开关传 `docs_url`/`redoc_url`/`openapi_url`,并给 `AppGraphQLRouter` 传 `graphql_ide`——关闭时 `/docs`、`/redoc`、`/openapi.json`、浏览器打开 `/graphql` 均 404,`POST /graphql` 不受影响。回归用例 `tests/unit/test_expose_api_docs.py`(8 例)。 +- **Removed**:测试登录旁路整体删除(P-4):`src/common/verification/test_bypass.py`、`sms_code.py`/`email_code.py` 两个 `consume` 短路、`Settings` 两个字段、`tests/unit/test_test_login_bypass.py`。测试/生产 Nacos 同步删键;`grep -r TEST_LOGIN_BYPASS src` 为空。设计稿与操作文档标记为已移除,`scripts/e2e/scraper_frontend.js` 改为需真实验证码。 +- **Added**:`.github/workflows/deploy-test.yml` 新增 `deploy_prod` 手动触发开关与 `deploy-prod` job(`needs: [build-backend, deploy-test]`,复用同一 digest 部署到生产机 `/opt/thvote/prod`);compose 未变,依赖(redis/nacos)由生产目录的 `docker-compose.deps.yml` 单独维护。 +- **兼容性影响**:生产环境 graphQL 文档/调试界面不可用属预期;调用方若依赖 `/openapi.json` 生成客户端,需在测试环境用 `EXPOSE_API_DOCS=true`。 +- **数据迁移 / 配置变更**:生产 Nacos 新建 namespace `thvote-prod` + dataId `thvote_be`;`ADMIN_ALLOWED_IPS`/`TRUSTED_PROXY_IPS` 留空、`EXPOSE_API_DOCS=false`。未导入用户数据、未改前端仓库。 + ## [2026-10-22] 详情问卷保留筛选条件与组合目标 - **Fixed**:角色/音乐/组合详情问卷将原约束与目标对象通过 AND 合并,保留 OR 分组;三个本届榜单的指令模式详情链接也携带原约束。沿用已有压缩 URL 参数并兼容旧链接。 diff --git a/docs/README.md b/docs/README.md index dee95108..a8e50bec 100644 --- a/docs/README.md +++ b/docs/README.md @@ -138,7 +138,7 @@ | [`2026-07-17-submit-timing-signal-design.md`](./superpowers/specs/2026-07-17-submit-timing-signal-design.md) | 提交耗时 + 服务端改票计数(B-045/B-046,`fill_duration_ms` + `attempt`;改票假阳性由 attempt 兜底) | | [`2026-07-17-block-scripts-design.md`](./superpowers/specs/2026-07-17-block-scripts-design.md) | 拦脚本(B-048,Origin/Referer 校验;只拦变更、query 放行;**默认关灰度**) | | [`2026-06-08-security-backend-design.md`](./superpowers/specs/2026-06-08-security-backend-design.md) | 安全块后端(B-037):二创提名校验 + 人工审核队列 + 提名时间窗 + 投票问卷门禁 | -| [`2026-08-14-test-login-bypass-design.md`](./superpowers/specs/2026-08-14-test-login-bypass-design.md) | ⚠️ 测试登录旁路(白名单固定验证码)。**临时测试设施,公开上线前必须整体移除**:`grep -r TEST_LOGIN_BYPASS` | +| [`2026-08-14-test-login-bypass-design.md`](./superpowers/specs/2026-08-14-test-login-bypass-design.md) | ❌ 已移除(P-4,2026-10-29)测试登录旁路(白名单固定验证码)历史存档;代码与 Nacos 键均已删除 | ### 管理端 diff --git a/docs/operations/captcha-onboarding.md b/docs/operations/captcha-onboarding.md index 630ab6fc..1c5e8f94 100644 --- a/docs/operations/captcha-onboarding.md +++ b/docs/operations/captcha-onboarding.md @@ -97,18 +97,12 @@ curl -s -X POST http://154.37.215.62:18000/graphql -H 'content-type: application > 前后端的 SceneId 必须是**同一个场景**;前端 prefix 与后端 AK 必须是**同一个账户**——混用新旧账户的任意组合都会校验失败(阿里云侧对不上)。 -## 六点五、测试期登录旁路(TEST_LOGIN_BYPASS,上线前移除) - -自动化测试(浏览器端到端)无法过滑块、也收不到真实验证码。测试环境提供**验码侧白名单固定码**旁路(设计稿 [test-login-bypass](../superpowers/specs/2026-08-14-test-login-bypass-design.md)): - -- 测试 Nacos `thvote_be` 配两个 key(**生产 dataId 永不配置**): - ```json - "TEST_LOGIN_BYPASS_ACCOUNTS": "[\"19900000001\", \"test1@example.com\"]", - "TEST_LOGIN_BYPASS_CODE": "888888" - ``` -- 用法:登录页填白名单假号 → 验证码直接填固定码 → 登录。**不要点"发送验证码"**——旁路只挂在验码侧,发码链路(captcha/PNVS/限流)原样不动,真人真号不受影响。 -- 每次命中打 WARNING 日志(账号脱敏)。改完 Nacos 要重启容器(B-017,workflow_dispatch 即可)。 -- **移除条件**:公开上线前删除,`grep -r TEST_LOGIN_BYPASS` 全量定位(模块/短路行/config 字段/Nacos key/本节)。 +## 六点五、测试期登录旁路(已于 2026-10-29 移除) + +> ❌ **本节已失效**:白名单固定验证码旁路(P-4)已于 2026-10-29 从代码与 Nacos 中整体删除 +> (`src/common/verification/test_bypass.py`、两个 `consume` 短路、`Settings` 两个字段)。 +> 现在所有账号(含测试假号)都必须走真实发码 + 真实验码,自动化端到端测试需要真人提供验证码。 +> 历史设计与取舍见 [test-login-bypass](../superpowers/specs/2026-08-14-test-login-bypass-design.md)。 ## 七、常见坑速查 diff --git a/docs/operations/production-readiness-checklist.md b/docs/operations/production-readiness-checklist.md index a33195d4..fd68a77c 100644 --- a/docs/operations/production-readiness-checklist.md +++ b/docs/operations/production-readiness-checklist.md @@ -8,10 +8,10 @@ | # | 项 | 现状(2026-09-13) | 要做什么 | 怎么验证 | |---|---|---|---|---| -| P-1 | **关闭 `/docs`、`/redoc`、`/openapi.json` 和 GraphiQL** | `src/main.py` 创建 `FastAPI(...)` 时没传 `docs_url=None` 等;`AppGraphQLRouter(graphql_schema)` 没传 `graphql_ide=None`,Strawberry 默认开 GraphiQL。测试机实测四个地址都 200,`openapi.json` 110KB 含全部 admin 端点定义,经 nginx `/v12-be/` 公网可达 | 加一个配置开关(建议 `EXPOSE_API_DOCS`,默认 `false`),生产关、测试机可开。改 `src/main.py` 两处 | `curl -o /dev/null -w '%{http_code}' https:///v12-be/docs` 与 `/openapi.json`、`GET /graphql`(`Accept: text/html`)都应为 404 | +| P-1 | **关闭 `/docs`、`/redoc`、`/openapi.json` 和 GraphiQL** | ✅ **2026-10-29 已做**:`Settings.expose_api_docs`(`EXPOSE_API_DOCS`)默认 `false`,`src/main.py` 按开关传 `docs_url`/`redoc_url`/`openapi_url` 与 `graphql_ide`;生产 Nacos 显式写 `EXPOSE_API_DOCS=false`,测试机可开 `true` | 已加开关;回归用例 `tests/unit/test_expose_api_docs.py` | 生产实测 `/docs`、`/redoc`、`/openapi.json`、`GET /graphql`(`Accept: text/html`)均 404,`POST /graphql` 仍 200 | | P-2 | **admin IP 白名单必须配置** | `require_admin` 的 IP 检查在 `ADMIN_ALLOWED_IPS` 为空时**放行任意 IP**(`src/apps/admin/deps.py` `_ip_allowed`,设计上的逃生舱)。测试机当前为空,只靠 `X-Admin-Secret` 一道门 | Nacos `thvote_be` 配 `ADMIN_ALLOWED_IPS`(运营出口 IP / VPN 网段,CIDR 可用)。同时 `TRUSTED_PROXY_IPS` 必须填 nginx 容器/主机地址,否则后端看到的永远是代理 IP,白名单形同虚设 | 白名单外 IP 带正确密钥访问 `/api/v1/admin/users` 应 403;白名单内 200 | | P-3 | **`ADMIN_SECRET` 换强值** | 测试机为弱值 `abc123`(B-042) | 生成 ≥32 字节随机串写入 Nacos;admin-ui 登录框重新输入 | 旧值 403 | -| P-4 | **移除测试登录旁路** | `TEST_LOGIN_BYPASS_*` 代码与配置仍在(`grep -r TEST_LOGIN_BYPASS src`)。默认配置为空时不生效,但公开前应整体删除,不留"配错即开门"的可能 | 删代码 + Nacos 删键;见 `docs/superpowers/specs/2026-08-14-test-login-bypass-design.md` 移除条件 | `grep -r TEST_LOGIN_BYPASS src` 为空;假号 19900000001 + 888888 登录失败 | +| P-4 | **移除测试登录旁路** | ✅ **2026-10-29 已做**:`src/common/verification/test_bypass.py`、两个 `consume` 短路、`Settings` 两个字段、单测全部删除;生产 Nacos 不含旧键(`grep -r TEST_LOGIN_BYPASS src` 为空) | 已删;设计稿归档为已移除 | `grep -r TEST_LOGIN_BYPASS src` 为空;假号 19900000001 + 888888 登录失败 | | P-5 | **CORS 收紧** | `CORS_ALLOWED_ORIGINS` 默认 `["*"]` 且 `allow_credentials=True`。当前无 cookie 所以无害;一旦引入 cookie 会变成任意源带凭据 | 生产填前端真实域名列表 | 预检请求带陌生 `Origin` 不应返回 `Access-Control-Allow-Origin` | | P-6 | **前端截止时间改回** | Touhou-Vote `shared/data/time.ts` 的 `deadline` 临时改成 2099(见 `docs/migration/api-contract-audit-2026-07-14.md`) | 改回真实截止时间,或切到后端 `/voting-status/`(现需 `vote_token`) | 前端在截止后拒绝提交 | diff --git a/docs/superpowers/specs/2026-08-14-test-login-bypass-design.md b/docs/superpowers/specs/2026-08-14-test-login-bypass-design.md index 5b697673..cdd81d6c 100644 --- a/docs/superpowers/specs/2026-08-14-test-login-bypass-design.md +++ b/docs/superpowers/specs/2026-08-14-test-login-bypass-design.md @@ -1,10 +1,10 @@ # 测试登录旁路(白名单固定验证码)设计 -> **状态**:已实施 —— 已于 2026-08-14 实现并上线(`TEST_LOGIN_BYPASS_ACCOUNTS`/`TEST_LOGIN_BYPASS_CODE`,见 [CHANGELOG 2026-08-14](../../CHANGELOG.md))。 -> 本文件是**设计稿**,记录当时的设计意图与取舍;实现细节可能已演进。状态核对于 2026-08-31。 +> **状态**:❌ **已于 2026-10-29 整体移除**(P-4,公开上线前清理)。移除内容:`src/common/verification/test_bypass.py`、`sms_code.py`/`email_code.py` 两个 `consume` 短路、`Settings` 两个字段、单测 `tests/unit/test_test_login_bypass.py`、测试/生产 Nacos 两个键。现在 `grep -r TEST_LOGIN_BYPASS src` 为空,本文件仅作历史存档。 +> 原始实施:2026-08-14 实现并上线(见 [CHANGELOG 2026-08-14](../../CHANGELOG.md))。 > 日期:2026-08-14 -> ⚠️ **临时测试设施——公开上线前必须整体移除**,全量定位方式:`grep -r TEST_LOGIN_BYPASS` +> ⚠️ **临时测试设施——公开上线前必须整体移除**(已于 2026-10-29 完成) ## 一、目的 diff --git a/docs/testing/scraper-e2e.md b/docs/testing/scraper-e2e.md index a1758c98..bef9b9d3 100644 --- a/docs/testing/scraper-e2e.md +++ b/docs/testing/scraper-e2e.md @@ -1,14 +1,17 @@ # 前端发起的 Scraper E2E > 目的:从真实前端“作品提名 → 自动获取信息”链路验证各站点爬虫,而不是只测 -> 后端 `/api/v1/scraper/scrape`。登录使用测试旁路账号,不需要真实短信。 +> 后端 `/api/v1/scraper/scrape`。 > 对应 B-063 的自动化缺口:单测见 `tests/unit/test_scraper_sites.py`、URL 路由见 > `tests/unit/test_scraper_parse_url.py`。 +> +> ⚠️ 2026-10-29 起测试登录旁路已删除(P-4),本脚本不再能自动过验证码: +> 需要真人先触发发码、把手机/邮箱收到的验证码通过 `CODE=` 传进来。 ## 前置 -1. 测试环境已配置 `TEST_LOGIN_BYPASS_ACCOUNTS` 包含 `19900000001`, - `TEST_LOGIN_BYPASS_CODE=888888`(当前测试 Nacos 已配置)。 +1. 测试环境(或本地)能收到真实验证码;`CODE` 由真人现场提供 + (旧的白名单固定码 `888888` 已失效)。 2. `19900000001` 已完成问卷(测试库该账号已有 year=12 的答案,可直接进提名页)。 3. 本机有 Node 与 Playwright: @@ -22,7 +25,7 @@ npx playwright install chromium ```bash cd thvote-be-re FRONTEND_BASE=http://154.37.215.62:8082/v11 \ -PHONE=19900000001 CODE=888888 \ +PHONE=19900000001 CODE=<真人收到的6位验证码> \ node scripts/e2e/scraper_frontend.js ``` diff --git a/scripts/e2e/scraper_frontend.js b/scripts/e2e/scraper_frontend.js index 658f2a34..bfd47162 100644 --- a/scripts/e2e/scraper_frontend.js +++ b/scripts/e2e/scraper_frontend.js @@ -2,16 +2,17 @@ /** * Frontend-driven scraper E2E (Playwright). * - * Logs in through the real login modal with the test-bypass phone/code, - * opens the doujin nomination editor and clicks “自动获取信息” for each URL. + * Logs in through the real login modal, opens the doujin nomination editor and + * clicks “自动获取信息” for each URL. * * Prereqs: * npm i playwright && npx playwright install chromium - * Test env must have TEST_LOGIN_BYPASS_ACCOUNTS/CODE configured. + * 2026-10-29 起测试登录旁路已删除(P-4):必须真人触发发码并把收到的验证码 + * 通过 CODE= 传入,固定码 888888 不再可用。 * * Usage: * FRONTEND_BASE=http://154.37.215.62:8082/v11 \ - * PHONE=19900000001 CODE=888888 \ + * PHONE=19900000001 CODE= \ * node scripts/e2e/scraper_frontend.js */ From e47e312910ea891f45c6d073372658f438b693d5 Mon Sep 17 00:00:00 2001 From: zfqxuz Date: Thu, 29 Oct 2026 02:21:13 +0800 Subject: [PATCH 05/11] fix(config): tolerate empty/CSV IP allowlists from Nacos TRUSTED_PROXY_IPS/ADMIN_ALLOWED_IPS are list[str], so pydantic-settings JSON-decoded them at the source layer and an empty string (the documented 'leave blank' form) raised SettingsError before any validator ran, crashing the process at import time. Switch both to NoDecode plus a tolerant parser accepting '', JSON arrays and comma-separated values. --- docs/CHANGELOG.md | 1 + src/common/config.py | 31 ++++++++++++++++++-- tests/unit/test_config_ip_lists.py | 45 ++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 tests/unit/test_config_ip_lists.py diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index d3db8b16..d57c7889 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -10,6 +10,7 @@ - **Added**:`Settings.expose_api_docs`(`EXPOSE_API_DOCS`,默认 `false`)。`src/main.py` 按开关传 `docs_url`/`redoc_url`/`openapi_url`,并给 `AppGraphQLRouter` 传 `graphql_ide`——关闭时 `/docs`、`/redoc`、`/openapi.json`、浏览器打开 `/graphql` 均 404,`POST /graphql` 不受影响。回归用例 `tests/unit/test_expose_api_docs.py`(8 例)。 - **Removed**:测试登录旁路整体删除(P-4):`src/common/verification/test_bypass.py`、`sms_code.py`/`email_code.py` 两个 `consume` 短路、`Settings` 两个字段、`tests/unit/test_test_login_bypass.py`。测试/生产 Nacos 同步删键;`grep -r TEST_LOGIN_BYPASS src` 为空。设计稿与操作文档标记为已移除,`scripts/e2e/scraper_frontend.js` 改为需真实验证码。 - **Added**:`.github/workflows/deploy-test.yml` 新增 `deploy_prod` 手动触发开关与 `deploy-prod` job(`needs: [build-backend, deploy-test]`,复用同一 digest 部署到生产机 `/opt/thvote/prod`);compose 未变,依赖(redis/nacos)由生产目录的 `docker-compose.deps.yml` 单独维护。 +- **Fixed**:`TRUSTED_PROXY_IPS` / `ADMIN_ALLOWED_IPS` 改为 `NoDecode` + 宽容解析,接受 `""`、JSON 数组、逗号分隔三种写法。此前把这两个键按文档写成空串会让 `Settings()` 在 import 期抛 `SettingsError`、进程起不来(生产首次部署实测踩中)。用例 `tests/unit/test_config_ip_lists.py`。 - **兼容性影响**:生产环境 graphQL 文档/调试界面不可用属预期;调用方若依赖 `/openapi.json` 生成客户端,需在测试环境用 `EXPOSE_API_DOCS=true`。 - **数据迁移 / 配置变更**:生产 Nacos 新建 namespace `thvote-prod` + dataId `thvote_be`;`ADMIN_ALLOWED_IPS`/`TRUSTED_PROXY_IPS` 留空、`EXPOSE_API_DOCS=false`。未导入用户数据、未改前端仓库。 diff --git a/src/common/config.py b/src/common/config.py index 191a0b4e..292213f9 100644 --- a/src/common/config.py +++ b/src/common/config.py @@ -287,11 +287,36 @@ def dojin_domain_allowlist(self) -> list[str]: # 安全配置 cors_allowed_origins: list[str] = Field(default_factory=lambda: ["*"]) - trusted_proxy_ips: list[str] = Field(default_factory=list) + # 下面两个 IP 列表同样要容忍 Nacos 里的各种写法。NoDecode 关掉 env source + # 的 JSON 预解析,否则写成 "" / "10.0.0.1" 会在 source 层抛 SettingsError, + # 整个进程起不来(2026-10-29 生产首次部署即因此失败)。 + trusted_proxy_ips: Annotated[list[str], NoDecode] = Field(default_factory=list) # 管理端 IP 白名单(B-049)。Nacos 里写 JSON 数组字符串, # 如 "ADMIN_ALLOWED_IPS": "[\"1.2.3.4\", \"10.0.0.0/8\"]"。 - # 空 = 不限 IP(仍需 X-Admin-Secret);pydantic-settings 自动 JSON 解码 list[str]。 - admin_allowed_ips: list[str] = Field(default_factory=list) + # 空 = 不限 IP(仍需 X-Admin-Secret)。 + admin_allowed_ips: Annotated[list[str], NoDecode] = Field(default_factory=list) + + @field_validator("trusted_proxy_ips", "admin_allowed_ips", mode="before") + @classmethod + def _parse_ip_lists(cls, value): + """接受 ``""``、JSON 数组、逗号分隔三种写法;空值 = 空列表。""" + if not isinstance(value, str): + return value + text = value.strip() + if not text: + return [] + if text[0] in "[{": + import json + + try: + parsed = json.loads(text) + except ValueError: + parsed = None + if isinstance(parsed, list): + return [str(item).strip() for item in parsed] + logger.error("IP 白名单配置 %r 不是合法的 JSON 数组,按空列表处理", value) + return [] + return [part.strip() for part in text.split(",") if part.strip()] # 拦裸脚本(B-048):对变更类请求(GraphQL mutation + REST 提交/登录/发码)要求 # 带浏览器自动附加的 Origin/Referer 头;curl/python 默认不带→403。默认关, # 经 Nacos 灰度开启(REQUIRE_BROWSER_ORIGIN=true)。不拦 query(codegen 免受影响)。 diff --git a/tests/unit/test_config_ip_lists.py b/tests/unit/test_config_ip_lists.py new file mode 100644 index 00000000..571416d7 --- /dev/null +++ b/tests/unit/test_config_ip_lists.py @@ -0,0 +1,45 @@ +"""``TRUSTED_PROXY_IPS`` / ``ADMIN_ALLOWED_IPS`` 的宽容解析。 + +Nacos 把配置原样写进环境变量,值都是字符串。pydantic-settings 对 ``list[str]`` +默认在 source 层按 JSON 预解析,于是把这两个键写成 ``""``(文档里的"留空"写法) +会让 ``Settings()`` 在 import 期直接抛 SettingsError、进程起不来——2026-10-29 +生产首次部署就是卡在这里。这两个键现在接受 ``""``、JSON 数组、逗号分隔三种写法。 +""" + +from __future__ import annotations + +import pytest + +from src.common.config import Settings + + +@pytest.mark.parametrize( + "raw, expected", + [ + ("", []), + ("[]", []), + ('["10.0.0.0/8"]', ["10.0.0.0/8"]), + ("10.0.0.0/8, 192.168.0.0/16", ["10.0.0.0/8", "192.168.0.0/16"]), + ("1.2.3.4", ["1.2.3.4"]), + ], +) +def test_ip_lists_accept_nacos_strings( + monkeypatch: pytest.MonkeyPatch, raw: str, expected: list[str] +) -> None: + monkeypatch.setenv("TRUSTED_PROXY_IPS", raw) + monkeypatch.setenv("ADMIN_ALLOWED_IPS", raw) + + settings = Settings() + + assert settings.trusted_proxy_ips == expected + assert settings.admin_allowed_ips == expected + + +def test_ip_lists_default_to_empty(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("TRUSTED_PROXY_IPS", raising=False) + monkeypatch.delenv("ADMIN_ALLOWED_IPS", raising=False) + + settings = Settings() + + assert settings.trusted_proxy_ips == [] + assert settings.admin_allowed_ips == [] From a2f5c02f53f2fbbb15b2daa67b16a90be49af468 Mon Sep 17 00:00:00 2001 From: zfqxuz Date: Fri, 30 Oct 2026 00:25:22 +0800 Subject: [PATCH 06/11] feat(pii): field-level encryption for legacy user import - src/common/crypto.py: AES-256-GCM + HKDF content/index keys + HMAC blind index - migration 0022: encrypted columns + (provider, subject_idx) unique index, plaintext nullable - user/user_identity: encrypted columns + *_plain accessors - DAO/service/admin: exact lookup by blind index, remove fuzzy identity search - import_legacy_voters.py: write ciphertext, dedup by subject_idx, --only-test-accounts - main.py: log PII DEK fingerprint at startup, fail fast if missing - docker-compose.prod.yml: mount pii_dek secret - scripts/verify_import.py: read-only import verification --- alembic/versions/0022_add_pii_encryption.py | 116 ++++++++++++++ docker/docker-compose.prod.yml | 11 ++ pyproject.toml | 2 + scripts/import_legacy_voters.py | 68 ++++++--- scripts/verify_import.py | 142 ++++++++++++++++++ src/apps/admin/router.py | 4 +- src/apps/user/dao.py | 24 ++- src/apps/user/identity_service.py | 27 +++- src/apps/user/schemas.py | 4 +- src/apps/user/service.py | 6 +- src/common/crypto.py | 118 +++++++++++++++ src/db_model/user.py | 22 ++- src/db_model/user_identity.py | 46 +++++- src/main.py | 5 + tests/conftest.py | 11 ++ tests/helpers/users.py | 22 ++- tests/integration/test_identity_binding.py | 12 +- .../test_identity_login_regressions.py | 4 +- tests/integration/test_legacy_login_compat.py | 17 ++- tests/integration/test_sso_flows.py | 6 +- tests/unit/test_crypto.py | 93 ++++++++++++ 21 files changed, 690 insertions(+), 70 deletions(-) create mode 100644 alembic/versions/0022_add_pii_encryption.py create mode 100644 scripts/verify_import.py create mode 100644 src/common/crypto.py create mode 100644 tests/unit/test_crypto.py diff --git a/alembic/versions/0022_add_pii_encryption.py b/alembic/versions/0022_add_pii_encryption.py new file mode 100644 index 00000000..e2605617 --- /dev/null +++ b/alembic/versions/0022_add_pii_encryption.py @@ -0,0 +1,116 @@ +"""0022 add encrypted PII columns and migrate the identity unique constraint. + +Adds AES-GCM ciphertext columns and HMAC blind-index columns so the plaintext +``subject`` / ``register_ip_address`` / ``created_ip`` can be retired: + +- ``user_identity.subject_enc`` / ``subject_idx`` (email/phone/qq/thbwiki) +- ``user_identity.created_ip_enc`` (per-identity bind IP) +- ``user.register_ip_enc`` / ``register_ip_prefix`` (account register IP + /24) + +The old plaintext columns are kept (made nullable, server defaults dropped) so +they end up NULL for every new write; migration 0023 drops them after the +import has been verified. The ``(provider, subject)`` unique constraint is +replaced by ``(provider, subject_idx)`` so equality lookup keeps working on the +blind index. + +``register_ip_prefix`` is a plain ``String`` (CIDR ``a.b.c.0/24``) rather than +``INET`` to keep the ORM model portable across Postgres and the sqlite test +database; there is no range-query consumer yet. +""" + +from alembic import op +import sqlalchemy as sa + +revision = "0022" +down_revision = "0021" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # Encrypted + blind-index columns. + op.add_column( + "user_identity", sa.Column("subject_enc", sa.LargeBinary(), nullable=True) + ) + op.add_column( + "user_identity", sa.Column("subject_idx", sa.LargeBinary(), nullable=True) + ) + op.add_column( + "user_identity", sa.Column("created_ip_enc", sa.LargeBinary(), nullable=True) + ) + op.add_column("user", sa.Column("register_ip_enc", sa.LargeBinary(), nullable=True)) + op.add_column( + "user", sa.Column("register_ip_prefix", sa.String(length=64), nullable=True) + ) + + # Plaintext columns become NULL-able; drop the "" server defaults so a + # missing value is NULL (and the §10.2 "is not null → 0" scan holds). + op.alter_column( + "user_identity", + "subject", + existing_type=sa.String(length=255), + nullable=True, + ) + op.alter_column( + "user", + "register_ip_address", + existing_type=sa.String(length=64), + nullable=True, + server_default=None, + ) + op.alter_column( + "user_identity", + "created_ip", + existing_type=sa.String(length=64), + nullable=True, + server_default=None, + ) + + # Unique constraint migrates from plaintext subject to the blind index. + op.drop_constraint( + "uq_user_identity_provider_subject", "user_identity", type_="unique" + ) + op.create_index( + "uq_user_identity_provider_idx", + "user_identity", + ["provider", "subject_idx"], + unique=True, + ) + op.create_index( + "ix_user_identity_subject_idx", "user_identity", ["subject_idx"] + ) + + +def downgrade() -> None: + op.drop_index("ix_user_identity_subject_idx", table_name="user_identity") + op.drop_index("uq_user_identity_provider_idx", table_name="user_identity") + op.create_unique_constraint( + "uq_user_identity_provider_subject", + "user_identity", + ["provider", "subject"], + ) + op.alter_column( + "user_identity", + "created_ip", + existing_type=sa.String(length=64), + nullable=False, + server_default="", + ) + op.alter_column( + "user", + "register_ip_address", + existing_type=sa.String(length=64), + nullable=False, + server_default="", + ) + op.alter_column( + "user_identity", + "subject", + existing_type=sa.String(length=255), + nullable=False, + ) + op.drop_column("user", "register_ip_prefix") + op.drop_column("user", "register_ip_enc") + op.drop_column("user_identity", "created_ip_enc") + op.drop_column("user_identity", "subject_idx") + op.drop_column("user_identity", "subject_enc") diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index b5dcb317..30a81c19 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -32,10 +32,17 @@ services: NACOS_PASSWORD: "${NACOS_PASSWORD}" APP_HOST: "0.0.0.0" APP_PORT: "8000" + PII_DEK_PATH: "/run/secrets/pii_dek" ports: - "18000:8000" volumes: - ./logs:/app/logs + secrets: + - source: pii_dek + target: pii_dek + uid: "1000" + gid: "1000" + mode: "0400" healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8000/health"] interval: 15s @@ -50,3 +57,7 @@ networks: volumes: redis-data: + +secrets: + pii_dek: + file: /opt/thvote/prod/secrets/pii_dek diff --git a/pyproject.toml b/pyproject.toml index 2d344ff6..441ccf63 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,8 @@ dependencies = [ "httpx>=0.27.0", # eMudhra/emSign 事件:certifi 2026.10.19 移除受影响根;显式下限防回退 "certifi>=2026.10.19", + # PII 字段级加密(AES-256-GCM + HKDF + HMAC 盲索引)——此前仅传递依赖,必须显式声明 + "cryptography>=42.0", "pytest>=8.0.0", "pytest-asyncio>=0.23.0", "strawberry-graphql[fastapi]>=0.219.0", diff --git a/scripts/import_legacy_voters.py b/scripts/import_legacy_voters.py index a972c0fa..c9a0ec62 100644 --- a/scripts/import_legacy_voters.py +++ b/scripts/import_legacy_voters.py @@ -48,6 +48,13 @@ sys.path.insert(0, str(_project_root)) from src.common.config import get_settings # noqa: E402 +from src.common.crypto import ( # noqa: E402 + aad_for, + blind_index, + encrypt, + fingerprint, + ip_prefix, +) from src.common.database import normalize_async_database_url # noqa: E402 from src.db_model.legacy_user_mapping import LegacyUserMapping # noqa: E402 from src.db_model.user import User # noqa: E402 @@ -308,11 +315,17 @@ def _identity_rows(voter: LegacyVoter, user_id: str) -> list[dict[str, Any]]: { "user_id": user_id, "provider": provider, - "subject": subject, + "subject_enc": encrypt( + subject, aad_for("user_identity", "subject", user_id) + ), + "subject_idx": blind_index(provider, subject), "verified": bool(verified), "verified_at": voter.created_at if verified else None, "created_at": voter.created_at, - "created_ip": voter.signup_ip, + "created_ip_enc": encrypt( + voter.signup_ip or None, + aad_for("user_identity", "created_ip", user_id), + ), "created_device_id": "", "last_login_at": None, } @@ -329,41 +342,44 @@ def _user_row(voter: LegacyVoter, user_id: str) -> dict[str, Any]: "legacy_salt": voter.legacy_salt if voter.is_bcrypt else None, "removed": voter.removed, "register_date": voter.created_at, - "register_ip_address": voter.signup_ip, + "register_ip_enc": encrypt( + voter.signup_ip or None, aad_for("user", "register_ip", user_id) + ), + "register_ip_prefix": ip_prefix(voter.signup_ip), "register_device_id": "", } async def _existing_identities(session, voters: list[LegacyVoter]) -> dict: - emails = [v.email for v in voters if v.email] - phones = [v.phone for v in voters if v.phone] - found: dict[tuple[str, str], str] = {} - if emails: + email_idx = [blind_index("email", v.email) for v in voters if v.email] + phone_idx = [blind_index("phone", v.phone) for v in voters if v.phone] + found: dict[tuple[str, bytes], str] = {} + if email_idx: result = await session.execute( select( UserIdentity.provider, - UserIdentity.subject, + UserIdentity.subject_idx, UserIdentity.user_id, ).where( UserIdentity.provider == "email", - UserIdentity.subject.in_(emails), + UserIdentity.subject_idx.in_(email_idx), ) ) - for provider, subject, user_id in result: - found[(provider, subject)] = user_id - if phones: + for provider, subject_idx, user_id in result: + found[(provider, subject_idx)] = user_id + if phone_idx: result = await session.execute( select( UserIdentity.provider, - UserIdentity.subject, + UserIdentity.subject_idx, UserIdentity.user_id, ).where( UserIdentity.provider == "phone", - UserIdentity.subject.in_(phones), + UserIdentity.subject_idx.in_(phone_idx), ) ) - for provider, subject, user_id in result: - found[(provider, subject)] = user_id + for provider, subject_idx, user_id in result: + found[(provider, subject_idx)] = user_id return found @@ -387,16 +403,16 @@ async def _load_mapped(session, *, commit: bool) -> set[str]: def _classify( - voter: LegacyVoter, existing: dict[tuple[str, str], str] + voter: LegacyVoter, existing: dict[tuple[str, bytes], str] ) -> tuple[str, str | None]: """Return (kind, owner_user_id). kind ∈ create / conflict / multi.""" owners: dict[str, str] = {} if voter.email: - owner = existing.get(("email", voter.email)) + owner = existing.get(("email", blind_index("email", voter.email))) if owner: owners[owner] = "email" if voter.phone: - owner = existing.get(("phone", voter.phone)) + owner = existing.get(("phone", blind_index("phone", voter.phone))) if owner: owners[owner] = "phone" if not owners: @@ -410,7 +426,7 @@ async def _process_batch( session, winners: list[LegacyVoter], plan: ImportPlan, - existing: dict[tuple[str, str], str], + existing: dict[tuple[str, bytes], str], mapped: set[str], batch_id: str, stats: Stats, @@ -614,6 +630,7 @@ async def run( dump_dir: Path, *, include_test_accounts: bool, + only_test_accounts: bool, commit: bool, batch_id: str, report_dir: Path, @@ -621,11 +638,14 @@ async def run( limit: int | None, ) -> Stats: started = datetime.now(UTC) + logger.info("PII DEK loaded: fp=%s", fingerprint()) voters: list[LegacyVoter] = [] for index, doc in enumerate(iter_voters(dump_dir / "voters.bson")): if limit is not None and index >= limit: break voters.append(normalize_voter(doc)) + if only_test_accounts: + voters = [v for v in voters if v.email in TEST_ACCOUNT_EMAILS] last_activity = load_last_activity(dump_dir / "voter_logs.bson") plan = plan_import( voters, @@ -727,6 +747,11 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="Import votetester1..5@touhou.ai (test DB only)", ) + parser.add_argument( + "--only-test-accounts", + action="store_true", + help="Import ONLY votetester1..5@touhou.ai (smoke test)", + ) parser.add_argument( "--preflight", action="store_true", @@ -759,11 +784,14 @@ def main(argv: list[str] | None = None) -> int: batch_id = args.batch_id or str(uuid.uuid4()) mode = "commit" if args.commit else "preflight" + if args.only_test_accounts: + args.include_test_accounts = True logger.info("batch_id=%s mode=%s", batch_id, mode) stats = asyncio.run( run( dump_dir, include_test_accounts=args.include_test_accounts, + only_test_accounts=args.only_test_accounts, commit=args.commit, batch_id=batch_id, report_dir=Path(args.report_dir), diff --git a/scripts/verify_import.py b/scripts/verify_import.py new file mode 100644 index 00000000..28937ba7 --- /dev/null +++ b/scripts/verify_import.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""Read-only verification of the legacy voter import. + +Connects to the target DB via ``get_settings()`` (Nacos), then prints a +PASS/FAIL checklist: row counts, plaintext-leak scan (the red-line check), +encrypted-column coverage, and one decrypted sample that must round-trip back +through the blind index. Never writes. + +Usage (one-off container, same image as the backend): + docker run --rm --network thvote-net \ + --env-file /opt/thvote/prod/.env \ + -e PII_DEK_PATH=/run/secrets/pii_dek \ + -v /opt/thvote/prod/secrets:/run/secrets:ro \ + \ + python scripts/verify_import.py \ + [--expect-users N] [--expect-identities N] [--expect-mappings N] +""" + +from __future__ import annotations + +import argparse +import asyncio +import sys +from pathlib import Path + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import create_async_engine + +_project_root = Path(__file__).resolve().parent.parent +if str(_project_root) not in sys.path: + sys.path.insert(0, str(_project_root)) + +from src.common.config import get_settings # noqa: E402 +from src.common.crypto import aad_for, blind_index, decrypt, fingerprint # noqa: E402 +from src.common.database import normalize_async_database_url # noqa: E402 + + +async def _scalar(conn, sql: str) -> int: + return int(await conn.scalar(text(sql))) + + +async def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--expect-users", type=int, default=None) + parser.add_argument("--expect-identities", type=int, default=None) + parser.add_argument("--expect-mappings", type=int, default=None) + args = parser.parse_args(argv) + + settings = get_settings() + engine = create_async_engine( + normalize_async_database_url(settings.database_url), echo=False + ) + checks: list[tuple[str, bool, str]] = [] + + def check(name: str, ok: bool, detail: str) -> None: + checks.append((name, ok, detail)) + print(f"[{'PASS' if ok else 'FAIL'}] {name}: {detail}") + + try: + async with engine.connect() as conn: + users = await _scalar(conn, 'select count(*) from "user"') + identities = await _scalar(conn, "select count(*) from user_identity") + mappings = await _scalar(conn, "select count(*) from legacy_user_mapping") + + subject_plain = await _scalar( + conn, "select count(*) from user_identity where subject is not null" + ) + regip_plain = await _scalar( + conn, + 'select count(*) from "user" where register_ip_address is not null', + ) + created_ip_plain = await _scalar( + conn, "select count(*) from user_identity where created_ip is not null" + ) + subject_enc = await _scalar( + conn, "select count(*) from user_identity where subject_enc is not null" + ) + regip_enc = await _scalar( + conn, 'select count(*) from "user" where register_ip_enc is not null' + ) + created_ip_enc = await _scalar( + conn, + "select count(*) from user_identity where created_ip_enc is not null", + ) + + sample = ( + await conn.execute( + text( + "select provider, subject_enc, subject_idx, user_id " + "from user_identity " + "where provider='email' and subject_enc is not null limit 1" + ) + ) + ).first() + sample_ok = False + sample_detail = "" + if sample is not None: + provider, enc, idx, uid = sample + try: + plain = decrypt(enc, aad_for("user_identity", "subject", uid)) + sample_ok = bool(plain) and blind_index(provider, plain) == idx + sample_detail = plain or "" + except Exception as exc: # noqa: BLE001 - report, don't crash + sample_detail = f"" + + print("PII DEK loaded: fp=%s" % fingerprint()) + print(f"users={users} identities={identities} mappings={mappings}") + + if args.expect_users is not None: + check("user count", users == args.expect_users, str(users)) + if args.expect_identities is not None: + check( + "identity count", + identities == args.expect_identities, + str(identities), + ) + if args.expect_mappings is not None: + check( + "mapping count", mappings == args.expect_mappings, str(mappings) + ) + + check("subject plaintext is NULL", subject_plain == 0, str(subject_plain)) + check("register_ip plaintext is NULL", regip_plain == 0, str(regip_plain)) + check( + "created_ip plaintext is NULL", + created_ip_plain == 0, + str(created_ip_plain), + ) + check("subject_enc populated", subject_enc > 0, str(subject_enc)) + check("register_ip_enc populated", regip_enc > 0, str(regip_enc)) + check("created_ip_enc populated", created_ip_enc > 0, str(created_ip_enc)) + check("sample decrypt + blind-index roundtrip", sample_ok, sample_detail) + finally: + await engine.dispose() + + failed = [c for c in checks if not c[1]] + print("RESULT:", "PASS" if not failed else f"{len(failed)} CHECK(S) FAILED") + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) diff --git a/src/apps/admin/router.py b/src/apps/admin/router.py index 03c94a22..354ff275 100644 --- a/src/apps/admin/router.py +++ b/src/apps/admin/router.py @@ -141,8 +141,8 @@ def _user_to_item(u) -> dict: return { "id": u.id, "nickname": u.nickname, - "email": email.subject if email is not None else None, - "phone": phone.subject if phone is not None else None, + "email": email.subject_plain if email is not None else None, + "phone": phone.subject_plain if phone is not None else None, "email_verified": bool(email is not None and email.verified), "phone_verified": bool(phone is not None and phone.verified), "register_date": u.register_date.isoformat() if u.register_date else None, diff --git a/src/apps/user/dao.py b/src/apps/user/dao.py index ed34da07..a68db124 100644 --- a/src/apps/user/dao.py +++ b/src/apps/user/dao.py @@ -19,6 +19,8 @@ from sqlalchemy.exc import InvalidRequestError from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker +from src.apps.user.identity import IdentityProvider, normalize_subject +from src.common.crypto import blind_index from src.db_model.activity_log import ActivityLog from src.db_model.user import User from src.db_model.user_identity import UserIdentity @@ -88,9 +90,17 @@ async def search_users( ) -> tuple[list[User], int]: query = select(User) if email: - query = query.where(_has_identity_like("email", email)) + try: + canonical = normalize_subject(IdentityProvider.EMAIL, email) + except ValueError: + return [], 0 + query = query.where(_has_identity_exact("email", canonical)) if phone: - query = query.where(_has_identity_like("phone", phone)) + try: + canonical = normalize_subject(IdentityProvider.PHONE, phone) + except ValueError: + return [], 0 + query = query.where(_has_identity_exact("phone", canonical)) count_result = await self.session.execute( select(func.count()).select_from(query.subquery()) ) @@ -110,11 +120,13 @@ async def set_removed(self, user_id: str, removed: bool) -> User | None: return user -def _has_identity_like(provider: str, pattern: str): +def _has_identity_exact(provider: str, subject: str): + """EXISTS(...) over the blind index — encrypted subjects cannot be + fuzzy-matched, so admin search is exact on the canonical identifier.""" return User.identities.any( and_( UserIdentity.provider == provider, - UserIdentity.subject.ilike(f"%{pattern}%"), + UserIdentity.subject_idx == blind_index(provider, subject), ) ) @@ -134,7 +146,7 @@ async def find_identity(self, provider: str, subject: str) -> UserIdentity | Non result = await self.session.execute( select(UserIdentity).where( UserIdentity.provider == provider, - UserIdentity.subject == subject, + UserIdentity.subject_idx == blind_index(provider, subject), ) ) return result.scalar_one_or_none() @@ -146,7 +158,7 @@ async def find_user(self, provider: str, subject: str) -> User | None: .join(User.identities) .where( UserIdentity.provider == provider, - UserIdentity.subject == subject, + UserIdentity.subject_idx == blind_index(provider, subject), ) ) return result.scalar_one_or_none() diff --git a/src/apps/user/identity_service.py b/src/apps/user/identity_service.py index e82ed324..741ceed4 100644 --- a/src/apps/user/identity_service.py +++ b/src/apps/user/identity_service.py @@ -21,6 +21,7 @@ from src.apps.user.dao import UserDAO, UserIdentityDAO from src.apps.user.identity import IdentityProvider, normalize_subject from src.apps.user.schemas import Meta, generate_user_id +from src.common.crypto import aad_for, blind_index, encrypt, ip_prefix from src.common.exceptions import UnauthorizedError, ValidationError from src.db_model.user import User from src.db_model.user_identity import UserIdentity @@ -81,14 +82,20 @@ async def register( """ subject = normalize_subject(provider, raw_subject) now = datetime.now(UTC) + user_id = generate_user_id() user = User( - id=generate_user_id(), + id=user_id, nickname=nickname, - register_ip_address=meta.user_ip or "", + register_ip_enc=encrypt( + meta.user_ip or None, aad_for("user", "register_ip", user_id) + ), + register_ip_prefix=ip_prefix(meta.user_ip or ""), register_device_id=meta.additional_fingureprint or "", ) user.identities.append( - _new_identity(provider, subject, meta, now, last_login_at=now) + _new_identity( + provider, subject, meta, now, user_id=user_id, last_login_at=now + ) ) return await self.user_dao.create(user) @@ -122,13 +129,15 @@ async def bind( return BindResult(identity=existing, replaced_subject=None, changed=False) current = user.identity(provider.value) - replaced = current.subject if current is not None else None + replaced = current.subject_plain if current is not None else None if current is not None: # Flush the DELETE before the INSERT: the unit of work would # otherwise emit INSERT first and trip uq_user_identity_user_provider. user.identities.remove(current) await self.identity_dao.session.flush() - identity = _new_identity(provider, subject, meta, datetime.now(UTC)) + identity = _new_identity( + provider, subject, meta, datetime.now(UTC), user_id=user.id + ) user.identities.append(identity) await self.user_dao.save(user) return BindResult(identity=identity, replaced_subject=replaced, changed=True) @@ -168,6 +177,7 @@ def _new_identity( meta: Meta, now: datetime, *, + user_id: str, last_login_at: datetime | None = None, ) -> UserIdentity: # Every creation path today runs after verification (code / OAuth), so @@ -175,11 +185,14 @@ def _new_identity( # "record first, verify later" flows (spec §3.4). return UserIdentity( provider=provider.value, - subject=subject, + subject_enc=encrypt(subject, aad_for("user_identity", "subject", user_id)), + subject_idx=blind_index(provider.value, subject), verified=True, verified_at=now, created_at=now, - created_ip=meta.user_ip or "", + created_ip_enc=encrypt( + meta.user_ip or None, aad_for("user_identity", "created_ip", user_id) + ), created_device_id=meta.additional_fingureprint or "", last_login_at=last_login_at, ) diff --git a/src/apps/user/schemas.py b/src/apps/user/schemas.py index 2d98a4b2..2bb91511 100644 --- a/src/apps/user/schemas.py +++ b/src/apps/user/schemas.py @@ -167,8 +167,8 @@ def voter_fe_from_user(user) -> VoterFE: username=user.nickname, pfp=user.pfp, password=bool(user.password_hash), - phone=phone.subject if phone is not None else None, - email=email.subject if email is not None else None, + phone=phone.subject_plain if phone is not None else None, + email=email.subject_plain if email is not None else None, thbwiki=user.identity("thbwiki") is not None, patchyvideo=False, created_at=user.register_date, diff --git a/src/apps/user/service.py b/src/apps/user/service.py index 5a3064fd..ee7da0e0 100644 --- a/src/apps/user/service.py +++ b/src/apps/user/service.py @@ -278,7 +278,7 @@ async def _rebind_contact( event_type=event_type, user_id=user.id, old_value=bound.replaced_subject, - new_value=bound.identity.subject, + new_value=bound.identity.subject_plain, requester_ip=meta.user_ip, additional_fingerprint=meta.additional_fingureprint, ) @@ -377,7 +377,7 @@ async def _merge_sso_session(self, user: User, sid, meta: Meta) -> None: # 任何人都能再去认领;而这里只是登录顺带的 courtesy,用户并未 # 表达换绑意图。换绑走 update_* / bind_sso 的显式入口。 current = user.identity(provider.value) - if current is not None and current.subject != subject: + if current is not None and current.subject_plain != subject: logger.warning( "SSO merge skipped for user_id=%s provider=%s: " "already bound to a different subject", @@ -401,7 +401,7 @@ async def _merge_sso_session(self, user: User, sid, meta: Meta) -> None: await self._safe_log( event_type="bind_sso", user_id=user.id, - new_value=bound.identity.subject, + new_value=bound.identity.subject_plain, requester_ip=meta.user_ip, additional_fingerprint=meta.additional_fingureprint, ) diff --git a/src/common/crypto.py b/src/common/crypto.py new file mode 100644 index 00000000..b2ac0b08 --- /dev/null +++ b/src/common/crypto.py @@ -0,0 +1,118 @@ +"""Field-level encryption and blind indexing for PII columns. + +One DEK (32 random bytes, base64-encoded) is read from ``PII_DEK_PATH`` and +split via HKDF into two independent keys: + +- ``content_key`` — AES-256-GCM for the ciphertext columns; +- ``index_key`` — HMAC-SHA256 for deterministic blind indices. + +Splitting them means a database dump cannot brute-force the phone-number space +against the content key; the index key alone only enables equality lookup. + +Ciphertext layout: ``version(1B) || nonce(12B) || ciphertext || tag(16B)``. +AAD binds a ciphertext to its row: ``"{table}.{column}:{row_id}"``. + +The DEK is deliberately *not* read from ``Settings``: it lives in a Docker +secret (``/run/secrets/pii_dek``) and must never appear in Nacos, ``.env``, +git or logs. ``fingerprint()`` returns a short one-way tag safe to log so an +operator can confirm the import job and the serving process share the same DEK. +""" + +from __future__ import annotations + +import base64 +import hashlib +import hmac +import ipaddress +import os +from functools import lru_cache +from pathlib import Path + +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from cryptography.hazmat.primitives.kdf.hkdf import HKDF + +VERSION = b"\x01" +DEK_PATH = Path(os.getenv("PII_DEK_PATH", "/run/secrets/pii_dek")) + +_CONTENT_INFO = b"thvote-pii-content" +_INDEX_INFO = b"thvote-pii-index" +_FINGERPRINT_INFO = b"thvote-dek-fingerprint" + + +@lru_cache(maxsize=1) +def _dek() -> bytes: + if not DEK_PATH.exists(): + raise RuntimeError(f"PII DEK missing: {DEK_PATH}") + raw = base64.b64decode(DEK_PATH.read_bytes().strip()) + if len(raw) != 32: + raise RuntimeError(f"PII DEK bad length: {len(raw)}") + return raw + + +@lru_cache(maxsize=1) +def _content_key() -> bytes: + return HKDF( + algorithm=hashes.SHA256(), length=32, salt=None, info=_CONTENT_INFO + ).derive(_dek()) + + +@lru_cache(maxsize=1) +def _index_key() -> bytes: + return HKDF( + algorithm=hashes.SHA256(), length=32, salt=None, info=_INDEX_INFO + ).derive(_dek()) + + +def fingerprint() -> str: + """Short one-way tag of the loaded DEK, safe to log.""" + return hmac.new(_dek(), _FINGERPRINT_INFO, hashlib.sha256).hexdigest()[:8] + + +def aad_for(table: str, column: str, row_id: str) -> bytes: + """AAD binding ciphertext to its row: ``table.column:row_id``.""" + return f"{table}.{column}:{row_id}".encode() + + +def encrypt(plaintext: str | None, aad: bytes) -> bytes | None: + """Encrypt *plaintext* under the content key, bound to *aad*.""" + if not plaintext: + return None + nonce = os.urandom(12) + ciphertext = AESGCM(_content_key()).encrypt(nonce, plaintext.encode(), aad) + return VERSION + nonce + ciphertext + + +def decrypt(blob: bytes | None, aad: bytes) -> str | None: + """Decrypt *blob*; raises on a wrong version or AAD mismatch.""" + if not blob: + return None + if blob[:1] != VERSION: + raise ValueError("unsupported ciphertext version") + return AESGCM(_content_key()).decrypt(blob[1:13], blob[13:], aad).decode() + + +def blind_index(provider: str, subject: str) -> bytes | None: + """Deterministic HMAC index over the canonical ``(provider, subject)``. + + *subject* must already be canonical (``normalize_subject``) — the same + value the DAO receives for lookups — so writes and reads agree. + """ + if not subject: + return None + message = f"{provider}:{subject}".encode() + return hmac.new(_index_key(), message, hashlib.sha256).digest() + + +def ip_prefix(ip: str) -> str | None: + """Return the /24 network of an IPv4 address as a CIDR string. + + Coarse bucket for range queries (anti-vote-farming clustering) that does + not require decrypting the full address. ``None`` for blank/non-IPv4. + """ + if not ip: + return None + try: + return str(ipaddress.ip_network(f"{ip.strip()}/24", strict=False)) + except ValueError: + return None diff --git a/src/db_model/user.py b/src/db_model/user.py index 0d4917b0..a487d935 100644 --- a/src/db_model/user.py +++ b/src/db_model/user.py @@ -8,9 +8,11 @@ from __future__ import annotations -from sqlalchemy import Boolean, DateTime, Index, String, func +from sqlalchemy import Boolean, DateTime, Index, LargeBinary, String, func from sqlalchemy.orm import Mapped, mapped_column, relationship +from src.common.crypto import aad_for, decrypt + from .base import Base from .user_identity import UserIdentity @@ -33,9 +35,11 @@ class User(Base): register_date: Mapped[DateTime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() ) - register_ip_address: Mapped[str] = mapped_column( - String(64), nullable=False, default="" - ) + # Plaintext IP stays NULL for new writes (encrypted into register_ip_enc); + # migration 0023 drops this column after import is verified. + register_ip_address: Mapped[str | None] = mapped_column(String(64), nullable=True) + register_ip_enc: Mapped[bytes | None] = mapped_column(LargeBinary, nullable=True) + register_ip_prefix: Mapped[str | None] = mapped_column(String(64), nullable=True) # 注册时的客户端设备指纹(localStorage UUID),反刷票取证用(B-044)。 register_device_id: Mapped[str] = mapped_column( String(128), nullable=False, default="", server_default="" @@ -49,6 +53,16 @@ class User(Base): cascade="all, delete-orphan", ) + @property + def register_ip_plain(self) -> str: + """Decrypted registration IP, falling back to the legacy column.""" + if self.register_ip_enc is not None: + value = decrypt( + self.register_ip_enc, aad_for("user", "register_ip", self.id) + ) + return value or "" + return self.register_ip_address or "" + def identity(self, provider: str) -> UserIdentity | None: """Return this account's identity for *provider*, if any.""" for identity in self.identities: diff --git a/src/db_model/user_identity.py b/src/db_model/user_identity.py index 14f96825..1b6a51f0 100644 --- a/src/db_model/user_identity.py +++ b/src/db_model/user_identity.py @@ -15,13 +15,17 @@ CheckConstraint, DateTime, ForeignKey, + Index, Integer, + LargeBinary, String, UniqueConstraint, func, ) from sqlalchemy.orm import Mapped, mapped_column, relationship +from src.common.crypto import aad_for, decrypt + from .base import Base if TYPE_CHECKING: @@ -43,7 +47,11 @@ class UserIdentity(Base): index=True, ) provider: Mapped[str] = mapped_column(String(16), nullable=False) - subject: Mapped[str] = mapped_column(String(255), nullable=False) + # Plaintext subject stays NULL for new writes (encrypted into subject_enc + # / subject_idx); migration 0023 drops this column after import is verified. + subject: Mapped[str | None] = mapped_column(String(255), nullable=True) + subject_enc: Mapped[bytes | None] = mapped_column(LargeBinary, nullable=True) + subject_idx: Mapped[bytes | None] = mapped_column(LargeBinary, nullable=True) verified: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) verified_at: Mapped[DateTime | None] = mapped_column( @@ -55,9 +63,8 @@ class UserIdentity(Base): created_at: Mapped[DateTime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() ) - created_ip: Mapped[str] = mapped_column( - String(64), nullable=False, default="", server_default="" - ) + created_ip: Mapped[str | None] = mapped_column(String(64), nullable=True) + created_ip_enc: Mapped[bytes | None] = mapped_column(LargeBinary, nullable=True) created_device_id: Mapped[str] = mapped_column( String(128), nullable=False, default="", server_default="" ) @@ -67,11 +74,36 @@ class UserIdentity(Base): user: Mapped["User"] = relationship(back_populates="identities") + @property + def subject_plain(self) -> str | None: + """Decrypted subject, falling back to the legacy plaintext column.""" + if self.subject_enc is not None: + return decrypt( + self.subject_enc, + aad_for("user_identity", "subject", self.user_id), + ) + return self.subject + + @property + def created_ip_plain(self) -> str: + """Decrypted bind IP, falling back to the legacy plaintext column.""" + if self.created_ip_enc is not None: + value = decrypt( + self.created_ip_enc, + aad_for("user_identity", "created_ip", self.user_id), + ) + return value or "" + return self.created_ip or "" + __table_args__ = ( - UniqueConstraint( - "provider", "subject", name="uq_user_identity_provider_subject" - ), UniqueConstraint("user_id", "provider", name="uq_user_identity_user_provider"), + Index( + "uq_user_identity_provider_idx", + "provider", + "subject_idx", + unique=True, + ), + Index("ix_user_identity_subject_idx", "subject_idx"), CheckConstraint( "provider IN (" + ", ".join(f"'{v}'" for v in PROVIDER_CHECK_VALUES) + ")", name="ck_user_identity_provider", diff --git a/src/main.py b/src/main.py index 7352e28c..ebce24dd 100644 --- a/src/main.py +++ b/src/main.py @@ -35,6 +35,7 @@ nacos_config_change_callback, reload_settings, ) +from .common.crypto import fingerprint as pii_dek_fingerprint from .common.database import get_db_session, get_session_maker, reload_engine from .common.middleware.logging import LoggingMiddleware from .apps.user.service import get_audit_log_failures @@ -45,6 +46,10 @@ @asynccontextmanager async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: """Application lifespan handler.""" + # Fail fast if the PII DEK is missing — encrypted columns cannot be read + # without it, so a missing DEK must never degrade into a running service. + logger.info("PII DEK loaded: fp=%s", pii_dek_fingerprint()) + # Verify DB connectivity — fail fast rather than silently running with no schema try: async with get_session_maker()() as session: diff --git a/tests/conftest.py b/tests/conftest.py index 0c577ac1..f6ba8d2a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,7 +10,10 @@ from __future__ import annotations +import base64 import os +import tempfile +from pathlib import Path # Ensure required env vars exist for Settings() before importing app modules. os.environ.setdefault("DATABASE_URL", "sqlite+aiosqlite:///./.test.db") @@ -18,3 +21,11 @@ os.environ.setdefault("JWT_SECRET_KEY", "test-secret-key-do-not-use-in-prod") os.environ.setdefault("VOTE_START_ISO", "2020-01-01T00:00:00+00:00") os.environ.setdefault("VOTE_END_ISO", "2099-12-31T23:59:59+00:00") + +# PII 字段级加密需要一个 DEK。测试进程用一个一次性随机密钥写入临时文件, +# 让 subject_enc / register_ip_enc / created_ip_enc 的读写路径都能跑通。 +# 必须在这里(导入任何 src 模块之前)设置,否则 crypto.DEK_PATH 读不到。 +_test_dek = base64.b64encode(os.urandom(32)).decode() +_test_dek_path = Path(tempfile.gettempdir()) / f"thvote-test-dek-{os.getpid()}" +_test_dek_path.write_text(_test_dek, encoding="utf-8") +os.environ["PII_DEK_PATH"] = str(_test_dek_path) diff --git a/tests/helpers/users.py b/tests/helpers/users.py index fa22f8d7..7be97a9a 100644 --- a/tests/helpers/users.py +++ b/tests/helpers/users.py @@ -12,6 +12,7 @@ from src.apps.user.schemas import generate_user_id from src.apps.user.utils.security import AuthProvider +from src.common.crypto import aad_for, blind_index, encrypt, ip_prefix from src.db_model.user import User from src.db_model.user_identity import UserIdentity @@ -31,13 +32,17 @@ def build_user( ) -> User: """Return an unsaved User with one identity per non-None identifier.""" now = datetime.now(UTC) + uid = user_id or generate_user_id() user = User( - id=user_id or generate_user_id(), + id=uid, nickname=nickname, password_hash=AuthProvider.hash_password(password) if password else None, removed=removed, register_date=now, - register_ip_address=register_ip, + register_ip_enc=encrypt( + register_ip or None, aad_for("user", "register_ip", uid) + ), + register_ip_prefix=ip_prefix(register_ip), ) for provider, subject in ( ("email", email), @@ -49,12 +54,21 @@ def build_user( continue user.identities.append( UserIdentity( + # 显式给 FK:未持久化的 build_user 对象上,append 不会立刻同步 + # user_id,而 subject_plain/created_ip_plain 的解密 AAD 依赖它。 + user_id=uid, provider=provider, - subject=subject, + subject_enc=encrypt( + subject, aad_for("user_identity", "subject", uid) + ), + subject_idx=blind_index(provider, subject), verified=verified, verified_at=now if verified else None, created_at=now, - created_ip=register_ip, + created_ip_enc=encrypt( + register_ip or None, + aad_for("user_identity", "created_ip", uid), + ), ) ) return user diff --git a/tests/integration/test_identity_binding.py b/tests/integration/test_identity_binding.py index 74cbc231..0253c234 100644 --- a/tests/integration/test_identity_binding.py +++ b/tests/integration/test_identity_binding.py @@ -48,12 +48,12 @@ async def test_first_login_creates_account_and_identity_with_same_meta( await _login_email(user_service, patch_redis, "new@example.com", nickname="n") user = await user_service.identities.resolve(IdentityProvider.EMAIL, "new@example.com") - assert user.register_ip_address == "203.0.113.7" + assert user.register_ip_plain == "203.0.113.7" assert user.register_device_id == "device-A" [identity] = user.identities - assert (identity.provider, identity.subject) == ("email", "new@example.com") + assert (identity.provider, identity.subject_plain) == ("email", "new@example.com") assert identity.verified is True and identity.verified_at is not None - assert identity.created_ip == "203.0.113.7" + assert identity.created_ip_plain == "203.0.113.7" assert identity.created_device_id == "device-A" assert identity.last_login_at is not None @@ -83,7 +83,7 @@ async def test_email_subject_is_case_insensitive(user_service, patch_redis): user = await user_service.identities.resolve(IdentityProvider.EMAIL, "Mixed@EXAMPLE.COM") assert user is not None - assert user.identity("email").subject == "mixed@example.com" + assert user.identity("email").subject_plain == "mixed@example.com" @pytest.mark.asyncio @@ -99,8 +99,8 @@ async def test_rebind_phone_replaces_row_with_new_meta(user_service, patch_redis ) rows = (await session.execute(select(UserIdentity))).scalars().all() - assert [(r.provider, r.subject) for r in rows] == [("phone", "13800000002")] - assert rows[0].created_ip == "198.51.100.9" + assert [(r.provider, r.subject_plain) for r in rows] == [("phone", "13800000002")] + assert rows[0].created_ip_plain == "198.51.100.9" assert rows[0].created_device_id == "device-B" assert await user_service.identities.resolve(IdentityProvider.PHONE, "13800000001") is None diff --git a/tests/integration/test_identity_login_regressions.py b/tests/integration/test_identity_login_regressions.py index 9a134cee..0e318074 100644 --- a/tests/integration/test_identity_login_regressions.py +++ b/tests/integration/test_identity_login_regressions.py @@ -118,7 +118,7 @@ async def test_sso_merge_does_not_replace_existing_binding( .where(UserIdentity.user_id == user.id, UserIdentity.provider == "qq") .execution_options(populate_existing=True) )).scalar_one() - assert identity.subject == "openid-original", "已有 qq 绑定被登录路径静默顶掉了" + assert identity.subject_plain == "openid-original", "已有 qq 绑定被登录路径静默顶掉了" @pytest.mark.asyncio @@ -142,7 +142,7 @@ async def test_sso_merge_still_binds_when_slot_is_free( .where(UserIdentity.user_id == user.id, UserIdentity.provider == "qq") .execution_options(populate_existing=True) )).scalar_one() - assert identity.subject == "openid-new" + assert identity.subject_plain == "openid-new" @pytest.mark.asyncio diff --git a/tests/integration/test_legacy_login_compat.py b/tests/integration/test_legacy_login_compat.py index 7075119e..4224a270 100644 --- a/tests/integration/test_legacy_login_compat.py +++ b/tests/integration/test_legacy_login_compat.py @@ -21,6 +21,7 @@ UpdatePasswordRequest, ) from src.common.exceptions import ValidationError +from src.common.crypto import aad_for, blind_index, encrypt, ip_prefix from src.common.security.password import verify_password from src.db_model.user import User from src.db_model.user_identity import UserIdentity @@ -55,24 +56,32 @@ async def _make_user( salt: str | None = None, ) -> User: now = datetime.now(UTC) + user_id = f"legacy-{email.split('@')[0]}" + ip = "198.51.100.9" user = User( - id=f"legacy-{email.split('@')[0]}", + id=user_id, nickname="legacy", password_hash=password_hash, legacy_salt=salt, removed=False, register_date=now, - register_ip_address="198.51.100.9", + register_ip_enc=encrypt(ip, aad_for("user", "register_ip", user_id)), + register_ip_prefix=ip_prefix(ip), register_device_id="", ) user.identities.append( UserIdentity( provider="email", - subject=email, + subject_enc=encrypt( + email, aad_for("user_identity", "subject", user_id) + ), + subject_idx=blind_index("email", email), verified=True, verified_at=now, created_at=now, - created_ip="198.51.100.9", + created_ip_enc=encrypt( + ip, aad_for("user_identity", "created_ip", user_id) + ), created_device_id="", ) ) diff --git a/tests/integration/test_sso_flows.py b/tests/integration/test_sso_flows.py index ae60c1f9..2a7f45cc 100644 --- a/tests/integration/test_sso_flows.py +++ b/tests/integration/test_sso_flows.py @@ -41,13 +41,13 @@ async def test_bind_sso_creates_qq_identity(session, session_maker, patch_redis) row = ( await session.execute( select(UserIdentity).where( - UserIdentity.provider == "qq", UserIdentity.subject == "test-openid-123" + UserIdentity.provider == "qq", UserIdentity.user_id == user.id ) ) ).scalar_one() assert row.user_id == user.id assert row.verified is True - assert row.created_ip == "9.9.9.9" + assert row.created_ip_plain == "9.9.9.9" assert row.created_device_id == "dev-1" @@ -114,7 +114,7 @@ async def test_login_merges_pending_sso_session(user_service, patch_redis): from src.apps.user.identity import IdentityProvider user = await user_service.identities.resolve(IdentityProvider.EMAIL, "m@example.com") - assert user.identity("qq").subject == "from-oauth" + assert user.identity("qq").subject_plain == "from-oauth" assert await patch_redis.get(f"sso-session:{sid}") is None, "sid must be one-shot" diff --git a/tests/unit/test_crypto.py b/tests/unit/test_crypto.py new file mode 100644 index 00000000..5be64858 --- /dev/null +++ b/tests/unit/test_crypto.py @@ -0,0 +1,93 @@ +"""Unit tests for ``src.common.crypto``. + +Cover the PII encryption primitives: roundtrip, AAD binding, blind-index +determinism/provider binding, DEK presence/length fail-fast, fingerprint and +the /24 prefix helper. +""" + +from __future__ import annotations + +import base64 +import os + +import pytest +from cryptography.exceptions import InvalidTag + +from src.common import crypto + + +@pytest.fixture +def fresh_dek(tmp_path, monkeypatch): + """Point the module at a throwaway DEK and reset its key caches.""" + dek = base64.b64encode(os.urandom(32)).decode() + dek_file = tmp_path / "pii_dek" + dek_file.write_text(dek, encoding="utf-8") + monkeypatch.setattr(crypto, "DEK_PATH", dek_file) + crypto._dek.cache_clear() + crypto._content_key.cache_clear() + crypto._index_key.cache_clear() + return dek_file + + +def test_encrypt_roundtrip(fresh_dek): + aad = crypto.aad_for("user_identity", "subject", "u1") + blob = crypto.encrypt("alice@example.com", aad) + assert isinstance(blob, bytes) + assert blob[:1] == crypto.VERSION + assert crypto.decrypt(blob, aad) == "alice@example.com" + + +def test_encrypt_empty_returns_none(fresh_dek): + assert crypto.encrypt("", b"aad") is None + assert crypto.encrypt(None, b"aad") is None + assert crypto.decrypt(None, b"aad") is None + + +def test_aad_mismatch_fails(fresh_dek): + blob = crypto.encrypt("alice@example.com", crypto.aad_for("t", "c", "u1")) + with pytest.raises(InvalidTag): + crypto.decrypt(blob, crypto.aad_for("t", "c", "u2")) + + +def test_blind_index_deterministic(fresh_dek): + a = crypto.blind_index("email", "alice@example.com") + b = crypto.blind_index("email", "alice@example.com") + assert a == b + assert isinstance(a, bytes) and len(a) == 32 + + +def test_blind_index_provider_and_subject_bound(fresh_dek): + base = crypto.blind_index("email", "alice@example.com") + assert base != crypto.blind_index("phone", "alice@example.com") + assert base != crypto.blind_index("email", "bob@example.com") + + +def test_missing_dek_raises(monkeypatch): + crypto._dek.cache_clear() + crypto._content_key.cache_clear() + crypto._index_key.cache_clear() + monkeypatch.setattr(crypto, "DEK_PATH", crypto.DEK_PATH / "does-not-exist") + with pytest.raises(RuntimeError, match="missing"): + crypto._dek() + + +def test_bad_length_dek_raises(tmp_path, monkeypatch): + bad = tmp_path / "bad_dek" + bad.write_text(base64.b64encode(b"short").decode(), encoding="utf-8") + crypto._dek.cache_clear() + crypto._content_key.cache_clear() + crypto._index_key.cache_clear() + monkeypatch.setattr(crypto, "DEK_PATH", bad) + with pytest.raises(RuntimeError, match="length"): + crypto._dek() + + +def test_fingerprint_stable(fresh_dek): + assert crypto.fingerprint() == crypto.fingerprint() + assert len(crypto.fingerprint()) == 8 + + +def test_ip_prefix(): + assert crypto.ip_prefix("203.0.113.7") == "203.0.113.0/24" + assert crypto.ip_prefix("") is None + assert crypto.ip_prefix("not-an-ip") is None From 9f5cfa5690a468e230b83a7837967f07e78e1322 Mon Sep 17 00:00:00 2001 From: zfqxuz Date: Fri, 30 Oct 2026 00:46:08 +0800 Subject: [PATCH 07/11] test(identity): expect the blind-index unique constraint name --- tests/integration/test_identity_pg_constraints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_identity_pg_constraints.py b/tests/integration/test_identity_pg_constraints.py index 935e8b2f..1adff205 100644 --- a/tests/integration/test_identity_pg_constraints.py +++ b/tests/integration/test_identity_pg_constraints.py @@ -41,7 +41,7 @@ async def test_same_subject_twice_rejected(pg_session): pg_session.add(build_user(email=subject)) with pytest.raises(IntegrityError) as exc_info: await pg_session.commit() - assert "uq_user_identity_provider_subject" in str(exc_info.value) + assert "uq_user_identity_provider_idx" in str(exc_info.value) @pytest.mark.asyncio From cc8045c272f1339817716dc28f2f4ff44c31eeb1 Mon Sep 17 00:00:00 2001 From: zfqxuz Date: Fri, 30 Oct 2026 00:56:49 +0800 Subject: [PATCH 08/11] fix(deploy): mount DEK via relative bind path, works for test and prod --- docker/docker-compose.prod.yml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index 30a81c19..c526d177 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -37,12 +37,9 @@ services: - "18000:8000" volumes: - ./logs:/app/logs - secrets: - - source: pii_dek - target: pii_dek - uid: "1000" - gid: "1000" - mode: "0400" + # DEK 用相对路径挂载:测试机解析到 /opt/thvote/test/secrets/,生产到 + # /opt/thvote/prod/secrets/。宿主文件权限(0400/uid1000)随 bind mount 保留。 + - ./secrets/pii_dek:/run/secrets/pii_dek:ro healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8000/health"] interval: 15s @@ -57,7 +54,3 @@ networks: volumes: redis-data: - -secrets: - pii_dek: - file: /opt/thvote/prod/secrets/pii_dek From 2fd875f548e64f748d38b8f0fd812e5776509d9c Mon Sep 17 00:00:00 2001 From: zfqxuz Date: Fri, 30 Oct 2026 01:06:02 +0800 Subject: [PATCH 09/11] fix(import): add pymongo to main deps so import script runs in image --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 441ccf63..9fc6277d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,8 @@ dependencies = [ "asyncpg>=0.29.0", "asyncmy>=0.2.0", "aiosqlite>=0.20.0", + # 运维脚本 scripts/import_legacy_voters.py 读 bson,需要 pymongo 的 bson 模块 + "pymongo>=4.0", "redis>=5.0.0", "aioredis>=2.0.0", "redis-om>=0.4.0", From 89057f5fc30f7ac14e4e7dce9e7d2087be3aa063 Mon Sep 17 00:00:00 2001 From: zfqxuz Date: Fri, 30 Oct 2026 01:08:12 +0800 Subject: [PATCH 10/11] revert: keep pymongo out of runtime deps (one-off import tool) --- pyproject.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9fc6277d..441ccf63 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,8 +12,6 @@ dependencies = [ "asyncpg>=0.29.0", "asyncmy>=0.2.0", "aiosqlite>=0.20.0", - # 运维脚本 scripts/import_legacy_voters.py 读 bson,需要 pymongo 的 bson 模块 - "pymongo>=4.0", "redis>=5.0.0", "aioredis>=2.0.0", "redis-om>=0.4.0", From 98e618f2c05fdefd7f02de938c959fb8a25ac784 Mon Sep 17 00:00:00 2001 From: zfqxuz Date: Fri, 30 Oct 2026 01:49:14 +0800 Subject: [PATCH 11/11] test(e2e): scraper login via email+password instead of SMS code --- docs/testing/scraper-e2e.md | 13 +++++++------ scripts/e2e/scraper_frontend.js | 21 +++++++++++---------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/docs/testing/scraper-e2e.md b/docs/testing/scraper-e2e.md index bef9b9d3..339a1e58 100644 --- a/docs/testing/scraper-e2e.md +++ b/docs/testing/scraper-e2e.md @@ -5,14 +5,15 @@ > 对应 B-063 的自动化缺口:单测见 `tests/unit/test_scraper_sites.py`、URL 路由见 > `tests/unit/test_scraper_parse_url.py`。 > -> ⚠️ 2026-10-29 起测试登录旁路已删除(P-4),本脚本不再能自动过验证码: -> 需要真人先触发发码、把手机/邮箱收到的验证码通过 `CODE=` 传进来。 +> ⚠️ 2026-10-29 起测试登录旁路已删除(P-4);2026-10-30 起本脚本改用 +> **邮箱+密码登录**(`loginEmailPassword`),不再依赖验证码/短信/旁路。 ## 前置 -1. 测试环境(或本地)能收到真实验证码;`CODE` 由真人现场提供 - (旧的白名单固定码 `888888` 已失效)。 -2. `19900000001` 已完成问卷(测试库该账号已有 year=12 的答案,可直接进提名页)。 +1. 测试库存在测试账号 `votetester1@touhou.ai`(带密码哈希,密码见 + `thvote-fe/test_accounts.md`)。 +2. 该账号需已过投票问卷门禁(提名页 `/doujin` 才可进);若被门禁挡, + 先人工把 `year=12` 问卷走完,或改用已过问卷的测试账号。 3. 本机有 Node 与 Playwright: ```bash @@ -25,7 +26,7 @@ npx playwright install chromium ```bash cd thvote-be-re FRONTEND_BASE=http://154.37.215.62:8082/v11 \ -PHONE=19900000001 CODE=<真人收到的6位验证码> \ +EMAIL=votetester1@touhou.ai PASSWORD= \ node scripts/e2e/scraper_frontend.js ``` diff --git a/scripts/e2e/scraper_frontend.js b/scripts/e2e/scraper_frontend.js index bfd47162..d5c857e0 100644 --- a/scripts/e2e/scraper_frontend.js +++ b/scripts/e2e/scraper_frontend.js @@ -7,20 +7,20 @@ * * Prereqs: * npm i playwright && npx playwright install chromium - * 2026-10-29 起测试登录旁路已删除(P-4):必须真人触发发码并把收到的验证码 - * 通过 CODE= 传入,固定码 888888 不再可用。 + * 2026-10-30 起改用邮箱+密码登录(loginEmailPassword):测试账号 + * votetester1@touhou.ai 带真实密码哈希,不再依赖验证码/短信/旁路。 * * Usage: * FRONTEND_BASE=http://154.37.215.62:8082/v11 \ - * PHONE=19900000001 CODE= \ + * EMAIL=votetester1@touhou.ai PASSWORD= \ * node scripts/e2e/scraper_frontend.js */ const { chromium } = require('playwright'); const BASE = process.env.FRONTEND_BASE || 'http://154.37.215.62:8082/v11'; -const PHONE = process.env.PHONE || '19900000001'; -const CODE = process.env.CODE || '888888'; +const EMAIL = process.env.EMAIL || 'votetester1@touhou.ai'; +const PASSWORD = process.env.PASSWORD || 'wlrg6M43tpI6919i'; const OUT = process.env.E2E_OUT_DIR || '/tmp/thvote-scraper-e2e'; // 测试机 154.37.215.62 的出口 IP 被 B 站风控时只能验收“友好报错”; // 在未被风控的网络(或配了 B 站代理)上设 BILI_EXPECT=ok 验收真实抓取。 @@ -70,17 +70,18 @@ async function closeNotice(page) { const errors = []; page.on('pageerror', e => errors.push(String(e).slice(0, 160))); - // login through the real modal + // login through the real modal (email + password, no SMS) await page.goto(BASE + '/', { waitUntil: 'domcontentloaded', timeout: 60000 }); await page.waitForTimeout(4000); await page.locator('a.float-arrow-box').click(); await page.waitForTimeout(1000); - await page.locator('input[placeholder="手机号(仅限国内)或邮箱"]').fill(PHONE); - await page.locator('input[placeholder="验证码"]').fill(CODE); + await page.getByText('使用旧版账号密码登录', { exact: true }).click(); + await page.locator('input[placeholder="邮箱"]').fill(EMAIL); + await page.locator('input[placeholder="密码"]').fill(PASSWORD); const loginResp = page - .waitForResponse(r => r.url().includes('/graphql') && r.request().postData()?.includes('loginPhone'), { timeout: 30000 }) + .waitForResponse(r => r.url().includes('/graphql') && r.request().postData()?.includes('loginEmailPassword'), { timeout: 30000 }) .catch(() => null); - await page.locator('button', { hasText: '开始投票' }).last().click(); + await page.locator('button', { hasText: '登录' }).last().click(); await loginResp; await page.waitForTimeout(5000); const session = await page.evaluate(() => !!localStorage.getItem('sessionToken'));