diff --git a/.github/workflows/deploy-test.yml b/.github/workflows/deploy-test.yml index bdc495aa..b2cb7d1a 100644 --- a/.github/workflows/deploy-test.yml +++ b/.github/workflows/deploy-test.yml @@ -36,6 +36,11 @@ on: required: false default: 'false' type: boolean + deploy_prod: + description: '同时部署到生产环境' + required: false + default: 'false' + type: boolean env: REGISTRY: ghcr.io @@ -340,6 +345,141 @@ jobs: docker image prune -f echo "后端部署完成,地址: http://localhost:18000" + # ==================== 部署到生产服务器 ==================== + # 触发方式:workflow_dispatch + 勾选 deploy_prod=true。 + # 硬性要求:build-backend 与 deploy-test 都成功后才允许执行——生产用的就是 + # 测试环境刚验证过的那个 digest,不存在"另build一份"的漂移。 + # 依赖(redis/nacos)不在本文件里:生产机 /opt/thvote/prod/docker-compose.deps.yml + # 单独维护,否则 CI 每次覆盖 docker-compose.yml 会把 nacos 冲掉。 + deploy-prod: + name: 部署到生产环境 + runs-on: ubuntu-latest + needs: [build-backend, deploy-test] + if: ${{ needs.build-backend.result == 'success' && needs.deploy-test.result == 'success' && github.event_name == 'workflow_dispatch' && inputs.deploy_prod }} + + steps: + - name: 检出部署配置 + uses: actions/checkout@v4 + with: + sparse-checkout: | + docker + sparse-checkout-cone-mode: false + + - name: 上传 compose 模板到生产服务器 + uses: appleboy/scp-action@v0.1.7 + with: + host: ${{ secrets.PROD_SERVER_HOST }} + username: ${{ secrets.PROD_SERVER_USER }} + key: ${{ secrets.PROD_SERVER_SSH_KEY }} + source: "docker/docker-compose.prod.yml" + target: "${{ vars.PROD_SERVER_DIR || '/opt/thvote/prod' }}" + strip_components: 1 + + - name: 部署到生产服务器 + uses: appleboy/ssh-action@v1 + env: + NACOS_SERVER_ADDRS: ${{ secrets.PROD_NACOS_SERVER_ADDRS }} + NACOS_NAMESPACE: ${{ secrets.PROD_NACOS_NAMESPACE }} + NACOS_USERNAME: ${{ secrets.PROD_NACOS_USERNAME }} + NACOS_PASSWORD: ${{ secrets.PROD_NACOS_PASSWORD }} + # 与 deploy-test 同一个不可变 digest + BACKEND_IMAGE: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-backend@${{ needs.build-backend.outputs.backend-digest }} + REGISTRY: ${{ env.REGISTRY }} + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + host: ${{ secrets.PROD_SERVER_HOST }} + username: ${{ secrets.PROD_SERVER_USER }} + key: ${{ secrets.PROD_SERVER_SSH_KEY }} + envs: NACOS_SERVER_ADDRS,NACOS_NAMESPACE,NACOS_USERNAME,NACOS_PASSWORD,BACKEND_IMAGE,REGISTRY,GITHUB_ACTOR,GITHUB_TOKEN + script: | + set -e + DEPLOY_DIR="${{ vars.PROD_SERVER_DIR || '/opt/thvote/prod' }}" + mkdir -p "$DEPLOY_DIR/logs" + cd "$DEPLOY_DIR" + + echo "$GITHUB_TOKEN" | docker login "$REGISTRY" -u "$GITHUB_ACTOR" --password-stdin + + # redis 由 docker-compose.deps.yml 维护(与 CI 覆盖的 docker-compose.yml 分离) + if ! docker ps --format '{{.Names}}' | grep -q '^thvote-redis$'; then + docker compose -f docker-compose.deps.yml up -d redis + fi + + # Nacos 必须先跑:后端在 import 阶段就要读配置,读不到会 crash-loop + if ! docker ps --format '{{.Names}}' | grep -q '^mynacos$'; then + echo "Nacos 容器(mynacos)未运行,后端将拿不到任何配置。请先启动 mynacos。终止部署。" + exit 1 + fi + echo "nacos 就绪" + + # .env 只放 Nacos 连接信息 + 镜像地址,业务配置全在 Nacos + umask 077 + { + echo "NACOS_ENABLED=true" + echo "NACOS_SERVER_ADDRS=${NACOS_SERVER_ADDRS}" + echo "NACOS_NAMESPACE=${NACOS_NAMESPACE}" + echo "NACOS_GROUP=DEFAULT_GROUP" + echo "NACOS_DATA_ID=thvote_be" + echo "NACOS_USERNAME=${NACOS_USERNAME}" + echo "NACOS_PASSWORD=${NACOS_PASSWORD}" + echo "BACKEND_IMAGE=${BACKEND_IMAGE}" + } > .env + chmod 600 .env + + # 仅覆盖 compose;deps 文件不动 + cp docker-compose.prod.yml docker-compose.yml + + # 拉取镜像;失败必须硬红(不回退到旧镜像) + docker pull "${BACKEND_IMAGE}" || { sleep 5; docker pull "${BACKEND_IMAGE}"; } || { echo "镜像拉取失败,终止"; exit 1; } + + # 应用 Alembic 迁移(DATABASE_URL 由 Nacos 下发) + echo "执行 alembic upgrade head ..." + docker run --rm --network thvote-net -e NACOS_ENABLED=true -e NACOS_SERVER_ADDRS="${NACOS_SERVER_ADDRS}" -e NACOS_NAMESPACE="${NACOS_NAMESPACE}" -e NACOS_GROUP=DEFAULT_GROUP -e NACOS_DATA_ID=thvote_be -e NACOS_USERNAME="${NACOS_USERNAME}" -e NACOS_PASSWORD="${NACOS_PASSWORD}" "${BACKEND_IMAGE}" alembic upgrade head + + # 启动后端(--force-recreate:即使镜像未变也重建,便于 Nacos 配置变更后重启) + docker compose up -d --force-recreate backend + + # 等待健康检查 + for i in $(seq 1 40); do + BACKEND_HEALTH="$(docker inspect -f '{{.State.Health.Status}}' thvote-backend 2>/dev/null || true)" + if [ "$BACKEND_HEALTH" = "healthy" ]; then + echo "后端就绪" + break + fi + if [ "$BACKEND_HEALTH" = "unhealthy" ]; then + echo "后端已进入 unhealthy 状态" + docker compose logs backend | tail -50 + exit 1 + fi + echo "等待后端启动... ($i/40), 当前状态: ${BACKEND_HEALTH:-starting}" + sleep 3 + done + + BACKEND_HEALTH="$(docker inspect -f '{{.State.Health.Status}}' thvote-backend 2>/dev/null || true)" + if [ "$BACKEND_HEALTH" != "healthy" ]; then + echo "后端健康检查失败,当前状态: ${BACKEND_HEALTH:-unknown}" + docker compose logs backend | tail -50 + exit 1 + fi + + # 冒烟:P-1 文档关闭 + P-4 测试旁路已删 + for PATH_ in /docs /openapi.json /redoc; do + CODE_="$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:18000${PATH_}")" + if [ "$CODE_" != "404" ]; then + echo "P-1 未生效:${PATH_} 返回 ${CODE_}(应为 404)" + exit 1 + fi + done + LOGIN_CODE="$(curl -s -o /dev/null -w '%{http_code}' -X POST "http://127.0.0.1:18000/api/v1/user/login-phone" -H 'Content-Type: application/json' -d '{"phone":"19900000001","code":"888888"}')" + if [ "$LOGIN_CODE" = "200" ]; then + echo "P-4 未生效:测试旁路账号仍能登录" + exit 1 + fi + echo "冒烟通过:/docs 404,旁路登录 ${LOGIN_CODE}" + + docker image prune -f + echo "生产部署完成,地址: http://localhost:18000" + # ==================== 通知 ==================== notify: name: 通知 diff --git a/alembic/versions/0022_add_pii_encryption.py b/alembic/versions/0022_add_pii_encryption.py new file mode 100644 index 00000000..e2605617 --- /dev/null +++ b/alembic/versions/0022_add_pii_encryption.py @@ -0,0 +1,116 @@ +"""0022 add encrypted PII columns and migrate the identity unique constraint. + +Adds AES-GCM ciphertext columns and HMAC blind-index columns so the plaintext +``subject`` / ``register_ip_address`` / ``created_ip`` can be retired: + +- ``user_identity.subject_enc`` / ``subject_idx`` (email/phone/qq/thbwiki) +- ``user_identity.created_ip_enc`` (per-identity bind IP) +- ``user.register_ip_enc`` / ``register_ip_prefix`` (account register IP + /24) + +The old plaintext columns are kept (made nullable, server defaults dropped) so +they end up NULL for every new write; migration 0023 drops them after the +import has been verified. The ``(provider, subject)`` unique constraint is +replaced by ``(provider, subject_idx)`` so equality lookup keeps working on the +blind index. + +``register_ip_prefix`` is a plain ``String`` (CIDR ``a.b.c.0/24``) rather than +``INET`` to keep the ORM model portable across Postgres and the sqlite test +database; there is no range-query consumer yet. +""" + +from alembic import op +import sqlalchemy as sa + +revision = "0022" +down_revision = "0021" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # Encrypted + blind-index columns. + op.add_column( + "user_identity", sa.Column("subject_enc", sa.LargeBinary(), nullable=True) + ) + op.add_column( + "user_identity", sa.Column("subject_idx", sa.LargeBinary(), nullable=True) + ) + op.add_column( + "user_identity", sa.Column("created_ip_enc", sa.LargeBinary(), nullable=True) + ) + op.add_column("user", sa.Column("register_ip_enc", sa.LargeBinary(), nullable=True)) + op.add_column( + "user", sa.Column("register_ip_prefix", sa.String(length=64), nullable=True) + ) + + # Plaintext columns become NULL-able; drop the "" server defaults so a + # missing value is NULL (and the §10.2 "is not null → 0" scan holds). + op.alter_column( + "user_identity", + "subject", + existing_type=sa.String(length=255), + nullable=True, + ) + op.alter_column( + "user", + "register_ip_address", + existing_type=sa.String(length=64), + nullable=True, + server_default=None, + ) + op.alter_column( + "user_identity", + "created_ip", + existing_type=sa.String(length=64), + nullable=True, + server_default=None, + ) + + # Unique constraint migrates from plaintext subject to the blind index. + op.drop_constraint( + "uq_user_identity_provider_subject", "user_identity", type_="unique" + ) + op.create_index( + "uq_user_identity_provider_idx", + "user_identity", + ["provider", "subject_idx"], + unique=True, + ) + op.create_index( + "ix_user_identity_subject_idx", "user_identity", ["subject_idx"] + ) + + +def downgrade() -> None: + op.drop_index("ix_user_identity_subject_idx", table_name="user_identity") + op.drop_index("uq_user_identity_provider_idx", table_name="user_identity") + op.create_unique_constraint( + "uq_user_identity_provider_subject", + "user_identity", + ["provider", "subject"], + ) + op.alter_column( + "user_identity", + "created_ip", + existing_type=sa.String(length=64), + nullable=False, + server_default="", + ) + op.alter_column( + "user", + "register_ip_address", + existing_type=sa.String(length=64), + nullable=False, + server_default="", + ) + op.alter_column( + "user_identity", + "subject", + existing_type=sa.String(length=255), + nullable=False, + ) + op.drop_column("user", "register_ip_prefix") + op.drop_column("user", "register_ip_enc") + op.drop_column("user_identity", "created_ip_enc") + op.drop_column("user_identity", "subject_idx") + op.drop_column("user_identity", "subject_enc") diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index b5dcb317..c526d177 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -32,10 +32,14 @@ services: NACOS_PASSWORD: "${NACOS_PASSWORD}" APP_HOST: "0.0.0.0" APP_PORT: "8000" + PII_DEK_PATH: "/run/secrets/pii_dek" ports: - "18000:8000" volumes: - ./logs:/app/logs + # DEK 用相对路径挂载:测试机解析到 /opt/thvote/test/secrets/,生产到 + # /opt/thvote/prod/secrets/。宿主文件权限(0400/uid1000)随 bind mount 保留。 + - ./secrets/pii_dek:/run/secrets/pii_dek:ro healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8000/health"] interval: 15s diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index fb64e2d6..d57c7889 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -5,6 +5,15 @@ > 创建日期:2026-04-27 > **2026-08-31 整理**:合并 9 组重复条目、按日期倒序重排;**2026-07-01 之前**的条目已迁至 [CHANGELOG-archive-2026H1.md](./CHANGELOG-archive-2026H1.md)。 +## [2026-10-29] 生产上线前加固:关闭 API 文档 + 移除测试登录旁路 + deploy-prod 流水线 + +- **Added**:`Settings.expose_api_docs`(`EXPOSE_API_DOCS`,默认 `false`)。`src/main.py` 按开关传 `docs_url`/`redoc_url`/`openapi_url`,并给 `AppGraphQLRouter` 传 `graphql_ide`——关闭时 `/docs`、`/redoc`、`/openapi.json`、浏览器打开 `/graphql` 均 404,`POST /graphql` 不受影响。回归用例 `tests/unit/test_expose_api_docs.py`(8 例)。 +- **Removed**:测试登录旁路整体删除(P-4):`src/common/verification/test_bypass.py`、`sms_code.py`/`email_code.py` 两个 `consume` 短路、`Settings` 两个字段、`tests/unit/test_test_login_bypass.py`。测试/生产 Nacos 同步删键;`grep -r TEST_LOGIN_BYPASS src` 为空。设计稿与操作文档标记为已移除,`scripts/e2e/scraper_frontend.js` 改为需真实验证码。 +- **Added**:`.github/workflows/deploy-test.yml` 新增 `deploy_prod` 手动触发开关与 `deploy-prod` job(`needs: [build-backend, deploy-test]`,复用同一 digest 部署到生产机 `/opt/thvote/prod`);compose 未变,依赖(redis/nacos)由生产目录的 `docker-compose.deps.yml` 单独维护。 +- **Fixed**:`TRUSTED_PROXY_IPS` / `ADMIN_ALLOWED_IPS` 改为 `NoDecode` + 宽容解析,接受 `""`、JSON 数组、逗号分隔三种写法。此前把这两个键按文档写成空串会让 `Settings()` 在 import 期抛 `SettingsError`、进程起不来(生产首次部署实测踩中)。用例 `tests/unit/test_config_ip_lists.py`。 +- **兼容性影响**:生产环境 graphQL 文档/调试界面不可用属预期;调用方若依赖 `/openapi.json` 生成客户端,需在测试环境用 `EXPOSE_API_DOCS=true`。 +- **数据迁移 / 配置变更**:生产 Nacos 新建 namespace `thvote-prod` + dataId `thvote_be`;`ADMIN_ALLOWED_IPS`/`TRUSTED_PROXY_IPS` 留空、`EXPOSE_API_DOCS=false`。未导入用户数据、未改前端仓库。 + ## [2026-10-22] 详情问卷保留筛选条件与组合目标 - **Fixed**:角色/音乐/组合详情问卷将原约束与目标对象通过 AND 合并,保留 OR 分组;三个本届榜单的指令模式详情链接也携带原约束。沿用已有压缩 URL 参数并兼容旧链接。 diff --git a/docs/README.md b/docs/README.md index dee95108..a8e50bec 100644 --- a/docs/README.md +++ b/docs/README.md @@ -138,7 +138,7 @@ | [`2026-07-17-submit-timing-signal-design.md`](./superpowers/specs/2026-07-17-submit-timing-signal-design.md) | 提交耗时 + 服务端改票计数(B-045/B-046,`fill_duration_ms` + `attempt`;改票假阳性由 attempt 兜底) | | [`2026-07-17-block-scripts-design.md`](./superpowers/specs/2026-07-17-block-scripts-design.md) | 拦脚本(B-048,Origin/Referer 校验;只拦变更、query 放行;**默认关灰度**) | | [`2026-06-08-security-backend-design.md`](./superpowers/specs/2026-06-08-security-backend-design.md) | 安全块后端(B-037):二创提名校验 + 人工审核队列 + 提名时间窗 + 投票问卷门禁 | -| [`2026-08-14-test-login-bypass-design.md`](./superpowers/specs/2026-08-14-test-login-bypass-design.md) | ⚠️ 测试登录旁路(白名单固定验证码)。**临时测试设施,公开上线前必须整体移除**:`grep -r TEST_LOGIN_BYPASS` | +| [`2026-08-14-test-login-bypass-design.md`](./superpowers/specs/2026-08-14-test-login-bypass-design.md) | ❌ 已移除(P-4,2026-10-29)测试登录旁路(白名单固定验证码)历史存档;代码与 Nacos 键均已删除 | ### 管理端 diff --git a/docs/operations/captcha-onboarding.md b/docs/operations/captcha-onboarding.md index 630ab6fc..1c5e8f94 100644 --- a/docs/operations/captcha-onboarding.md +++ b/docs/operations/captcha-onboarding.md @@ -97,18 +97,12 @@ curl -s -X POST http://154.37.215.62:18000/graphql -H 'content-type: application > 前后端的 SceneId 必须是**同一个场景**;前端 prefix 与后端 AK 必须是**同一个账户**——混用新旧账户的任意组合都会校验失败(阿里云侧对不上)。 -## 六点五、测试期登录旁路(TEST_LOGIN_BYPASS,上线前移除) - -自动化测试(浏览器端到端)无法过滑块、也收不到真实验证码。测试环境提供**验码侧白名单固定码**旁路(设计稿 [test-login-bypass](../superpowers/specs/2026-08-14-test-login-bypass-design.md)): - -- 测试 Nacos `thvote_be` 配两个 key(**生产 dataId 永不配置**): - ```json - "TEST_LOGIN_BYPASS_ACCOUNTS": "[\"19900000001\", \"test1@example.com\"]", - "TEST_LOGIN_BYPASS_CODE": "888888" - ``` -- 用法:登录页填白名单假号 → 验证码直接填固定码 → 登录。**不要点"发送验证码"**——旁路只挂在验码侧,发码链路(captcha/PNVS/限流)原样不动,真人真号不受影响。 -- 每次命中打 WARNING 日志(账号脱敏)。改完 Nacos 要重启容器(B-017,workflow_dispatch 即可)。 -- **移除条件**:公开上线前删除,`grep -r TEST_LOGIN_BYPASS` 全量定位(模块/短路行/config 字段/Nacos key/本节)。 +## 六点五、测试期登录旁路(已于 2026-10-29 移除) + +> ❌ **本节已失效**:白名单固定验证码旁路(P-4)已于 2026-10-29 从代码与 Nacos 中整体删除 +> (`src/common/verification/test_bypass.py`、两个 `consume` 短路、`Settings` 两个字段)。 +> 现在所有账号(含测试假号)都必须走真实发码 + 真实验码,自动化端到端测试需要真人提供验证码。 +> 历史设计与取舍见 [test-login-bypass](../superpowers/specs/2026-08-14-test-login-bypass-design.md)。 ## 七、常见坑速查 diff --git a/docs/operations/production-readiness-checklist.md b/docs/operations/production-readiness-checklist.md index a33195d4..fd68a77c 100644 --- a/docs/operations/production-readiness-checklist.md +++ b/docs/operations/production-readiness-checklist.md @@ -8,10 +8,10 @@ | # | 项 | 现状(2026-09-13) | 要做什么 | 怎么验证 | |---|---|---|---|---| -| P-1 | **关闭 `/docs`、`/redoc`、`/openapi.json` 和 GraphiQL** | `src/main.py` 创建 `FastAPI(...)` 时没传 `docs_url=None` 等;`AppGraphQLRouter(graphql_schema)` 没传 `graphql_ide=None`,Strawberry 默认开 GraphiQL。测试机实测四个地址都 200,`openapi.json` 110KB 含全部 admin 端点定义,经 nginx `/v12-be/` 公网可达 | 加一个配置开关(建议 `EXPOSE_API_DOCS`,默认 `false`),生产关、测试机可开。改 `src/main.py` 两处 | `curl -o /dev/null -w '%{http_code}' https:///v12-be/docs` 与 `/openapi.json`、`GET /graphql`(`Accept: text/html`)都应为 404 | +| P-1 | **关闭 `/docs`、`/redoc`、`/openapi.json` 和 GraphiQL** | ✅ **2026-10-29 已做**:`Settings.expose_api_docs`(`EXPOSE_API_DOCS`)默认 `false`,`src/main.py` 按开关传 `docs_url`/`redoc_url`/`openapi_url` 与 `graphql_ide`;生产 Nacos 显式写 `EXPOSE_API_DOCS=false`,测试机可开 `true` | 已加开关;回归用例 `tests/unit/test_expose_api_docs.py` | 生产实测 `/docs`、`/redoc`、`/openapi.json`、`GET /graphql`(`Accept: text/html`)均 404,`POST /graphql` 仍 200 | | P-2 | **admin IP 白名单必须配置** | `require_admin` 的 IP 检查在 `ADMIN_ALLOWED_IPS` 为空时**放行任意 IP**(`src/apps/admin/deps.py` `_ip_allowed`,设计上的逃生舱)。测试机当前为空,只靠 `X-Admin-Secret` 一道门 | Nacos `thvote_be` 配 `ADMIN_ALLOWED_IPS`(运营出口 IP / VPN 网段,CIDR 可用)。同时 `TRUSTED_PROXY_IPS` 必须填 nginx 容器/主机地址,否则后端看到的永远是代理 IP,白名单形同虚设 | 白名单外 IP 带正确密钥访问 `/api/v1/admin/users` 应 403;白名单内 200 | | P-3 | **`ADMIN_SECRET` 换强值** | 测试机为弱值 `abc123`(B-042) | 生成 ≥32 字节随机串写入 Nacos;admin-ui 登录框重新输入 | 旧值 403 | -| P-4 | **移除测试登录旁路** | `TEST_LOGIN_BYPASS_*` 代码与配置仍在(`grep -r TEST_LOGIN_BYPASS src`)。默认配置为空时不生效,但公开前应整体删除,不留"配错即开门"的可能 | 删代码 + Nacos 删键;见 `docs/superpowers/specs/2026-08-14-test-login-bypass-design.md` 移除条件 | `grep -r TEST_LOGIN_BYPASS src` 为空;假号 19900000001 + 888888 登录失败 | +| P-4 | **移除测试登录旁路** | ✅ **2026-10-29 已做**:`src/common/verification/test_bypass.py`、两个 `consume` 短路、`Settings` 两个字段、单测全部删除;生产 Nacos 不含旧键(`grep -r TEST_LOGIN_BYPASS src` 为空) | 已删;设计稿归档为已移除 | `grep -r TEST_LOGIN_BYPASS src` 为空;假号 19900000001 + 888888 登录失败 | | P-5 | **CORS 收紧** | `CORS_ALLOWED_ORIGINS` 默认 `["*"]` 且 `allow_credentials=True`。当前无 cookie 所以无害;一旦引入 cookie 会变成任意源带凭据 | 生产填前端真实域名列表 | 预检请求带陌生 `Origin` 不应返回 `Access-Control-Allow-Origin` | | P-6 | **前端截止时间改回** | Touhou-Vote `shared/data/time.ts` 的 `deadline` 临时改成 2099(见 `docs/migration/api-contract-audit-2026-07-14.md`) | 改回真实截止时间,或切到后端 `/voting-status/`(现需 `vote_token`) | 前端在截止后拒绝提交 | diff --git a/docs/superpowers/specs/2026-08-14-test-login-bypass-design.md b/docs/superpowers/specs/2026-08-14-test-login-bypass-design.md index 5b697673..cdd81d6c 100644 --- a/docs/superpowers/specs/2026-08-14-test-login-bypass-design.md +++ b/docs/superpowers/specs/2026-08-14-test-login-bypass-design.md @@ -1,10 +1,10 @@ # 测试登录旁路(白名单固定验证码)设计 -> **状态**:已实施 —— 已于 2026-08-14 实现并上线(`TEST_LOGIN_BYPASS_ACCOUNTS`/`TEST_LOGIN_BYPASS_CODE`,见 [CHANGELOG 2026-08-14](../../CHANGELOG.md))。 -> 本文件是**设计稿**,记录当时的设计意图与取舍;实现细节可能已演进。状态核对于 2026-08-31。 +> **状态**:❌ **已于 2026-10-29 整体移除**(P-4,公开上线前清理)。移除内容:`src/common/verification/test_bypass.py`、`sms_code.py`/`email_code.py` 两个 `consume` 短路、`Settings` 两个字段、单测 `tests/unit/test_test_login_bypass.py`、测试/生产 Nacos 两个键。现在 `grep -r TEST_LOGIN_BYPASS src` 为空,本文件仅作历史存档。 +> 原始实施:2026-08-14 实现并上线(见 [CHANGELOG 2026-08-14](../../CHANGELOG.md))。 > 日期:2026-08-14 -> ⚠️ **临时测试设施——公开上线前必须整体移除**,全量定位方式:`grep -r TEST_LOGIN_BYPASS` +> ⚠️ **临时测试设施——公开上线前必须整体移除**(已于 2026-10-29 完成) ## 一、目的 diff --git a/docs/testing/scraper-e2e.md b/docs/testing/scraper-e2e.md index a1758c98..339a1e58 100644 --- a/docs/testing/scraper-e2e.md +++ b/docs/testing/scraper-e2e.md @@ -1,15 +1,19 @@ # 前端发起的 Scraper E2E > 目的:从真实前端“作品提名 → 自动获取信息”链路验证各站点爬虫,而不是只测 -> 后端 `/api/v1/scraper/scrape`。登录使用测试旁路账号,不需要真实短信。 +> 后端 `/api/v1/scraper/scrape`。 > 对应 B-063 的自动化缺口:单测见 `tests/unit/test_scraper_sites.py`、URL 路由见 > `tests/unit/test_scraper_parse_url.py`。 +> +> ⚠️ 2026-10-29 起测试登录旁路已删除(P-4);2026-10-30 起本脚本改用 +> **邮箱+密码登录**(`loginEmailPassword`),不再依赖验证码/短信/旁路。 ## 前置 -1. 测试环境已配置 `TEST_LOGIN_BYPASS_ACCOUNTS` 包含 `19900000001`, - `TEST_LOGIN_BYPASS_CODE=888888`(当前测试 Nacos 已配置)。 -2. `19900000001` 已完成问卷(测试库该账号已有 year=12 的答案,可直接进提名页)。 +1. 测试库存在测试账号 `votetester1@touhou.ai`(带密码哈希,密码见 + `thvote-fe/test_accounts.md`)。 +2. 该账号需已过投票问卷门禁(提名页 `/doujin` 才可进);若被门禁挡, + 先人工把 `year=12` 问卷走完,或改用已过问卷的测试账号。 3. 本机有 Node 与 Playwright: ```bash @@ -22,7 +26,7 @@ npx playwright install chromium ```bash cd thvote-be-re FRONTEND_BASE=http://154.37.215.62:8082/v11 \ -PHONE=19900000001 CODE=888888 \ +EMAIL=votetester1@touhou.ai PASSWORD= \ node scripts/e2e/scraper_frontend.js ``` diff --git a/pyproject.toml b/pyproject.toml index 2d344ff6..441ccf63 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,8 @@ dependencies = [ "httpx>=0.27.0", # eMudhra/emSign 事件:certifi 2026.10.19 移除受影响根;显式下限防回退 "certifi>=2026.10.19", + # PII 字段级加密(AES-256-GCM + HKDF + HMAC 盲索引)——此前仅传递依赖,必须显式声明 + "cryptography>=42.0", "pytest>=8.0.0", "pytest-asyncio>=0.23.0", "strawberry-graphql[fastapi]>=0.219.0", diff --git a/scripts/e2e/scraper_frontend.js b/scripts/e2e/scraper_frontend.js index 658f2a34..d5c857e0 100644 --- a/scripts/e2e/scraper_frontend.js +++ b/scripts/e2e/scraper_frontend.js @@ -2,24 +2,25 @@ /** * Frontend-driven scraper E2E (Playwright). * - * Logs in through the real login modal with the test-bypass phone/code, - * opens the doujin nomination editor and clicks “自动获取信息” for each URL. + * Logs in through the real login modal, opens the doujin nomination editor and + * clicks “自动获取信息” for each URL. * * Prereqs: * npm i playwright && npx playwright install chromium - * Test env must have TEST_LOGIN_BYPASS_ACCOUNTS/CODE configured. + * 2026-10-30 起改用邮箱+密码登录(loginEmailPassword):测试账号 + * votetester1@touhou.ai 带真实密码哈希,不再依赖验证码/短信/旁路。 * * Usage: * FRONTEND_BASE=http://154.37.215.62:8082/v11 \ - * PHONE=19900000001 CODE=888888 \ + * EMAIL=votetester1@touhou.ai PASSWORD= \ * node scripts/e2e/scraper_frontend.js */ const { chromium } = require('playwright'); const BASE = process.env.FRONTEND_BASE || 'http://154.37.215.62:8082/v11'; -const PHONE = process.env.PHONE || '19900000001'; -const CODE = process.env.CODE || '888888'; +const EMAIL = process.env.EMAIL || 'votetester1@touhou.ai'; +const PASSWORD = process.env.PASSWORD || 'wlrg6M43tpI6919i'; const OUT = process.env.E2E_OUT_DIR || '/tmp/thvote-scraper-e2e'; // 测试机 154.37.215.62 的出口 IP 被 B 站风控时只能验收“友好报错”; // 在未被风控的网络(或配了 B 站代理)上设 BILI_EXPECT=ok 验收真实抓取。 @@ -69,17 +70,18 @@ async function closeNotice(page) { const errors = []; page.on('pageerror', e => errors.push(String(e).slice(0, 160))); - // login through the real modal + // login through the real modal (email + password, no SMS) await page.goto(BASE + '/', { waitUntil: 'domcontentloaded', timeout: 60000 }); await page.waitForTimeout(4000); await page.locator('a.float-arrow-box').click(); await page.waitForTimeout(1000); - await page.locator('input[placeholder="手机号(仅限国内)或邮箱"]').fill(PHONE); - await page.locator('input[placeholder="验证码"]').fill(CODE); + await page.getByText('使用旧版账号密码登录', { exact: true }).click(); + await page.locator('input[placeholder="邮箱"]').fill(EMAIL); + await page.locator('input[placeholder="密码"]').fill(PASSWORD); const loginResp = page - .waitForResponse(r => r.url().includes('/graphql') && r.request().postData()?.includes('loginPhone'), { timeout: 30000 }) + .waitForResponse(r => r.url().includes('/graphql') && r.request().postData()?.includes('loginEmailPassword'), { timeout: 30000 }) .catch(() => null); - await page.locator('button', { hasText: '开始投票' }).last().click(); + await page.locator('button', { hasText: '登录' }).last().click(); await loginResp; await page.waitForTimeout(5000); const session = await page.evaluate(() => !!localStorage.getItem('sessionToken')); diff --git a/scripts/import_legacy_voters.py b/scripts/import_legacy_voters.py index a972c0fa..c9a0ec62 100644 --- a/scripts/import_legacy_voters.py +++ b/scripts/import_legacy_voters.py @@ -48,6 +48,13 @@ sys.path.insert(0, str(_project_root)) from src.common.config import get_settings # noqa: E402 +from src.common.crypto import ( # noqa: E402 + aad_for, + blind_index, + encrypt, + fingerprint, + ip_prefix, +) from src.common.database import normalize_async_database_url # noqa: E402 from src.db_model.legacy_user_mapping import LegacyUserMapping # noqa: E402 from src.db_model.user import User # noqa: E402 @@ -308,11 +315,17 @@ def _identity_rows(voter: LegacyVoter, user_id: str) -> list[dict[str, Any]]: { "user_id": user_id, "provider": provider, - "subject": subject, + "subject_enc": encrypt( + subject, aad_for("user_identity", "subject", user_id) + ), + "subject_idx": blind_index(provider, subject), "verified": bool(verified), "verified_at": voter.created_at if verified else None, "created_at": voter.created_at, - "created_ip": voter.signup_ip, + "created_ip_enc": encrypt( + voter.signup_ip or None, + aad_for("user_identity", "created_ip", user_id), + ), "created_device_id": "", "last_login_at": None, } @@ -329,41 +342,44 @@ def _user_row(voter: LegacyVoter, user_id: str) -> dict[str, Any]: "legacy_salt": voter.legacy_salt if voter.is_bcrypt else None, "removed": voter.removed, "register_date": voter.created_at, - "register_ip_address": voter.signup_ip, + "register_ip_enc": encrypt( + voter.signup_ip or None, aad_for("user", "register_ip", user_id) + ), + "register_ip_prefix": ip_prefix(voter.signup_ip), "register_device_id": "", } async def _existing_identities(session, voters: list[LegacyVoter]) -> dict: - emails = [v.email for v in voters if v.email] - phones = [v.phone for v in voters if v.phone] - found: dict[tuple[str, str], str] = {} - if emails: + email_idx = [blind_index("email", v.email) for v in voters if v.email] + phone_idx = [blind_index("phone", v.phone) for v in voters if v.phone] + found: dict[tuple[str, bytes], str] = {} + if email_idx: result = await session.execute( select( UserIdentity.provider, - UserIdentity.subject, + UserIdentity.subject_idx, UserIdentity.user_id, ).where( UserIdentity.provider == "email", - UserIdentity.subject.in_(emails), + UserIdentity.subject_idx.in_(email_idx), ) ) - for provider, subject, user_id in result: - found[(provider, subject)] = user_id - if phones: + for provider, subject_idx, user_id in result: + found[(provider, subject_idx)] = user_id + if phone_idx: result = await session.execute( select( UserIdentity.provider, - UserIdentity.subject, + UserIdentity.subject_idx, UserIdentity.user_id, ).where( UserIdentity.provider == "phone", - UserIdentity.subject.in_(phones), + UserIdentity.subject_idx.in_(phone_idx), ) ) - for provider, subject, user_id in result: - found[(provider, subject)] = user_id + for provider, subject_idx, user_id in result: + found[(provider, subject_idx)] = user_id return found @@ -387,16 +403,16 @@ async def _load_mapped(session, *, commit: bool) -> set[str]: def _classify( - voter: LegacyVoter, existing: dict[tuple[str, str], str] + voter: LegacyVoter, existing: dict[tuple[str, bytes], str] ) -> tuple[str, str | None]: """Return (kind, owner_user_id). kind ∈ create / conflict / multi.""" owners: dict[str, str] = {} if voter.email: - owner = existing.get(("email", voter.email)) + owner = existing.get(("email", blind_index("email", voter.email))) if owner: owners[owner] = "email" if voter.phone: - owner = existing.get(("phone", voter.phone)) + owner = existing.get(("phone", blind_index("phone", voter.phone))) if owner: owners[owner] = "phone" if not owners: @@ -410,7 +426,7 @@ async def _process_batch( session, winners: list[LegacyVoter], plan: ImportPlan, - existing: dict[tuple[str, str], str], + existing: dict[tuple[str, bytes], str], mapped: set[str], batch_id: str, stats: Stats, @@ -614,6 +630,7 @@ async def run( dump_dir: Path, *, include_test_accounts: bool, + only_test_accounts: bool, commit: bool, batch_id: str, report_dir: Path, @@ -621,11 +638,14 @@ async def run( limit: int | None, ) -> Stats: started = datetime.now(UTC) + logger.info("PII DEK loaded: fp=%s", fingerprint()) voters: list[LegacyVoter] = [] for index, doc in enumerate(iter_voters(dump_dir / "voters.bson")): if limit is not None and index >= limit: break voters.append(normalize_voter(doc)) + if only_test_accounts: + voters = [v for v in voters if v.email in TEST_ACCOUNT_EMAILS] last_activity = load_last_activity(dump_dir / "voter_logs.bson") plan = plan_import( voters, @@ -727,6 +747,11 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="Import votetester1..5@touhou.ai (test DB only)", ) + parser.add_argument( + "--only-test-accounts", + action="store_true", + help="Import ONLY votetester1..5@touhou.ai (smoke test)", + ) parser.add_argument( "--preflight", action="store_true", @@ -759,11 +784,14 @@ def main(argv: list[str] | None = None) -> int: batch_id = args.batch_id or str(uuid.uuid4()) mode = "commit" if args.commit else "preflight" + if args.only_test_accounts: + args.include_test_accounts = True logger.info("batch_id=%s mode=%s", batch_id, mode) stats = asyncio.run( run( dump_dir, include_test_accounts=args.include_test_accounts, + only_test_accounts=args.only_test_accounts, commit=args.commit, batch_id=batch_id, report_dir=Path(args.report_dir), diff --git a/scripts/verify_import.py b/scripts/verify_import.py new file mode 100644 index 00000000..28937ba7 --- /dev/null +++ b/scripts/verify_import.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""Read-only verification of the legacy voter import. + +Connects to the target DB via ``get_settings()`` (Nacos), then prints a +PASS/FAIL checklist: row counts, plaintext-leak scan (the red-line check), +encrypted-column coverage, and one decrypted sample that must round-trip back +through the blind index. Never writes. + +Usage (one-off container, same image as the backend): + docker run --rm --network thvote-net \ + --env-file /opt/thvote/prod/.env \ + -e PII_DEK_PATH=/run/secrets/pii_dek \ + -v /opt/thvote/prod/secrets:/run/secrets:ro \ + \ + python scripts/verify_import.py \ + [--expect-users N] [--expect-identities N] [--expect-mappings N] +""" + +from __future__ import annotations + +import argparse +import asyncio +import sys +from pathlib import Path + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import create_async_engine + +_project_root = Path(__file__).resolve().parent.parent +if str(_project_root) not in sys.path: + sys.path.insert(0, str(_project_root)) + +from src.common.config import get_settings # noqa: E402 +from src.common.crypto import aad_for, blind_index, decrypt, fingerprint # noqa: E402 +from src.common.database import normalize_async_database_url # noqa: E402 + + +async def _scalar(conn, sql: str) -> int: + return int(await conn.scalar(text(sql))) + + +async def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--expect-users", type=int, default=None) + parser.add_argument("--expect-identities", type=int, default=None) + parser.add_argument("--expect-mappings", type=int, default=None) + args = parser.parse_args(argv) + + settings = get_settings() + engine = create_async_engine( + normalize_async_database_url(settings.database_url), echo=False + ) + checks: list[tuple[str, bool, str]] = [] + + def check(name: str, ok: bool, detail: str) -> None: + checks.append((name, ok, detail)) + print(f"[{'PASS' if ok else 'FAIL'}] {name}: {detail}") + + try: + async with engine.connect() as conn: + users = await _scalar(conn, 'select count(*) from "user"') + identities = await _scalar(conn, "select count(*) from user_identity") + mappings = await _scalar(conn, "select count(*) from legacy_user_mapping") + + subject_plain = await _scalar( + conn, "select count(*) from user_identity where subject is not null" + ) + regip_plain = await _scalar( + conn, + 'select count(*) from "user" where register_ip_address is not null', + ) + created_ip_plain = await _scalar( + conn, "select count(*) from user_identity where created_ip is not null" + ) + subject_enc = await _scalar( + conn, "select count(*) from user_identity where subject_enc is not null" + ) + regip_enc = await _scalar( + conn, 'select count(*) from "user" where register_ip_enc is not null' + ) + created_ip_enc = await _scalar( + conn, + "select count(*) from user_identity where created_ip_enc is not null", + ) + + sample = ( + await conn.execute( + text( + "select provider, subject_enc, subject_idx, user_id " + "from user_identity " + "where provider='email' and subject_enc is not null limit 1" + ) + ) + ).first() + sample_ok = False + sample_detail = "" + if sample is not None: + provider, enc, idx, uid = sample + try: + plain = decrypt(enc, aad_for("user_identity", "subject", uid)) + sample_ok = bool(plain) and blind_index(provider, plain) == idx + sample_detail = plain or "" + except Exception as exc: # noqa: BLE001 - report, don't crash + sample_detail = f"" + + print("PII DEK loaded: fp=%s" % fingerprint()) + print(f"users={users} identities={identities} mappings={mappings}") + + if args.expect_users is not None: + check("user count", users == args.expect_users, str(users)) + if args.expect_identities is not None: + check( + "identity count", + identities == args.expect_identities, + str(identities), + ) + if args.expect_mappings is not None: + check( + "mapping count", mappings == args.expect_mappings, str(mappings) + ) + + check("subject plaintext is NULL", subject_plain == 0, str(subject_plain)) + check("register_ip plaintext is NULL", regip_plain == 0, str(regip_plain)) + check( + "created_ip plaintext is NULL", + created_ip_plain == 0, + str(created_ip_plain), + ) + check("subject_enc populated", subject_enc > 0, str(subject_enc)) + check("register_ip_enc populated", regip_enc > 0, str(regip_enc)) + check("created_ip_enc populated", created_ip_enc > 0, str(created_ip_enc)) + check("sample decrypt + blind-index roundtrip", sample_ok, sample_detail) + finally: + await engine.dispose() + + failed = [c for c in checks if not c[1]] + print("RESULT:", "PASS" if not failed else f"{len(failed)} CHECK(S) FAILED") + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) diff --git a/src/apps/admin/router.py b/src/apps/admin/router.py index 03c94a22..354ff275 100644 --- a/src/apps/admin/router.py +++ b/src/apps/admin/router.py @@ -141,8 +141,8 @@ def _user_to_item(u) -> dict: return { "id": u.id, "nickname": u.nickname, - "email": email.subject if email is not None else None, - "phone": phone.subject if phone is not None else None, + "email": email.subject_plain if email is not None else None, + "phone": phone.subject_plain if phone is not None else None, "email_verified": bool(email is not None and email.verified), "phone_verified": bool(phone is not None and phone.verified), "register_date": u.register_date.isoformat() if u.register_date else None, diff --git a/src/apps/user/dao.py b/src/apps/user/dao.py index ed34da07..a68db124 100644 --- a/src/apps/user/dao.py +++ b/src/apps/user/dao.py @@ -19,6 +19,8 @@ from sqlalchemy.exc import InvalidRequestError from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker +from src.apps.user.identity import IdentityProvider, normalize_subject +from src.common.crypto import blind_index from src.db_model.activity_log import ActivityLog from src.db_model.user import User from src.db_model.user_identity import UserIdentity @@ -88,9 +90,17 @@ async def search_users( ) -> tuple[list[User], int]: query = select(User) if email: - query = query.where(_has_identity_like("email", email)) + try: + canonical = normalize_subject(IdentityProvider.EMAIL, email) + except ValueError: + return [], 0 + query = query.where(_has_identity_exact("email", canonical)) if phone: - query = query.where(_has_identity_like("phone", phone)) + try: + canonical = normalize_subject(IdentityProvider.PHONE, phone) + except ValueError: + return [], 0 + query = query.where(_has_identity_exact("phone", canonical)) count_result = await self.session.execute( select(func.count()).select_from(query.subquery()) ) @@ -110,11 +120,13 @@ async def set_removed(self, user_id: str, removed: bool) -> User | None: return user -def _has_identity_like(provider: str, pattern: str): +def _has_identity_exact(provider: str, subject: str): + """EXISTS(...) over the blind index — encrypted subjects cannot be + fuzzy-matched, so admin search is exact on the canonical identifier.""" return User.identities.any( and_( UserIdentity.provider == provider, - UserIdentity.subject.ilike(f"%{pattern}%"), + UserIdentity.subject_idx == blind_index(provider, subject), ) ) @@ -134,7 +146,7 @@ async def find_identity(self, provider: str, subject: str) -> UserIdentity | Non result = await self.session.execute( select(UserIdentity).where( UserIdentity.provider == provider, - UserIdentity.subject == subject, + UserIdentity.subject_idx == blind_index(provider, subject), ) ) return result.scalar_one_or_none() @@ -146,7 +158,7 @@ async def find_user(self, provider: str, subject: str) -> User | None: .join(User.identities) .where( UserIdentity.provider == provider, - UserIdentity.subject == subject, + UserIdentity.subject_idx == blind_index(provider, subject), ) ) return result.scalar_one_or_none() diff --git a/src/apps/user/identity_service.py b/src/apps/user/identity_service.py index e82ed324..741ceed4 100644 --- a/src/apps/user/identity_service.py +++ b/src/apps/user/identity_service.py @@ -21,6 +21,7 @@ from src.apps.user.dao import UserDAO, UserIdentityDAO from src.apps.user.identity import IdentityProvider, normalize_subject from src.apps.user.schemas import Meta, generate_user_id +from src.common.crypto import aad_for, blind_index, encrypt, ip_prefix from src.common.exceptions import UnauthorizedError, ValidationError from src.db_model.user import User from src.db_model.user_identity import UserIdentity @@ -81,14 +82,20 @@ async def register( """ subject = normalize_subject(provider, raw_subject) now = datetime.now(UTC) + user_id = generate_user_id() user = User( - id=generate_user_id(), + id=user_id, nickname=nickname, - register_ip_address=meta.user_ip or "", + register_ip_enc=encrypt( + meta.user_ip or None, aad_for("user", "register_ip", user_id) + ), + register_ip_prefix=ip_prefix(meta.user_ip or ""), register_device_id=meta.additional_fingureprint or "", ) user.identities.append( - _new_identity(provider, subject, meta, now, last_login_at=now) + _new_identity( + provider, subject, meta, now, user_id=user_id, last_login_at=now + ) ) return await self.user_dao.create(user) @@ -122,13 +129,15 @@ async def bind( return BindResult(identity=existing, replaced_subject=None, changed=False) current = user.identity(provider.value) - replaced = current.subject if current is not None else None + replaced = current.subject_plain if current is not None else None if current is not None: # Flush the DELETE before the INSERT: the unit of work would # otherwise emit INSERT first and trip uq_user_identity_user_provider. user.identities.remove(current) await self.identity_dao.session.flush() - identity = _new_identity(provider, subject, meta, datetime.now(UTC)) + identity = _new_identity( + provider, subject, meta, datetime.now(UTC), user_id=user.id + ) user.identities.append(identity) await self.user_dao.save(user) return BindResult(identity=identity, replaced_subject=replaced, changed=True) @@ -168,6 +177,7 @@ def _new_identity( meta: Meta, now: datetime, *, + user_id: str, last_login_at: datetime | None = None, ) -> UserIdentity: # Every creation path today runs after verification (code / OAuth), so @@ -175,11 +185,14 @@ def _new_identity( # "record first, verify later" flows (spec §3.4). return UserIdentity( provider=provider.value, - subject=subject, + subject_enc=encrypt(subject, aad_for("user_identity", "subject", user_id)), + subject_idx=blind_index(provider.value, subject), verified=True, verified_at=now, created_at=now, - created_ip=meta.user_ip or "", + created_ip_enc=encrypt( + meta.user_ip or None, aad_for("user_identity", "created_ip", user_id) + ), created_device_id=meta.additional_fingureprint or "", last_login_at=last_login_at, ) diff --git a/src/apps/user/schemas.py b/src/apps/user/schemas.py index 2d98a4b2..2bb91511 100644 --- a/src/apps/user/schemas.py +++ b/src/apps/user/schemas.py @@ -167,8 +167,8 @@ def voter_fe_from_user(user) -> VoterFE: username=user.nickname, pfp=user.pfp, password=bool(user.password_hash), - phone=phone.subject if phone is not None else None, - email=email.subject if email is not None else None, + phone=phone.subject_plain if phone is not None else None, + email=email.subject_plain if email is not None else None, thbwiki=user.identity("thbwiki") is not None, patchyvideo=False, created_at=user.register_date, diff --git a/src/apps/user/service.py b/src/apps/user/service.py index 5a3064fd..ee7da0e0 100644 --- a/src/apps/user/service.py +++ b/src/apps/user/service.py @@ -278,7 +278,7 @@ async def _rebind_contact( event_type=event_type, user_id=user.id, old_value=bound.replaced_subject, - new_value=bound.identity.subject, + new_value=bound.identity.subject_plain, requester_ip=meta.user_ip, additional_fingerprint=meta.additional_fingureprint, ) @@ -377,7 +377,7 @@ async def _merge_sso_session(self, user: User, sid, meta: Meta) -> None: # 任何人都能再去认领;而这里只是登录顺带的 courtesy,用户并未 # 表达换绑意图。换绑走 update_* / bind_sso 的显式入口。 current = user.identity(provider.value) - if current is not None and current.subject != subject: + if current is not None and current.subject_plain != subject: logger.warning( "SSO merge skipped for user_id=%s provider=%s: " "already bound to a different subject", @@ -401,7 +401,7 @@ async def _merge_sso_session(self, user: User, sid, meta: Meta) -> None: await self._safe_log( event_type="bind_sso", user_id=user.id, - new_value=bound.identity.subject, + new_value=bound.identity.subject_plain, requester_ip=meta.user_ip, additional_fingerprint=meta.additional_fingureprint, ) diff --git a/src/common/config.py b/src/common/config.py index 400cf796..292213f9 100644 --- a/src/common/config.py +++ b/src/common/config.py @@ -287,17 +287,46 @@ def dojin_domain_allowlist(self) -> list[str]: # 安全配置 cors_allowed_origins: list[str] = Field(default_factory=lambda: ["*"]) - trusted_proxy_ips: list[str] = Field(default_factory=list) + # 下面两个 IP 列表同样要容忍 Nacos 里的各种写法。NoDecode 关掉 env source + # 的 JSON 预解析,否则写成 "" / "10.0.0.1" 会在 source 层抛 SettingsError, + # 整个进程起不来(2026-10-29 生产首次部署即因此失败)。 + trusted_proxy_ips: Annotated[list[str], NoDecode] = Field(default_factory=list) # 管理端 IP 白名单(B-049)。Nacos 里写 JSON 数组字符串, # 如 "ADMIN_ALLOWED_IPS": "[\"1.2.3.4\", \"10.0.0.0/8\"]"。 - # 空 = 不限 IP(仍需 X-Admin-Secret);pydantic-settings 自动 JSON 解码 list[str]。 - admin_allowed_ips: list[str] = Field(default_factory=list) + # 空 = 不限 IP(仍需 X-Admin-Secret)。 + admin_allowed_ips: Annotated[list[str], NoDecode] = Field(default_factory=list) + + @field_validator("trusted_proxy_ips", "admin_allowed_ips", mode="before") + @classmethod + def _parse_ip_lists(cls, value): + """接受 ``""``、JSON 数组、逗号分隔三种写法;空值 = 空列表。""" + if not isinstance(value, str): + return value + text = value.strip() + if not text: + return [] + if text[0] in "[{": + import json + + try: + parsed = json.loads(text) + except ValueError: + parsed = None + if isinstance(parsed, list): + return [str(item).strip() for item in parsed] + logger.error("IP 白名单配置 %r 不是合法的 JSON 数组,按空列表处理", value) + return [] + return [part.strip() for part in text.split(",") if part.strip()] # 拦裸脚本(B-048):对变更类请求(GraphQL mutation + REST 提交/登录/发码)要求 # 带浏览器自动附加的 Origin/Referer 头;curl/python 默认不带→403。默认关, # 经 Nacos 灰度开启(REQUIRE_BROWSER_ORIGIN=true)。不拦 query(codegen 免受影响)。 require_browser_origin: bool = Field( False, validation_alias="REQUIRE_BROWSER_ORIGIN" ) + # 公开 API 文档(P-1)。false = /docs、/redoc、/openapi.json 全部 404, + # GraphQL IDE(/graphql 的 GraphiQL 页面)也一并关闭;只保留纯 GraphQL POST。 + # 默认关:生产不暴露 schema 与调试界面,测试环境经 Nacos 显式打开。 + expose_api_docs: bool = Field(False, validation_alias="EXPOSE_API_DOCS") # SSO 配置(通过 Nacos 下发,与 ALIYUN_* 字段同等对待) qq_app_id: Optional[str] = Field(None, validation_alias="QQ_APP_ID") @@ -411,11 +440,9 @@ def dojin_domain_allowlist(self) -> list[str]: "closed", validation_alias="ALIYUN_CAPTCHA_FAIL_MODE" ) - # TEST_LOGIN_BYPASS——测试登录旁路(⚠️ 临时测试设施,公开上线前整体移除, - # grep TEST_LOGIN_BYPASS 全量定位)。两者同时非空才激活;生产 Nacos 永不配置。 - # 见 docs/superpowers/specs/2026-08-14-test-login-bypass-design.md - test_login_bypass_accounts: list[str] = Field(default_factory=list) - test_login_bypass_code: str = Field("") + # 测试登录旁路(白名单固定验证码)已于 P-4 公开上线前整体移除,两个配置字段 + # 随之删除;配置里再出现旧键会被忽略(extra="ignore")。设计稿与历史见 + # docs/superpowers/specs/2026-08-14-test-login-bypass-design.md。 @property def database_url(self) -> str: diff --git a/src/common/crypto.py b/src/common/crypto.py new file mode 100644 index 00000000..b2ac0b08 --- /dev/null +++ b/src/common/crypto.py @@ -0,0 +1,118 @@ +"""Field-level encryption and blind indexing for PII columns. + +One DEK (32 random bytes, base64-encoded) is read from ``PII_DEK_PATH`` and +split via HKDF into two independent keys: + +- ``content_key`` — AES-256-GCM for the ciphertext columns; +- ``index_key`` — HMAC-SHA256 for deterministic blind indices. + +Splitting them means a database dump cannot brute-force the phone-number space +against the content key; the index key alone only enables equality lookup. + +Ciphertext layout: ``version(1B) || nonce(12B) || ciphertext || tag(16B)``. +AAD binds a ciphertext to its row: ``"{table}.{column}:{row_id}"``. + +The DEK is deliberately *not* read from ``Settings``: it lives in a Docker +secret (``/run/secrets/pii_dek``) and must never appear in Nacos, ``.env``, +git or logs. ``fingerprint()`` returns a short one-way tag safe to log so an +operator can confirm the import job and the serving process share the same DEK. +""" + +from __future__ import annotations + +import base64 +import hashlib +import hmac +import ipaddress +import os +from functools import lru_cache +from pathlib import Path + +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from cryptography.hazmat.primitives.kdf.hkdf import HKDF + +VERSION = b"\x01" +DEK_PATH = Path(os.getenv("PII_DEK_PATH", "/run/secrets/pii_dek")) + +_CONTENT_INFO = b"thvote-pii-content" +_INDEX_INFO = b"thvote-pii-index" +_FINGERPRINT_INFO = b"thvote-dek-fingerprint" + + +@lru_cache(maxsize=1) +def _dek() -> bytes: + if not DEK_PATH.exists(): + raise RuntimeError(f"PII DEK missing: {DEK_PATH}") + raw = base64.b64decode(DEK_PATH.read_bytes().strip()) + if len(raw) != 32: + raise RuntimeError(f"PII DEK bad length: {len(raw)}") + return raw + + +@lru_cache(maxsize=1) +def _content_key() -> bytes: + return HKDF( + algorithm=hashes.SHA256(), length=32, salt=None, info=_CONTENT_INFO + ).derive(_dek()) + + +@lru_cache(maxsize=1) +def _index_key() -> bytes: + return HKDF( + algorithm=hashes.SHA256(), length=32, salt=None, info=_INDEX_INFO + ).derive(_dek()) + + +def fingerprint() -> str: + """Short one-way tag of the loaded DEK, safe to log.""" + return hmac.new(_dek(), _FINGERPRINT_INFO, hashlib.sha256).hexdigest()[:8] + + +def aad_for(table: str, column: str, row_id: str) -> bytes: + """AAD binding ciphertext to its row: ``table.column:row_id``.""" + return f"{table}.{column}:{row_id}".encode() + + +def encrypt(plaintext: str | None, aad: bytes) -> bytes | None: + """Encrypt *plaintext* under the content key, bound to *aad*.""" + if not plaintext: + return None + nonce = os.urandom(12) + ciphertext = AESGCM(_content_key()).encrypt(nonce, plaintext.encode(), aad) + return VERSION + nonce + ciphertext + + +def decrypt(blob: bytes | None, aad: bytes) -> str | None: + """Decrypt *blob*; raises on a wrong version or AAD mismatch.""" + if not blob: + return None + if blob[:1] != VERSION: + raise ValueError("unsupported ciphertext version") + return AESGCM(_content_key()).decrypt(blob[1:13], blob[13:], aad).decode() + + +def blind_index(provider: str, subject: str) -> bytes | None: + """Deterministic HMAC index over the canonical ``(provider, subject)``. + + *subject* must already be canonical (``normalize_subject``) — the same + value the DAO receives for lookups — so writes and reads agree. + """ + if not subject: + return None + message = f"{provider}:{subject}".encode() + return hmac.new(_index_key(), message, hashlib.sha256).digest() + + +def ip_prefix(ip: str) -> str | None: + """Return the /24 network of an IPv4 address as a CIDR string. + + Coarse bucket for range queries (anti-vote-farming clustering) that does + not require decrypting the full address. ``None`` for blank/non-IPv4. + """ + if not ip: + return None + try: + return str(ipaddress.ip_network(f"{ip.strip()}/24", strict=False)) + except ValueError: + return None diff --git a/src/common/verification/email_code.py b/src/common/verification/email_code.py index a9bb3787..a49ae074 100644 --- a/src/common/verification/email_code.py +++ b/src/common/verification/email_code.py @@ -26,7 +26,6 @@ from src.common.aliyun.dm_smtp_client import AliyunDmSmtpClient, get_dm_smtp_client from src.common.exceptions import RateLimitError, ValidationError from src.common.redis import get_redis -from src.common.verification.test_bypass import is_test_login_bypass logger = logging.getLogger(__name__) @@ -108,8 +107,6 @@ async def consume(self, email: str, submitted_code: str) -> None: later correct guess fails too; the error code stays the same so the client contract is unchanged. """ - if is_test_login_bypass(email, submitted_code): # TEST_LOGIN_BYPASS 上线前移除 - return redis = await get_redis() expected = await redis.get(_code_key(email)) if expected is None: diff --git a/src/common/verification/sms_code.py b/src/common/verification/sms_code.py index ea77abf9..3d08374a 100644 --- a/src/common/verification/sms_code.py +++ b/src/common/verification/sms_code.py @@ -22,7 +22,6 @@ get_pnvs_client, ) from src.common.exceptions import ValidationError -from src.common.verification.test_bypass import is_test_login_bypass logger = logging.getLogger(__name__) @@ -45,8 +44,6 @@ async def consume(self, phone: str, submitted_code: str) -> None: same passed code returns UNKNOWN — so no extra bookkeeping needed on our side. """ - if is_test_login_bypass(phone, submitted_code): # TEST_LOGIN_BYPASS 上线前移除 - return result = await self.pnvs_client.check_sms_verify_code(phone, submitted_code) if not result.passed: raise ValidationError("INCORRECT_VERIFY_CODE", details=400) diff --git a/src/common/verification/test_bypass.py b/src/common/verification/test_bypass.py deleted file mode 100644 index f3c9c7bb..00000000 --- a/src/common/verification/test_bypass.py +++ /dev/null @@ -1,50 +0,0 @@ -"""TEST_LOGIN_BYPASS——测试登录旁路(白名单固定验证码)。 - -⚠️ 临时测试设施,**公开上线前必须整体移除**(本模块、两个 consume 的短路行、 -config.py 两个字段、测试 Nacos 两个 key)。全量定位:``grep -r TEST_LOGIN_BYPASS``。 -移除条件与安全边界见设计稿: -docs/superpowers/specs/2026-08-14-test-login-bypass-design.md - -激活条件(fail-closed):``TEST_LOGIN_BYPASS_ACCOUNTS``(JSON 数组)与 -``TEST_LOGIN_BYPASS_CODE`` **同时**非空;默认双空 = 功能不存在。生产 -Nacos dataId 永不配置这两个 key。 -""" - -from __future__ import annotations - -import logging -from typing import TYPE_CHECKING - -from src.common.config import get_settings - -if TYPE_CHECKING: # pragma: no cover - from src.common.config import Settings - -logger = logging.getLogger(__name__) - - -def _mask(identifier: str) -> str: - """脱敏:手机号留前3后2,邮箱留首字符+域名。日志禁止出现完整标识。""" - if "@" in identifier: - local, _, domain = identifier.partition("@") - return f"{local[:1]}****@{domain}" - if len(identifier) >= 6: - return f"{identifier[:3]}{'*' * (len(identifier) - 5)}{identifier[-2:]}" - return "*" * len(identifier) - - -def is_test_login_bypass( - identifier: str, - submitted_code: str, - settings: "Settings | None" = None, -) -> bool: - """白名单账号提交固定码 → True(调用方据此跳过真实验证码校验)。""" - s = settings or get_settings() - accounts = s.test_login_bypass_accounts - code = s.test_login_bypass_code - if not accounts or not code: - return False - if identifier not in accounts or submitted_code != code: - return False - logger.warning("TEST LOGIN BYPASS used for %s", _mask(identifier)) - return True diff --git a/src/db_model/user.py b/src/db_model/user.py index 0d4917b0..a487d935 100644 --- a/src/db_model/user.py +++ b/src/db_model/user.py @@ -8,9 +8,11 @@ from __future__ import annotations -from sqlalchemy import Boolean, DateTime, Index, String, func +from sqlalchemy import Boolean, DateTime, Index, LargeBinary, String, func from sqlalchemy.orm import Mapped, mapped_column, relationship +from src.common.crypto import aad_for, decrypt + from .base import Base from .user_identity import UserIdentity @@ -33,9 +35,11 @@ class User(Base): register_date: Mapped[DateTime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() ) - register_ip_address: Mapped[str] = mapped_column( - String(64), nullable=False, default="" - ) + # Plaintext IP stays NULL for new writes (encrypted into register_ip_enc); + # migration 0023 drops this column after import is verified. + register_ip_address: Mapped[str | None] = mapped_column(String(64), nullable=True) + register_ip_enc: Mapped[bytes | None] = mapped_column(LargeBinary, nullable=True) + register_ip_prefix: Mapped[str | None] = mapped_column(String(64), nullable=True) # 注册时的客户端设备指纹(localStorage UUID),反刷票取证用(B-044)。 register_device_id: Mapped[str] = mapped_column( String(128), nullable=False, default="", server_default="" @@ -49,6 +53,16 @@ class User(Base): cascade="all, delete-orphan", ) + @property + def register_ip_plain(self) -> str: + """Decrypted registration IP, falling back to the legacy column.""" + if self.register_ip_enc is not None: + value = decrypt( + self.register_ip_enc, aad_for("user", "register_ip", self.id) + ) + return value or "" + return self.register_ip_address or "" + def identity(self, provider: str) -> UserIdentity | None: """Return this account's identity for *provider*, if any.""" for identity in self.identities: diff --git a/src/db_model/user_identity.py b/src/db_model/user_identity.py index 14f96825..1b6a51f0 100644 --- a/src/db_model/user_identity.py +++ b/src/db_model/user_identity.py @@ -15,13 +15,17 @@ CheckConstraint, DateTime, ForeignKey, + Index, Integer, + LargeBinary, String, UniqueConstraint, func, ) from sqlalchemy.orm import Mapped, mapped_column, relationship +from src.common.crypto import aad_for, decrypt + from .base import Base if TYPE_CHECKING: @@ -43,7 +47,11 @@ class UserIdentity(Base): index=True, ) provider: Mapped[str] = mapped_column(String(16), nullable=False) - subject: Mapped[str] = mapped_column(String(255), nullable=False) + # Plaintext subject stays NULL for new writes (encrypted into subject_enc + # / subject_idx); migration 0023 drops this column after import is verified. + subject: Mapped[str | None] = mapped_column(String(255), nullable=True) + subject_enc: Mapped[bytes | None] = mapped_column(LargeBinary, nullable=True) + subject_idx: Mapped[bytes | None] = mapped_column(LargeBinary, nullable=True) verified: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) verified_at: Mapped[DateTime | None] = mapped_column( @@ -55,9 +63,8 @@ class UserIdentity(Base): created_at: Mapped[DateTime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() ) - created_ip: Mapped[str] = mapped_column( - String(64), nullable=False, default="", server_default="" - ) + created_ip: Mapped[str | None] = mapped_column(String(64), nullable=True) + created_ip_enc: Mapped[bytes | None] = mapped_column(LargeBinary, nullable=True) created_device_id: Mapped[str] = mapped_column( String(128), nullable=False, default="", server_default="" ) @@ -67,11 +74,36 @@ class UserIdentity(Base): user: Mapped["User"] = relationship(back_populates="identities") + @property + def subject_plain(self) -> str | None: + """Decrypted subject, falling back to the legacy plaintext column.""" + if self.subject_enc is not None: + return decrypt( + self.subject_enc, + aad_for("user_identity", "subject", self.user_id), + ) + return self.subject + + @property + def created_ip_plain(self) -> str: + """Decrypted bind IP, falling back to the legacy plaintext column.""" + if self.created_ip_enc is not None: + value = decrypt( + self.created_ip_enc, + aad_for("user_identity", "created_ip", self.user_id), + ) + return value or "" + return self.created_ip or "" + __table_args__ = ( - UniqueConstraint( - "provider", "subject", name="uq_user_identity_provider_subject" - ), UniqueConstraint("user_id", "provider", name="uq_user_identity_user_provider"), + Index( + "uq_user_identity_provider_idx", + "provider", + "subject_idx", + unique=True, + ), + Index("ix_user_identity_subject_idx", "subject_idx"), CheckConstraint( "provider IN (" + ", ".join(f"'{v}'" for v in PROVIDER_CHECK_VALUES) + ")", name="ck_user_identity_provider", diff --git a/src/main.py b/src/main.py index 5033394c..ebce24dd 100644 --- a/src/main.py +++ b/src/main.py @@ -35,6 +35,7 @@ nacos_config_change_callback, reload_settings, ) +from .common.crypto import fingerprint as pii_dek_fingerprint from .common.database import get_db_session, get_session_maker, reload_engine from .common.middleware.logging import LoggingMiddleware from .apps.user.service import get_audit_log_failures @@ -45,6 +46,10 @@ @asynccontextmanager async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: """Application lifespan handler.""" + # Fail fast if the PII DEK is missing — encrypted columns cannot be read + # without it, so a missing DEK must never degrade into a running service. + logger.info("PII DEK loaded: fp=%s", pii_dek_fingerprint()) + # Verify DB connectivity — fail fast rather than silently running with no schema try: async with get_session_maker()() as session: @@ -109,6 +114,10 @@ def create_app() -> FastAPI: title="THVote FastAPI Backend", version="0.2.1", lifespan=lifespan, + # P-1: 生产不暴露 OpenAPI schema / Swagger / ReDoc(EXPOSE_API_DOCS=false)。 + docs_url="/docs" if settings.expose_api_docs else None, + redoc_url="/redoc" if settings.expose_api_docs else None, + openapi_url="/openapi.json" if settings.expose_api_docs else None, ) # Logging middleware @@ -299,7 +308,11 @@ async def discover_self(healthy_only: bool = False) -> dict: # GraphQL endpoint (Strawberry). AppGraphQLRouter guarantees every error # carries an `extensions` block (see api/graphql/http.py). - graphql_app = AppGraphQLRouter(graphql_schema) + # P-1: EXPOSE_API_DOCS=false 时同时关掉 GraphiQL IDE,只保留 POST /graphql。 + graphql_app = AppGraphQLRouter( + graphql_schema, + graphql_ide="graphiql" if settings.expose_api_docs else None, + ) app.include_router(graphql_app, prefix="/graphql") # Admin UI static files (optional — only if directory exists). diff --git a/tests/conftest.py b/tests/conftest.py index 0c577ac1..f6ba8d2a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,7 +10,10 @@ from __future__ import annotations +import base64 import os +import tempfile +from pathlib import Path # Ensure required env vars exist for Settings() before importing app modules. os.environ.setdefault("DATABASE_URL", "sqlite+aiosqlite:///./.test.db") @@ -18,3 +21,11 @@ os.environ.setdefault("JWT_SECRET_KEY", "test-secret-key-do-not-use-in-prod") os.environ.setdefault("VOTE_START_ISO", "2020-01-01T00:00:00+00:00") os.environ.setdefault("VOTE_END_ISO", "2099-12-31T23:59:59+00:00") + +# PII 字段级加密需要一个 DEK。测试进程用一个一次性随机密钥写入临时文件, +# 让 subject_enc / register_ip_enc / created_ip_enc 的读写路径都能跑通。 +# 必须在这里(导入任何 src 模块之前)设置,否则 crypto.DEK_PATH 读不到。 +_test_dek = base64.b64encode(os.urandom(32)).decode() +_test_dek_path = Path(tempfile.gettempdir()) / f"thvote-test-dek-{os.getpid()}" +_test_dek_path.write_text(_test_dek, encoding="utf-8") +os.environ["PII_DEK_PATH"] = str(_test_dek_path) diff --git a/tests/helpers/users.py b/tests/helpers/users.py index fa22f8d7..7be97a9a 100644 --- a/tests/helpers/users.py +++ b/tests/helpers/users.py @@ -12,6 +12,7 @@ from src.apps.user.schemas import generate_user_id from src.apps.user.utils.security import AuthProvider +from src.common.crypto import aad_for, blind_index, encrypt, ip_prefix from src.db_model.user import User from src.db_model.user_identity import UserIdentity @@ -31,13 +32,17 @@ def build_user( ) -> User: """Return an unsaved User with one identity per non-None identifier.""" now = datetime.now(UTC) + uid = user_id or generate_user_id() user = User( - id=user_id or generate_user_id(), + id=uid, nickname=nickname, password_hash=AuthProvider.hash_password(password) if password else None, removed=removed, register_date=now, - register_ip_address=register_ip, + register_ip_enc=encrypt( + register_ip or None, aad_for("user", "register_ip", uid) + ), + register_ip_prefix=ip_prefix(register_ip), ) for provider, subject in ( ("email", email), @@ -49,12 +54,21 @@ def build_user( continue user.identities.append( UserIdentity( + # 显式给 FK:未持久化的 build_user 对象上,append 不会立刻同步 + # user_id,而 subject_plain/created_ip_plain 的解密 AAD 依赖它。 + user_id=uid, provider=provider, - subject=subject, + subject_enc=encrypt( + subject, aad_for("user_identity", "subject", uid) + ), + subject_idx=blind_index(provider, subject), verified=verified, verified_at=now if verified else None, created_at=now, - created_ip=register_ip, + created_ip_enc=encrypt( + register_ip or None, + aad_for("user_identity", "created_ip", uid), + ), ) ) return user diff --git a/tests/integration/test_identity_binding.py b/tests/integration/test_identity_binding.py index 74cbc231..0253c234 100644 --- a/tests/integration/test_identity_binding.py +++ b/tests/integration/test_identity_binding.py @@ -48,12 +48,12 @@ async def test_first_login_creates_account_and_identity_with_same_meta( await _login_email(user_service, patch_redis, "new@example.com", nickname="n") user = await user_service.identities.resolve(IdentityProvider.EMAIL, "new@example.com") - assert user.register_ip_address == "203.0.113.7" + assert user.register_ip_plain == "203.0.113.7" assert user.register_device_id == "device-A" [identity] = user.identities - assert (identity.provider, identity.subject) == ("email", "new@example.com") + assert (identity.provider, identity.subject_plain) == ("email", "new@example.com") assert identity.verified is True and identity.verified_at is not None - assert identity.created_ip == "203.0.113.7" + assert identity.created_ip_plain == "203.0.113.7" assert identity.created_device_id == "device-A" assert identity.last_login_at is not None @@ -83,7 +83,7 @@ async def test_email_subject_is_case_insensitive(user_service, patch_redis): user = await user_service.identities.resolve(IdentityProvider.EMAIL, "Mixed@EXAMPLE.COM") assert user is not None - assert user.identity("email").subject == "mixed@example.com" + assert user.identity("email").subject_plain == "mixed@example.com" @pytest.mark.asyncio @@ -99,8 +99,8 @@ async def test_rebind_phone_replaces_row_with_new_meta(user_service, patch_redis ) rows = (await session.execute(select(UserIdentity))).scalars().all() - assert [(r.provider, r.subject) for r in rows] == [("phone", "13800000002")] - assert rows[0].created_ip == "198.51.100.9" + assert [(r.provider, r.subject_plain) for r in rows] == [("phone", "13800000002")] + assert rows[0].created_ip_plain == "198.51.100.9" assert rows[0].created_device_id == "device-B" assert await user_service.identities.resolve(IdentityProvider.PHONE, "13800000001") is None diff --git a/tests/integration/test_identity_login_regressions.py b/tests/integration/test_identity_login_regressions.py index 9a134cee..0e318074 100644 --- a/tests/integration/test_identity_login_regressions.py +++ b/tests/integration/test_identity_login_regressions.py @@ -118,7 +118,7 @@ async def test_sso_merge_does_not_replace_existing_binding( .where(UserIdentity.user_id == user.id, UserIdentity.provider == "qq") .execution_options(populate_existing=True) )).scalar_one() - assert identity.subject == "openid-original", "已有 qq 绑定被登录路径静默顶掉了" + assert identity.subject_plain == "openid-original", "已有 qq 绑定被登录路径静默顶掉了" @pytest.mark.asyncio @@ -142,7 +142,7 @@ async def test_sso_merge_still_binds_when_slot_is_free( .where(UserIdentity.user_id == user.id, UserIdentity.provider == "qq") .execution_options(populate_existing=True) )).scalar_one() - assert identity.subject == "openid-new" + assert identity.subject_plain == "openid-new" @pytest.mark.asyncio diff --git a/tests/integration/test_identity_pg_constraints.py b/tests/integration/test_identity_pg_constraints.py index 935e8b2f..1adff205 100644 --- a/tests/integration/test_identity_pg_constraints.py +++ b/tests/integration/test_identity_pg_constraints.py @@ -41,7 +41,7 @@ async def test_same_subject_twice_rejected(pg_session): pg_session.add(build_user(email=subject)) with pytest.raises(IntegrityError) as exc_info: await pg_session.commit() - assert "uq_user_identity_provider_subject" in str(exc_info.value) + assert "uq_user_identity_provider_idx" in str(exc_info.value) @pytest.mark.asyncio diff --git a/tests/integration/test_legacy_login_compat.py b/tests/integration/test_legacy_login_compat.py index 7075119e..4224a270 100644 --- a/tests/integration/test_legacy_login_compat.py +++ b/tests/integration/test_legacy_login_compat.py @@ -21,6 +21,7 @@ UpdatePasswordRequest, ) from src.common.exceptions import ValidationError +from src.common.crypto import aad_for, blind_index, encrypt, ip_prefix from src.common.security.password import verify_password from src.db_model.user import User from src.db_model.user_identity import UserIdentity @@ -55,24 +56,32 @@ async def _make_user( salt: str | None = None, ) -> User: now = datetime.now(UTC) + user_id = f"legacy-{email.split('@')[0]}" + ip = "198.51.100.9" user = User( - id=f"legacy-{email.split('@')[0]}", + id=user_id, nickname="legacy", password_hash=password_hash, legacy_salt=salt, removed=False, register_date=now, - register_ip_address="198.51.100.9", + register_ip_enc=encrypt(ip, aad_for("user", "register_ip", user_id)), + register_ip_prefix=ip_prefix(ip), register_device_id="", ) user.identities.append( UserIdentity( provider="email", - subject=email, + subject_enc=encrypt( + email, aad_for("user_identity", "subject", user_id) + ), + subject_idx=blind_index("email", email), verified=True, verified_at=now, created_at=now, - created_ip="198.51.100.9", + created_ip_enc=encrypt( + ip, aad_for("user_identity", "created_ip", user_id) + ), created_device_id="", ) ) diff --git a/tests/integration/test_sso_flows.py b/tests/integration/test_sso_flows.py index ae60c1f9..2a7f45cc 100644 --- a/tests/integration/test_sso_flows.py +++ b/tests/integration/test_sso_flows.py @@ -41,13 +41,13 @@ async def test_bind_sso_creates_qq_identity(session, session_maker, patch_redis) row = ( await session.execute( select(UserIdentity).where( - UserIdentity.provider == "qq", UserIdentity.subject == "test-openid-123" + UserIdentity.provider == "qq", UserIdentity.user_id == user.id ) ) ).scalar_one() assert row.user_id == user.id assert row.verified is True - assert row.created_ip == "9.9.9.9" + assert row.created_ip_plain == "9.9.9.9" assert row.created_device_id == "dev-1" @@ -114,7 +114,7 @@ async def test_login_merges_pending_sso_session(user_service, patch_redis): from src.apps.user.identity import IdentityProvider user = await user_service.identities.resolve(IdentityProvider.EMAIL, "m@example.com") - assert user.identity("qq").subject == "from-oauth" + assert user.identity("qq").subject_plain == "from-oauth" assert await patch_redis.get(f"sso-session:{sid}") is None, "sid must be one-shot" diff --git a/tests/unit/test_config_ip_lists.py b/tests/unit/test_config_ip_lists.py new file mode 100644 index 00000000..571416d7 --- /dev/null +++ b/tests/unit/test_config_ip_lists.py @@ -0,0 +1,45 @@ +"""``TRUSTED_PROXY_IPS`` / ``ADMIN_ALLOWED_IPS`` 的宽容解析。 + +Nacos 把配置原样写进环境变量,值都是字符串。pydantic-settings 对 ``list[str]`` +默认在 source 层按 JSON 预解析,于是把这两个键写成 ``""``(文档里的"留空"写法) +会让 ``Settings()`` 在 import 期直接抛 SettingsError、进程起不来——2026-10-29 +生产首次部署就是卡在这里。这两个键现在接受 ``""``、JSON 数组、逗号分隔三种写法。 +""" + +from __future__ import annotations + +import pytest + +from src.common.config import Settings + + +@pytest.mark.parametrize( + "raw, expected", + [ + ("", []), + ("[]", []), + ('["10.0.0.0/8"]', ["10.0.0.0/8"]), + ("10.0.0.0/8, 192.168.0.0/16", ["10.0.0.0/8", "192.168.0.0/16"]), + ("1.2.3.4", ["1.2.3.4"]), + ], +) +def test_ip_lists_accept_nacos_strings( + monkeypatch: pytest.MonkeyPatch, raw: str, expected: list[str] +) -> None: + monkeypatch.setenv("TRUSTED_PROXY_IPS", raw) + monkeypatch.setenv("ADMIN_ALLOWED_IPS", raw) + + settings = Settings() + + assert settings.trusted_proxy_ips == expected + assert settings.admin_allowed_ips == expected + + +def test_ip_lists_default_to_empty(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("TRUSTED_PROXY_IPS", raising=False) + monkeypatch.delenv("ADMIN_ALLOWED_IPS", raising=False) + + settings = Settings() + + assert settings.trusted_proxy_ips == [] + assert settings.admin_allowed_ips == [] diff --git a/tests/unit/test_crypto.py b/tests/unit/test_crypto.py new file mode 100644 index 00000000..5be64858 --- /dev/null +++ b/tests/unit/test_crypto.py @@ -0,0 +1,93 @@ +"""Unit tests for ``src.common.crypto``. + +Cover the PII encryption primitives: roundtrip, AAD binding, blind-index +determinism/provider binding, DEK presence/length fail-fast, fingerprint and +the /24 prefix helper. +""" + +from __future__ import annotations + +import base64 +import os + +import pytest +from cryptography.exceptions import InvalidTag + +from src.common import crypto + + +@pytest.fixture +def fresh_dek(tmp_path, monkeypatch): + """Point the module at a throwaway DEK and reset its key caches.""" + dek = base64.b64encode(os.urandom(32)).decode() + dek_file = tmp_path / "pii_dek" + dek_file.write_text(dek, encoding="utf-8") + monkeypatch.setattr(crypto, "DEK_PATH", dek_file) + crypto._dek.cache_clear() + crypto._content_key.cache_clear() + crypto._index_key.cache_clear() + return dek_file + + +def test_encrypt_roundtrip(fresh_dek): + aad = crypto.aad_for("user_identity", "subject", "u1") + blob = crypto.encrypt("alice@example.com", aad) + assert isinstance(blob, bytes) + assert blob[:1] == crypto.VERSION + assert crypto.decrypt(blob, aad) == "alice@example.com" + + +def test_encrypt_empty_returns_none(fresh_dek): + assert crypto.encrypt("", b"aad") is None + assert crypto.encrypt(None, b"aad") is None + assert crypto.decrypt(None, b"aad") is None + + +def test_aad_mismatch_fails(fresh_dek): + blob = crypto.encrypt("alice@example.com", crypto.aad_for("t", "c", "u1")) + with pytest.raises(InvalidTag): + crypto.decrypt(blob, crypto.aad_for("t", "c", "u2")) + + +def test_blind_index_deterministic(fresh_dek): + a = crypto.blind_index("email", "alice@example.com") + b = crypto.blind_index("email", "alice@example.com") + assert a == b + assert isinstance(a, bytes) and len(a) == 32 + + +def test_blind_index_provider_and_subject_bound(fresh_dek): + base = crypto.blind_index("email", "alice@example.com") + assert base != crypto.blind_index("phone", "alice@example.com") + assert base != crypto.blind_index("email", "bob@example.com") + + +def test_missing_dek_raises(monkeypatch): + crypto._dek.cache_clear() + crypto._content_key.cache_clear() + crypto._index_key.cache_clear() + monkeypatch.setattr(crypto, "DEK_PATH", crypto.DEK_PATH / "does-not-exist") + with pytest.raises(RuntimeError, match="missing"): + crypto._dek() + + +def test_bad_length_dek_raises(tmp_path, monkeypatch): + bad = tmp_path / "bad_dek" + bad.write_text(base64.b64encode(b"short").decode(), encoding="utf-8") + crypto._dek.cache_clear() + crypto._content_key.cache_clear() + crypto._index_key.cache_clear() + monkeypatch.setattr(crypto, "DEK_PATH", bad) + with pytest.raises(RuntimeError, match="length"): + crypto._dek() + + +def test_fingerprint_stable(fresh_dek): + assert crypto.fingerprint() == crypto.fingerprint() + assert len(crypto.fingerprint()) == 8 + + +def test_ip_prefix(): + assert crypto.ip_prefix("203.0.113.7") == "203.0.113.0/24" + assert crypto.ip_prefix("") is None + assert crypto.ip_prefix("not-an-ip") is None diff --git a/tests/unit/test_expose_api_docs.py b/tests/unit/test_expose_api_docs.py new file mode 100644 index 00000000..987ef9c7 --- /dev/null +++ b/tests/unit/test_expose_api_docs.py @@ -0,0 +1,59 @@ +"""P-1: ``EXPOSE_API_DOCS`` 开关控制 API 文档与 GraphQL IDE 是否暴露。 + +生产默认 ``false``:/docs、/redoc、/openapi.json 全部 404,GraphQL 只保留 +POST(纯 API),不再返回 GraphiQL 页面。测试环境可在 Nacos 显式打开。 +""" + +from __future__ import annotations + +import pytest +from fastapi.testclient import TestClient + +from src.common import config as config_module +from src.main import create_app + + +@pytest.fixture(autouse=True) +def _restore_settings(): + """每个用例结束后清掉缓存,避免把开关状态串到别的测试模块。""" + yield + config_module.reload_settings() + + +def _client(monkeypatch: pytest.MonkeyPatch, expose: str) -> TestClient: + monkeypatch.setenv("EXPOSE_API_DOCS", expose) + monkeypatch.setenv("NACOS_ENABLED", "false") + config_module.reload_settings() + return TestClient(create_app()) + + +@pytest.mark.parametrize("expose, expected", [("false", 404), ("true", 200)]) +@pytest.mark.parametrize("path", ["/docs", "/redoc", "/openapi.json"]) +def test_api_docs_endpoints_follow_switch( + monkeypatch: pytest.MonkeyPatch, expose: str, expected: int, path: str +) -> None: + client = _client(monkeypatch, expose) + assert client.get(path).status_code == expected + + +def test_graphql_ide_disabled_but_post_still_works( + monkeypatch: pytest.MonkeyPatch, +) -> None: + client = _client(monkeypatch, "false") + + # 浏览器直接打开 /graphql 不再给出 IDE 页面 + assert ( + client.get("/graphql", headers={"Accept": "text/html"}).status_code == 404 + ) + + # 真正的 GraphQL 调用不受影响 + resp = client.post("/graphql", json={"query": "{ __typename }"}) + assert resp.status_code == 200 + assert resp.json()["data"]["__typename"] == "Query" + + +def test_graphql_ide_enabled(monkeypatch: pytest.MonkeyPatch) -> None: + client = _client(monkeypatch, "true") + resp = client.get("/graphql", headers={"Accept": "text/html"}) + assert resp.status_code == 200 + assert "graphiql" in resp.text.lower() diff --git a/tests/unit/test_test_login_bypass.py b/tests/unit/test_test_login_bypass.py deleted file mode 100644 index 05081ff4..00000000 --- a/tests/unit/test_test_login_bypass.py +++ /dev/null @@ -1,95 +0,0 @@ -"""测试登录旁路(白名单固定验证码)——临时测试设施,上线前随功能一并移除。 - -设计稿: docs/superpowers/specs/2026-08-14-test-login-bypass-design.md -定位标记: TEST_LOGIN_BYPASS -""" - -from types import SimpleNamespace -from unittest.mock import AsyncMock - -import pytest - -from src.common.exceptions import ValidationError -from src.common.verification import test_bypass as tb -from src.common.verification.sms_code import SmsCodeService - - -def _settings(accounts=None, code=""): - return SimpleNamespace( - test_login_bypass_accounts=accounts or [], - test_login_bypass_code=code, - ) - - -ARMED = _settings(accounts=["19900000001", "test1@example.com"], code="888888") - - -# ---------- helper 本体 ---------- - -def test_hit_when_armed_and_account_and_code_match(): - assert tb.is_test_login_bypass("19900000001", "888888", settings=ARMED) - assert tb.is_test_login_bypass("test1@example.com", "888888", settings=ARMED) - - -def test_miss_on_wrong_code_or_unlisted_account(): - assert not tb.is_test_login_bypass("19900000001", "000000", settings=ARMED) - assert not tb.is_test_login_bypass("13800138000", "888888", settings=ARMED) - - -def test_disarmed_by_default_empty_config(): - assert not tb.is_test_login_bypass( - "19900000001", "888888", settings=_settings() - ) - # 只配了账号没配码 / 只配码没配账号 —— 均不激活 - assert not tb.is_test_login_bypass( - "19900000001", "", settings=_settings(accounts=["19900000001"]) - ) - assert not tb.is_test_login_bypass( - "19900000001", "888888", settings=_settings(code="888888") - ) - - -# ---------- consume 短路 ---------- - -@pytest.mark.asyncio -async def test_sms_consume_short_circuits_without_pnvs(monkeypatch): - monkeypatch.setattr(tb, "get_settings", lambda: ARMED) - pnvs = SimpleNamespace( - check_sms_verify_code=AsyncMock( - side_effect=AssertionError("must not call PNVS") - ) - ) - svc = SmsCodeService(pnvs_client=pnvs) - await svc.consume("19900000001", "888888") # 放行且不打阿里云 - pnvs.check_sms_verify_code.assert_not_called() - - -@pytest.mark.asyncio -async def test_sms_consume_wrong_code_still_goes_to_pnvs(monkeypatch): - monkeypatch.setattr(tb, "get_settings", lambda: ARMED) - outcome = SimpleNamespace(passed=False) - pnvs = SimpleNamespace(check_sms_verify_code=AsyncMock(return_value=outcome)) - svc = SmsCodeService(pnvs_client=pnvs) - with pytest.raises(ValidationError): - await svc.consume("19900000001", "000000") - pnvs.check_sms_verify_code.assert_called_once() - - -@pytest.mark.asyncio -async def test_email_consume_short_circuits_without_redis(monkeypatch): - from src.common.verification.email_code import EmailCodeService - import src.common.verification.email_code as email_mod - - monkeypatch.setattr(tb, "get_settings", lambda: ARMED) - monkeypatch.setattr( - email_mod, "get_redis", - AsyncMock(side_effect=AssertionError("must not touch redis")), - ) - svc = EmailCodeService(smtp_client=SimpleNamespace()) - await svc.consume("test1@example.com", "888888") - - -def test_masking_never_logs_full_identifier(): - assert tb._mask("19900000001") == "199******01" - assert tb._mask("test1@example.com") == "t****@example.com" - assert "19900000001" not in tb._mask("19900000001")