diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9ef6e2ab..5b245580 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -590,8 +590,7 @@ jobs: core-rust, build, e2e, - lighthouse, - vrt, + browser-quality, ] if: always() steps: @@ -617,8 +616,7 @@ jobs: fi [ "${{ needs.build.result }}" = "success" ] || FAIL=1 [ "${{ needs.e2e.result }}" = "success" ] || FAIL=1 - [ "${{ needs.lighthouse.result }}" = "success" ] || FAIL=1 - [ "${{ needs.vrt.result }}" = "success" ] || FAIL=1 + [ "${{ needs.browser-quality.result }}" = "success" ] || FAIL=1 if [ "$FAIL" = "1" ]; then echo "One or more required jobs did not succeed:" echo " workflow-policy: ${{ needs.workflow-policy.result }}" @@ -631,8 +629,7 @@ jobs: echo " core-rust: ${{ needs.core-rust.result }} (skipped = OK, crates/ untouched)" echo " build: ${{ needs.build.result }}" echo " e2e: ${{ needs.e2e.result }}" - echo " lighthouse: ${{ needs.lighthouse.result }}" - echo " vrt: ${{ needs.vrt.result }}" + echo " browser-quality: ${{ needs.browser-quality.result }}" exit 1 fi echo "All required jobs succeeded (or were legitimately skipped)." @@ -820,13 +817,14 @@ jobs: fail_ci_if_error: false # ---------------------------------------------------------- - # 6. LIGHTHOUSE: Performance budgets (after build) + # 6. BROWSER QUALITY: VRT + Lighthouse (after build) # ---------------------------------------------------------- - lighthouse: - name: 🔦 Lighthouse CI + # QNBS-v3: consolidated from separate vrt/lighthouse jobs (Wave 2 CI-performance) — both already ran off the critical path; sharing checkout/setup/artifact-download once saves runner-minutes, measured to fit inside this baseline's critical-path slack, not a universal zero-wall-clock guarantee. + browser-quality: + name: 🔦🖼 Browser Quality (VRT + Lighthouse) runs-on: ubuntu-latest - # QNBS-v3: 25 min — two lhci runs (mobile 3× + desktop 2×); was 15 min for mobile only - timeout-minutes: 25 + # QNBS-v3: union of the former 15 min (vrt) + 25 min (lighthouse) timeouts, now sequential in one job. + timeout-minutes: 35 needs: [build] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -838,18 +836,83 @@ jobs: uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: dist - path: ./dist + path: dist/ + + # ---- VRT phase ---- + # QNBS-v3: /var/cache/apt/archives is root-owned by default — actions/cache's restore + # (tar extraction) runs as the unprivileged runner user and fails with "Permission denied" + # on every file without this, silently degrading to a cache miss every time (real CI + # evidence: "Cache hit" followed immediately by "Cache not found" from the failed tar). + - name: Make apt archive cache dir writable by the runner user + run: sudo mkdir -p /var/cache/apt/archives/partial && sudo chown -R runner:runner /var/cache/apt/archives && sudo chmod -R 755 /var/cache/apt/archives + + - name: Cache apt packages (Playwright system deps) + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: /var/cache/apt/archives/*.deb + key: apt-playwright-chromium-deps-v1 + + - name: Install Playwright browsers + run: pnpm exec playwright install --with-deps chromium + + - name: Serve dist + run VRT + env: + CI: true + run: | + # QNBS-v3: The GitHub Pages build has base=/WorldScript-Studio/. Mirror that structure + # so http-server serves assets at the expected paths matching playwright.config baseURL. + mkdir -p serve_root/WorldScript-Studio + cp -r dist/. serve_root/WorldScript-Studio/ + pnpm exec http-server serve_root -p 3000 -s & + pnpm exec wait-on http://127.0.0.1:3000 --timeout 30000 + pnpm run test:vrt + + - name: Upload VRT screenshots + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: vrt-screenshots + # QNBS-v3: upload both baselines and diff outputs so PR reviewers can compare visually. + path: | + tests/e2e/visual-regression.spec.ts-snapshots/ + tests/e2e/test-results/ + retention-days: 7 + + # QNBS-v3: same playwright.config.ts reporter as the required e2e job, produced independently on this runner — previously never uploaded. + - name: Upload VRT JUnit results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: vrt-junit + path: tests/e2e/results/junit.xml + if-no-files-found: warn + retention-days: 7 + + - name: Publish VRT test results to Codecov + if: ${{ !cancelled() }} + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 + with: + token: ${{ secrets.CODECOV_TOKEN }} + report_type: test_results + files: tests/e2e/results/junit.xml + disable_search: true + flags: vrt + name: vrt + fail_ci_if_error: false + # ---- Lighthouse phase ---- + # QNBS-v3: !cancelled(), not the step default success() — VRT failing must not skip Lighthouse; each phase is an independent diagnostic gate that must produce its own evidence, matching what two separate jobs would have done. # QNBS-v3: no --assert.exitCode flag — let .lighthouserc.cjs assertions drive the # exit code; error-level assertions (CLS, accessibility) will correctly fail this job. - name: Run Lighthouse CI (mobile) + if: ${{ !cancelled() }} run: pnpm exec lhci autorun env: LHCI_GITHUB_APP_TOKEN: ${{ secrets.LHCI_GITHUB_APP_TOKEN }} - # QNBS-v3: desktop run is continue-on-error until baselines stabilise; accessibility - # and CLS are still error gates so critical issues surface immediately. + # QNBS-v3: !cancelled() so desktop still runs even if VRT or mobile Lighthouse failed; pre-existing continue-on-error (unchanged by this consolidation) means its accessibility/CLS error assertions can still fail this step internally without failing the job — promotion to blocking stays gated on the 5-consecutive-main-run exit criterion in docs/CI.md. - name: Run Lighthouse CI (desktop) + if: ${{ !cancelled() }} run: pnpm exec lhci autorun --config=.lighthouserc.desktop.cjs continue-on-error: true env: @@ -958,85 +1021,3 @@ jobs: flags: storybook name: storybook fail_ci_if_error: false - - # ---------------------------------------------------------- - # 8. VRT: Visual regression (Playwright screenshots, chromium only) - # ---------------------------------------------------------- - vrt: - name: 🖼 Visual Regression - runs-on: ubuntu-latest - timeout-minutes: 15 - needs: [build] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - uses: ./.github/actions/setup - - - name: Download dist artifact - # QNBS-v3: SHA-pinned to match all other download-artifact usages in this workflow - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: dist - path: dist/ - - # QNBS-v3: /var/cache/apt/archives is root-owned by default — actions/cache's restore - # (tar extraction) runs as the unprivileged runner user and fails with "Permission denied" - # on every file without this, silently degrading to a cache miss every time (real CI - # evidence: "Cache hit" followed immediately by "Cache not found" from the failed tar). - - name: Make apt archive cache dir writable by the runner user - run: sudo mkdir -p /var/cache/apt/archives/partial && sudo chown -R runner:runner /var/cache/apt/archives && sudo chmod -R 755 /var/cache/apt/archives - - - name: Cache apt packages (Playwright system deps) - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: /var/cache/apt/archives/*.deb - key: apt-playwright-chromium-deps-v1 - - - name: Install Playwright browsers - run: pnpm exec playwright install --with-deps chromium - - - name: Serve dist + run VRT - env: - CI: true - run: | - # QNBS-v3: The GitHub Pages build has base=/WorldScript-Studio/. Mirror that structure - # so http-server serves assets at the expected paths matching playwright.config baseURL. - mkdir -p serve_root/WorldScript-Studio - cp -r dist/. serve_root/WorldScript-Studio/ - pnpm exec http-server serve_root -p 3000 -s & - pnpm exec wait-on http://127.0.0.1:3000 --timeout 30000 - pnpm run test:vrt - - - name: Upload VRT screenshots - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: vrt-screenshots - # QNBS-v3: upload both baselines and diff outputs so PR reviewers can compare visually. - path: | - tests/e2e/visual-regression.spec.ts-snapshots/ - tests/e2e/test-results/ - retention-days: 7 - - # QNBS-v3: same playwright.config.ts reporter as the required e2e job, produced independently on this runner — previously never uploaded. - - name: Upload VRT JUnit results - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: vrt-junit - path: tests/e2e/results/junit.xml - if-no-files-found: warn - retention-days: 7 - - - name: Publish VRT test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 - with: - token: ${{ secrets.CODECOV_TOKEN }} - report_type: test_results - files: tests/e2e/results/junit.xml - disable_search: true - flags: vrt - name: vrt - fail_ci_if_error: false diff --git a/docs/CI.md b/docs/CI.md index b0538583..97902e53 100644 --- a/docs/CI.md +++ b/docs/CI.md @@ -38,16 +38,19 @@ CI runs for the affected test path before removing a temporary quarantine. ### Gate authority `✅ CI Success` is the required branch-protection status and aggregates `workflow-policy`, `pr-size`, -`security`, `signatures`, `quality`, `changes`, `rust-tauri`, `core-rust`, `build`, `e2e`, `lighthouse`, and `vrt` (`pr-size` -only runs on `pull_request` events — legitimately skipped otherwise). `e2e-deep` and -`storybook` are explicitly advisory at job level while their stability criteria are measured. The -`deploy` job depends only on that aggregate and remains main-push-only. +`security`, `signatures`, `quality`, `changes`, `rust-tauri`, `core-rust`, `build`, `e2e`, and +`browser-quality` (`pr-size` only runs on `pull_request` events — legitimately skipped otherwise). +`e2e-deep` and `storybook` are explicitly advisory at job level while their stability criteria are +measured. The `deploy` job depends only on that aggregate and remains main-push-only. `storybook` and `e2e-deep` are separately executed advisory jobs with explicit job-level `continue-on-error: true`; their failures remain visible and must still be investigated before merge -under the repository's full-suite policy. Lighthouse is part of the required aggregate because its -mobile accessibility and CLS assertions are blocking; only its desktop performance step remains -non-blocking under the exit criteria documented below. The coverage ratchet remains informational. +under the repository's full-suite policy. `browser-quality` (VRT + Lighthouse, consolidated Wave 2 +CI-performance work) is part of the required aggregate because VRT and Lighthouse's mobile +accessibility/CLS assertions are blocking; only the Lighthouse desktop step remains non-blocking +under the exit criteria documented below — each phase runs independently of the others' outcome +(`if: ${{ !cancelled() }}`), so one phase failing never skips a later phase's own evidence. The +coverage ratchet remains informational. **Post-merge doc update workflow:** 1. Push the commit → CI starts automatically. @@ -139,12 +142,11 @@ registry gzip-decoding failure mode, while OSV failures remain blocking. | `quality` | `security` | Matrix **Node 22** and **24** → Biome lint, **`pnpm run i18n:check`**, **`pnpm run docs:check`**, **`pnpm run csp:verify`**, **`pnpm run parity:check`**, `pnpm run typecheck`, Vitest + coverage (+ non-blocking coverage-ratchet suggestion), Codecov coverage (Node 22 only, optional token), Codecov Test Analytics per Node version (`unit-node22`/`unit-node24` flags), coverage + JUnit artifacts | | `rust-tauri` | `security` | Rust `cargo fmt --check`, `cargo check --locked`, `cargo clippy --locked --all-targets -- -D warnings`, and `cargo test --locked`; compile/lint signal for Tauri changes without building installers on every PR | | `build` | `quality` | Production `pnpm run build`, **`bundle:budget`**, **`analyze`** (upload `bundle-analysis.html`; Codecov Bundle Analysis runs inside this same build — see below), **`pnpm run smoke:prod`** (headless-Chromium prod-build + CSP-runtime gate — see below), `dist` artifact; on `main` (non-PR): Pages artifact + **SLSA build provenance attestation**. No `if:` on the job itself — `smoke:prod` runs on every PR, not just `main` pushes. | -| `e2e` | `quality` | Playwright **Chromium** + **Mobile Chrome** (Pixel 5) — `CI=true`, 2× retries, 50 min timeout; browser cache via `actions/cache@v5`. Firefox optional locally. `PLAYWRIGHT_SKIP_VRT=true` (VRT is its own job). | -| `lighthouse` | `build` | LHCI (mobile): **accessibility error gate** `minScore: 0.95`; **CLS error** ≤ 0.1; performance/SEO warn. Desktop run: `continue-on-error: true` until baselines stabilise. Timeout 25 min. | -| `storybook` | `quality` | Cloud-first — Storybook build + test-runner only run in CI (not locally); Playwright browser cache `v5`; `--maxWorkers=2 --junit` (non-blocking, `continue-on-error: true` — see [exit criteria](#non-blocking-gates--exit-criteria-f-13)); artifacts uploaded always. Debug: manual `storybook-debug.yml` workflow. | -| `vrt` | `build` | Visual regression against production `dist`; `toHaveScreenshot()` with committed PNG baselines (4 views × Chromium); artifacts uploaded always | +| `e2e` | `quality` | Playwright **Chromium** + **Mobile Chrome** (Pixel 5) — `CI=true`, 2× retries, 50 min timeout; browser cache via `actions/cache@v6.1.0`. Firefox optional locally. `PLAYWRIGHT_SKIP_VRT=true` (VRT runs in `browser-quality` instead). | +| `browser-quality` | `build` | **Consolidated (Wave 2 CI-performance)**: VRT then Lighthouse, sharing one checkout/setup/build-artifact download. VRT: visual regression against production `dist`, `toHaveScreenshot()` with committed PNG baselines (4 views × Chromium). Lighthouse (mobile): **accessibility error gate** `minScore: 0.95`; **CLS error** ≤ 0.1; performance/SEO warn — blocking. Lighthouse (desktop): accessibility/CLS are error-level LHCI assertions same as mobile and can fail the step itself, but the pre-existing `continue-on-error: true` (unchanged by this consolidation) keeps that failure from failing the job; promotion to blocking is gated on the 5-run exit criterion below. Each phase runs on `if: ${{ !cancelled() }}` relative to the others — a VRT or mobile-Lighthouse failure never skips a later phase's own evidence; only desktop Lighthouse's own failure is swallowed (job-level result). Artifacts uploaded always. Timeout 35 min. | +| `storybook` | `quality` | Cloud-first — Storybook build + test-runner only run in CI (not locally); Playwright browser cache `v6.1.0`; `--maxWorkers=2 --junit` (non-blocking, `continue-on-error: true` — see [exit criteria](#non-blocking-gates--exit-criteria-f-13)); artifacts uploaded always. Debug: manual `storybook-debug.yml` workflow. | | `signatures` | `security` | Read-only GitHub API verification of every commit in the complete introduced range; pull-request commit pagination; and annotated release-tag plus target-commit verification. | -| `ci-success` | `workflow-policy`, `pr-size`, `security`, `signatures`, `quality`, `changes`, `rust-tauri`, `core-rust`, `build`, `e2e`, `lighthouse`, `vrt` | Required-status **aggregator** — `if: always()`, fails if any required release-safety job does not resolve to `success`; signature verification is authoritative; Storybook and deep-E2E are explicitly advisory. Rust jobs are legitimately skipped when their paths are untouched; `pr-size` is legitimately skipped only on non-`pull_request` events. | +| `ci-success` | `workflow-policy`, `pr-size`, `security`, `signatures`, `quality`, `changes`, `rust-tauri`, `core-rust`, `build`, `e2e`, `browser-quality` | Required-status **aggregator** — `if: always()`, fails if any required release-safety job does not resolve to `success`; signature verification is authoritative; Storybook and deep-E2E are explicitly advisory. Rust jobs are legitimately skipped when their paths are untouched; `pr-size` is legitimately skipped only on non-`pull_request` events. | | `deploy` | `ci-success` | **Only** `main` push (not PR), and only after the aggregate gate succeeds; the Pages artifact is resolved from the same workflow run. | > **Desktop:** On-demand / tag-driven Tauri bundles live in [`tauri-build.yml`](../.github/workflows/tauri-build.yml); **`v*` tags** additionally publish installers on a **GitHub Release**. See [`docs/TAURI-CI.md`](TAURI-CI.md). Desktop CI does not block the web deploy graph above. @@ -276,7 +278,7 @@ in `ci.yml` is listed here with why it isn't blocking today and what has to be t |------|------------------------|-----------------| | Storybook `test-storybook` (`storybook` job, `continue-on-error: true`) | Fixed 2026-07-29 (F-12): the invocation was calling flags this test-runner version doesn't support (`--max-workers`/`--retries`/`--screenshot-on-failure`), so it failed on argument parsing before running a single story, on every prior run. This is the first run where it will actually execute real stories/a11y checks — no track record exists yet. Moved from `\|\| true` to job-level `continue-on-error: true` the same day (CodeRabbit-caught): `\|\| true` swallowed the exit code so the step showed green even while genuinely failing — the exact mechanism that let the F-12 bug go unnoticed. | Re-evaluate after **~10 real (non-argument-error) runs on `main`**; drop `continue-on-error` and make blocking if none fail on a genuine story/a11y assertion. | | `e2e-deep` (feature-flag matrix job, `continue-on-error: true`) | Deliberately informational by design — parametrizes across the full `testConfigurations` flag matrix (`tests/e2e/config/test-matrix.ts`) specifically to surface flag-interaction regressions that the required `e2e` gate's default-flag-state run cannot see; failures here are diagnostic signal, not necessarily a merge-blocking defect in the default configuration. | Promote a **specific flag combination** to blocking (add it to the required `e2e` spec instead) once it has been stable for **3 consecutive weeks of `main` runs** — do not flip the entire matrix job blocking at once, since that reintroduces the flakiness-cascade risk `e2e-deep` was created to avoid. | -| Lighthouse **Desktop** step (`lighthouse` job, `continue-on-error: true`) — note: accessibility (`minScore: 0.95`) and CLS (`≤ 0.1`) stay **error**-level gates even on this step; only the broader desktop performance/SEO scores are non-blocking | Desktop performance baselines haven't been formally re-verified as stable since the last CI-runner change | Re-run `pnpm exec lhci autorun --config=.lighthouserc.desktop.cjs` locally or via `storybook-debug.yml`-style manual dispatch across **5 consecutive `main` runs**; if performance/SEO scores stay within the existing `warn` thresholds each time, remove `continue-on-error` for the step. | +| Lighthouse **Desktop** step (`browser-quality` job, `continue-on-error: true`) — note: accessibility (`minScore: 0.95`) and CLS (`≤ 0.1`) are still evaluated as **error**-level LHCI assertions and can fail the step itself, but `continue-on-error` swallows that failure at job/merge level along with the broader desktop performance/SEO scores — none of it blocks today | Desktop performance baselines haven't been formally re-verified as stable since the last CI-runner change | Re-run `pnpm exec lhci autorun --config=.lighthouserc.desktop.cjs` locally or via `storybook-debug.yml`-style manual dispatch across **5 consecutive `main` runs**; if performance/SEO scores stay within the existing `warn` thresholds each time, remove `continue-on-error` for the step. | | Coverage ratchet (`scripts/check-coverage-ratchet.mjs`, `continue-on-error: true`) | **Deliberately, permanently advisory** — by design (see `vitest.config.ts`'s ratchet-history comment), it exists to *suggest* the next threshold bump, not to gate a merge on hitting one. | None — this is the one gate above intentionally without an exit criterion; its purpose is met by staying advisory. Reviewed here for completeness so it isn't mistaken for forgotten debt. | --- diff --git a/tests/unit/workflowPolicy.test.ts b/tests/unit/workflowPolicy.test.ts index 8febefbb..10769f29 100644 --- a/tests/unit/workflowPolicy.test.ts +++ b/tests/unit/workflowPolicy.test.ts @@ -181,11 +181,10 @@ describe('CI workflow policy', () => { 'core-rust', 'build', 'e2e', - 'lighthouse', - 'vrt', + 'browser-quality', ]); expect(ciSuccessBlock).toMatch(/\$\{\{\s*needs\.signatures\.result\s*\}\}/); - expect(ciSuccessBlock).toMatch(/\$\{\{\s*needs\.lighthouse\.result\s*\}\}/); + expect(ciSuccessBlock).toMatch(/\$\{\{\s*needs\.browser-quality\.result\s*\}\}/); for (const jobName of ['e2e-deep', 'storybook']) { const jobBlock = extractJobBlock(workflowSource, jobName); @@ -327,7 +326,7 @@ describe('Tauri release workflow policy', () => { name: 'storybook', }, { - job: 'vrt', + job: 'browser-quality', step: 'Publish VRT test results to Codecov', files: 'tests/e2e/results/junit.xml', flags: 'vrt',