OEL-4248: VRT Backstop

.gitattributes

@@ -0,0 +1,60 @@
+# Drupal git normalization
+# @see https://www.kernel.org/pub/software/scm/git/docs/gitattributes.html
+# @see https://www.drupal.org/node/1542048
+# Normally these settings would be done with macro attributes for improved
+# readability and easier maintenance. However macros can only be defined at the
+# repository root directory. Drupal avoids making any assumptions about where it
+# is installed.
+# Define text file attributes.
+# - Treat them as text.
+# - Ensure no CRLF line-endings, neither on checkout nor on checkin.
+# - Detect whitespace errors.
+#   - Exposed by default in `git diff --color` on the CLI.
+#   - Validate with `git diff --check`.
+#   - Deny applying with `git apply --whitespace=error-all`.
+#   - Fix automatically with `git apply --whitespace=fix`.
+*.config  text eol=lf whitespace=blank-at-eol,-blank-at-eof,-space-before-tab,tab-in-indent,tabwidth=2
+*.css     text eol=lf whitespace=blank-at-eol,-blank-at-eof,-space-before-tab,tab-in-indent,tabwidth=2
+*.dist    text eol=lf whitespace=blank-at-eol,-blank-at-eof,-space-before-tab,tab-in-indent,tabwidth=2
+*.engine  text eol=lf whitespace=blank-at-eol,-blank-at-eof,-space-before-tab,tab-in-indent,tabwidth=2 diff=php linguist-language=php
+*.html    text eol=lf whitespace=blank-at-eol,-blank-at-eof,-space-before-tab,tab-in-indent,tabwidth=2 diff=html
+*.inc     text eol=lf whitespace=blank-at-eol,-blank-at-eof,-space-before-tab,tab-in-indent,tabwidth=2 diff=php linguist-language=php
+*.install text eol=lf whitespace=blank-at-eol,-blank-at-eof,-space-before-tab,tab-in-indent,tabwidth=2 diff=php linguist-language=php
+*.js      text eol=lf whitespace=blank-at-eol,-blank-at-eof,-space-before-tab,tab-in-indent,tabwidth=2
+*.json    text eol=lf whitespace=blank-at-eol,-blank-at-eof,-space-before-tab,tab-in-indent,tabwidth=2
+*.lock    text eol=lf whitespace=blank-at-eol,-blank-at-eof,-space-before-tab,tab-in-indent,tabwidth=2
+*.map     text eol=lf whitespace=blank-at-eol,-blank-at-eof,-space-before-tab,tab-in-indent,tabwidth=2
+*.md      text eol=lf whitespace=blank-at-eol,-blank-at-eof,-space-before-tab,tab-in-indent,tabwidth=2
+*.module  text eol=lf whitespace=blank-at-eol,-blank-at-eof,-space-before-tab,tab-in-indent,tabwidth=2 diff=php linguist-language=php
+*.php     text eol=lf whitespace=blank-at-eol,-blank-at-eof,-space-before-tab,tab-in-indent,tabwidth=2 diff=php linguist-language=php
+*.po      text eol=lf whitespace=blank-at-eol,-blank-at-eof,-space-before-tab,tab-in-indent,tabwidth=2
+*.profile text eol=lf whitespace=blank-at-eol,-blank-at-eof,-space-before-tab,tab-in-indent,tabwidth=2 diff=php linguist-language=php
+*.script  text eol=lf whitespace=blank-at-eol,-blank-at-eof,-space-before-tab,tab-in-indent,tabwidth=2
+*.sh      text eol=lf whitespace=blank-at-eol,-blank-at-eof,-space-before-tab,tab-in-indent,tabwidth=2 diff=php linguist-language=php
+*.sql     text eol=lf whitespace=blank-at-eol,-blank-at-eof,-space-before-tab,tab-in-indent,tabwidth=2
+*.svg     text eol=lf whitespace=blank-at-eol,-blank-at-eof,-space-before-tab,tab-in-indent,tabwidth=2
+*.theme   text eol=lf whitespace=blank-at-eol,-blank-at-eof,-space-before-tab,tab-in-indent,tabwidth=2 diff=php linguist-language=php
+*.twig    text eol=lf whitespace=blank-at-eol,-blank-at-eof,-space-before-tab,tab-in-indent,tabwidth=2
+*.txt     text eol=lf whitespace=blank-at-eol,-blank-at-eof,-space-before-tab,tab-in-indent,tabwidth=2
+*.xml     text eol=lf whitespace=blank-at-eol,-blank-at-eof,-space-before-tab,tab-in-indent,tabwidth=2
+*.yml     text eol=lf whitespace=blank-at-eol,-blank-at-eof,-space-before-tab,tab-in-indent,tabwidth=2
+# PHPStan's baseline uses tabs instead of spaces.
+core/.phpstan-baseline.php text eol=lf whitespace=blank-at-eol,-blank-at-eof,-space-before-tab,tabwidth=2 diff=php linguist-language=php
+# Define binary file attributes.
+# - Do not treat them as text.
+# - Include binary diff in patches instead of "binary files differ."
+*.avif    -text diff
+*.eot     -text diff
+*.exe     -text diff
+*.gif     -text diff
+*.gz      -text diff
+*.ico     -text diff
+*.jpeg    -text diff
+*.jpg     -text diff
+*.otf     -text diff
+*.phar    -text diff
+*.png filter=lfs diff=lfs merge=lfs -text
+*.svgz    -text diff
+*.ttf     -text diff
+*.woff    -text diff
+*.woff2   -text diff

.github/workflows/backstop.yml

@@ -0,0 +1,71 @@
+name: BackstopJS VRT
+
+on:
+  pull_request:
+
+jobs:
+  backstop:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          lfs: true
+
+      - name: Setup Node
+        uses: actions/setup-node@v6
+
+      - name: Install dependencies
+        run: |
+          npm install
+          npx playwright install --with-deps
+
+      - name: Start Drupal stack
+        run: docker compose up -d mysql web
+
+      - name: Install Drupal
+        run: |
+          docker compose exec -T web composer install
+          docker compose exec -T web ./vendor/bin/run drupal:site-install
+
+      - name: Build Backstop config
+        run: npm run vrt:build
+
+      - name: Run BackstopJS
+        id: backstop
+        run: |
+          npm run vrt:test
+        # Keep artifact uploads running even when Backstop finds visual diffs.
+        continue-on-error: true
+
+      - name: Check Backstop report exists
+        if: always()
+        run: |
+          if [ ! -f tests/vrt/html_report/index.html ]; then
+            echo "Backstop HTML report was not generated"
+            exit 1
+          fi
+
+      - name: Upload Backstop results
+        id: upload_report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          # Preserve the HTML report together with test and reference screenshots.
+          name: backstop-results-${{ github.sha }}
+          path: |
+            tests/vrt/html_report
+            tests/vrt/bitmaps_test
+            tests/vrt/bitmaps_reference
+          if-no-files-found: warn
+
+      - name: Fail if Backstop failed
+        if: always() && steps.backstop.outcome != 'success'
+        run: |
+          echo "::error title=Run BackstopJS failed::Check the 'Run BackstopJS' step above."
+          exit 1
+
+      - name: Stop Drupal stack
+        if: always()
+        run: docker compose down

.gitignore

@@ -16,9 +16,14 @@
 /docker-compose.*.yml
 /.sass-cache
 /.env
-/.gitattributes
 /.phpunit.result.cache
 # Ignore the directory generated by `toolkit:test-phpunit --junit`.
 /junit-export/
 # Toolkit sets the config sync directory to '../config/sync/'.
 /config/sync/
+# BackstopJS
+tests/vrt/bitmaps_test/
+tests/vrt/html_report/
+tests/vrt/scenarios/patterns.json
+tests/vrt/backstop.generated.json
+

README.md

@@ -120,6 +120,10 @@ regardless of your local host configuration.
 
 - [Docker](https://www.docker.com/get-docker)
 - [Docker Compose](https://docs.docker.com/compose/)
+- [Git LFS](https://docs.github.com/en/repositories/working-with-files/managing-large-files/installing-git-large-file-storage)
+
+
+> __Important__: Git LFS is required for development. Builds may fail or assets may be missing if Git LFS is not installed and initialized before cloning the repository.
 
 #### Override docker settings
 
@@ -209,6 +213,23 @@ docker-compose exec web npm run build
 docker-compose exec web npm run watch
 ```
 
+## Visual regression testing (BackstopJS)
+
+BackstopJS lives under `tests/vrt/`. The scenario builder and Playwright helper scripts in this directory generate `tests/vrt/scenarios/patterns.json` and `tests/vrt/backstop.generated.json`, which Backstop consumes for visual checks.
+
+### Prerequisites
+
+- A running Drupal site reachable at `http://localhost:8080/build`. Override the target with the `VRT_BASE_URL` environment variable if needed.
+- Valid credentials. The default `admin` / `admin` values can be overridden with `DRUPAL_USER` and `DRUPAL_PASS`.
+- Playwright: `docker compose exec web npx playwright install-deps`
+
+### Available commands
+
+- `docker compose exec web npm run vrt:build` – scrapes the UI Patterns catalog and generates `tests/vrt/scenarios/patterns.json` plus `tests/vrt/backstop.generated.json`.
+- `docker compose exec web npm run vrt:reference` – captures fresh reference screenshots into `tests/vrt/bitmaps_reference`.
+- `docker compose exec web npm run vrt:test` – compares the current UI against the references, writing results to `tests/vrt/bitmaps_test` and `tests/vrt/html_report`.
+- `docker compose exec web npm run vrt:approve` – promotes the latest passing screenshots to the reference set.
+
 ## Patch BCL components
 
 BCL components can be patched by using the [`patch-package`](https://www.npmjs.com/package/patch-package) NPM project.

backstop.json

@@ -0,0 +1,45 @@
+{
+  "id": "oe_bootstrap_theme",
+  "viewports": [
+    {
+      "label": "mobile",
+      "width": 375,
+      "height": 667
+    },
+    {
+      "label": "tablet",
+      "width": 768,
+      "height": 1024
+    },
+    {
+      "label": "desktop",
+      "width": 1280,
+      "height": 800
+    },
+    {
+      "label": "wide",
+      "width": 1440,
+      "height": 900
+    }
+  ],
+  "onBeforeScript": "playwright/onBefore.js",
+  "onReadyScript": "playwright/onReady.js",
+  "paths": {
+    "bitmaps_reference": "tests/vrt/bitmaps_reference",
+    "bitmaps_test": "tests/vrt/bitmaps_test",
+    "engine_scripts": "tests/vrt/engine_scripts",
+    "html_report": "tests/vrt/html_report",
+    "ci_report": "tests/vrt/ci_report"
+  },
+  "report": [
+    "browser"
+  ],
+  "engine": "playwright",
+  "engineOptions": {
+    "browser": "chromium"
+  },
+  "asyncCaptureLimit": 5,
+  "asyncCompareLimit": 50,
+  "debug": false,
+  "debugWindow": false
+}

package.json

@@ -12,17 +12,24 @@
     "watch": "npm-run-all --parallel watch:* -ln",
     "postinstall": "patch-package --patch-dir=patches/npm",
     "prepare": "npm-run-all build:*",
-    "production": "npm-run-all build:*"
+    "production": "npm-run-all build:*",
+    "vrt:build": "node tests/vrt/build-pattern-scenarios.js && node tests/vrt/build-config.js",
+    "vrt:reference": "backstop reference --config=tests/vrt/backstop.generated.json",
+    "vrt:test": "backstop test --config=tests/vrt/backstop.generated.json",
+    "vrt:approve": "backstop approve --config=tests/vrt/backstop.generated.json"
   },
   "devDependencies": {
-    "@openeuropa/bcl-builder": "^1.13",
-    "@openeuropa/bcl-theme-default": "^1.13",
+    "@openeuropa/bcl-builder": "0.1.202603061200",
+    "@openeuropa/bcl-theme-default": "0.1.202603061200",
+    "backstopjs": "^6.3.25",
     "bootstrap-ie11": "5.0.2",
+    "cheerio": "^1.1.2",
     "chokidar-cli": "^3.0.0",
     "copyfiles": "2.4.1",
     "cross-env": "7.0.3",
     "glob": "10.0.0",
     "npm-run-all": "4.1.5",
-    "patch-package": "^6.4.7"
+    "patch-package": "^6.4.7",
+    "playwright": "^1.57.0"
   }
 }