From 4e50e3c2859a16988aa818cfb7dbf3a639b40a08 Mon Sep 17 00:00:00 2001 From: vict0rsch Date: Sat, 30 May 2026 23:34:17 +0200 Subject: [PATCH 01/19] refactor: use cloakbrowser instead of puppeteer --- .github/workflows/_test-matrix.yml | 9 + contributing.md | 2 +- .../0001-cloakbrowser-stealth-test-browser.md | 30 + package-lock.json | 721 ++++-------------- package.json | 6 +- src/shared/js/utils/urls.js | 80 +- system-architecture.md | 8 +- test/browser.js | 14 +- test/citations-scraper.js | 7 +- 9 files changed, 226 insertions(+), 651 deletions(-) create mode 100644 docs/adr/0001-cloakbrowser-stealth-test-browser.md diff --git a/.github/workflows/_test-matrix.yml b/.github/workflows/_test-matrix.yml index 8dfc8609..28d44e19 100644 --- a/.github/workflows/_test-matrix.yml +++ b/.github/workflows/_test-matrix.yml @@ -45,6 +45,15 @@ jobs: - name: Install Dependencies run: npm ci + - name: Cache cloakbrowser binary + uses: actions/cache@v4 + with: + path: ~/.cloakbrowser + key: cloakbrowser-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('package-lock.json') }} + + - name: Prefetch cloakbrowser binary + run: npx cloakbrowser install + - name: Build Extension run: npm run build:chrome diff --git a/contributing.md b/contributing.md index 9921818d..30386261 100644 --- a/contributing.md +++ b/contributing.md @@ -279,7 +279,7 @@ Sources are **stateless** `BasePaperSource` subclasses (the registry does a name ## Tests -Tests use Puppeteer to launch a real Chrome instance with the extension loaded. `npm test` builds the extension first (`pretest` runs `npm run build:chrome`), then runs all test suites. +Tests use [cloakbrowser](https://github.com/CloakHQ/cloakbrowser) (a source-level stealth Chromium with the Puppeteer-core API) to launch a real Chrome instance with the extension loaded. On first run, cloakbrowser auto-downloads its custom Chromium binary (~200MB) to `~/.cloakbrowser`; subsequent runs use the cached binary. `CHROME_PATH` is not used for the stealth browser. `npm test` builds the extension first (`pretest` runs `npm run build:chrome`), then runs all test suites. The integration suites `test/test-duplicates.js` and `test/test-storage.js` hit many live publisher pages; they are valuable as smoke checks but are **not expected to pass entirely** (timeouts, bot walls, and DOM drift are common). `npm test` excludes `test/test-sync.js`; run it explicitly with `npm run test:file test/test-sync.js` when working on Gist sync — that file has the same “best effort” expectations. diff --git a/docs/adr/0001-cloakbrowser-stealth-test-browser.md b/docs/adr/0001-cloakbrowser-stealth-test-browser.md new file mode 100644 index 00000000..02a9bbff --- /dev/null +++ b/docs/adr/0001-cloakbrowser-stealth-test-browser.md @@ -0,0 +1,30 @@ +# Use cloakbrowser as the headless browser for integration tests + +The test suite drives a real Chrome instance with the unpacked extension loaded to +visit live publisher pages. The previous stack (`puppeteer-extra` + +`puppeteer-extra-plugin-stealth`) injects JavaScript patches that antibot systems +increasingly detect — causing intermittent failures on Cloudflare-fronted and other +bot-walled academic publisher pages. We replaced it with +[cloakbrowser](https://github.com/CloakHQ/cloakbrowser), which applies stealth at +the Chromium C++ source level rather than via JavaScript injection, making the patches +transparent to bot detection. + +## Considered options + +- **Playwright mode** (`import { launch } from 'cloakbrowser'`) — the recommended + cloakbrowser API, better reCAPTCHA resistance. Rejected: would require rewriting all + test helpers that assume the Puppeteer `Browser`/`Page` API (`browser.pages()`, + `page.evaluate`, `waitForSelector`, `screenshot`, etc.). +- **puppeteer-extra-plugin-stealth** (status quo) — easy but config-level: patches + break on every Chrome update and are detectable themselves. + +## Consequences + +- A custom Chromium binary (~200MB) auto-downloads to `~/.cloakbrowser` on first use + per platform. CI caches it via `actions/cache` keyed on `package-lock.json`. +- On macOS the binary trails one Chromium major version behind linux/windows (145 vs + 146 as of the initial adoption). This has not caused test divergence. +- `CHROME_PATH` / the system-installed Chrome is no longer used for tests. +- `--disable-web-security` and `--disable-gpu` remain in the test args for now; + they partially undermine stealth and are candidates for removal once stability is + confirmed. diff --git a/package-lock.json b/package-lock.json index 6f544b9c..2c949ec4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "papermemory", - "version": "1.1.0", + "version": "1.1.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "papermemory", - "version": "1.1.0", + "version": "1.1.1", "hasInstallScript": true, "license": "MIT", "dependencies": { @@ -15,6 +15,7 @@ "tom-select": "^2.6.0" }, "devDependencies": { + "cloakbrowser": "^0.3.31", "expect": "^29.7.0", "glob": "^10.3.10", "heap": "^0.2.7", @@ -23,9 +24,7 @@ "ora": "^5.4.1", "postcss": "^8.5.6", "prettier": "^3.8.3", - "puppeteer": "^24.16.0", - "puppeteer-extra": "^3.3.6", - "puppeteer-extra-plugin-stealth": "^2.11.2", + "puppeteer-core": "^24.16.0", "readline-sync": "^1.4.10", "uuid": "^9.0.1", "wxt": "^0.20.25", @@ -984,6 +983,19 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, + "node_modules/@isaacs/fs-minipass": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", + "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "dev": true, + "license": "ISC", + "dependencies": { + "minipass": "^7.0.4" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@jest/expect-utils": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-29.7.0.tgz", @@ -1573,16 +1585,6 @@ "tslib": "^2.4.0" } }, - "node_modules/@types/debug": { - "version": "4.1.13", - "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.13.tgz", - "integrity": "sha512-KSVgmQmzMwPlmtljOomayoR89W4FynCAi3E8PPs7vmDVPe84hT+vGPKkJfThkmXs0x0jAaa9U8uW8bbfyS2fWw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/ms": "*" - } - }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", @@ -1641,13 +1643,6 @@ "@types/istanbul-lib-report": "*" } }, - "node_modules/@types/ms": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz", - "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==", - "dev": true, - "license": "MIT" - }, "node_modules/@types/node": { "version": "24.2.0", "resolved": "https://registry.npmjs.org/@types/node/-/node-24.2.0.tgz", @@ -1862,16 +1857,6 @@ "dev": true, "license": "Python-2.0" }, - "node_modules/arr-union": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/arr-union/-/arr-union-3.1.0.tgz", - "integrity": "sha512-sKpyeERZ02v1FeCZT8lrfJq5u6goHCtpTAzPwJYe7c8SPFOboNjNg1vz2L4VTn9T4PQxEx13TbXLmYUcS6Ug7Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/array-differ": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/array-differ/-/array-differ-4.0.0.tgz", @@ -2467,16 +2452,6 @@ "node": ">= 0.4" } }, - "node_modules/callsites": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", - "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -2519,6 +2494,16 @@ "fsevents": "~2.3.2" } }, + "node_modules/chownr": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", + "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, "node_modules/chrome-launcher": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/chrome-launcher/-/chrome-launcher-1.2.0.tgz", @@ -2692,44 +2677,50 @@ "node": ">=12" } }, - "node_modules/clone": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz", - "integrity": "sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.8" - } - }, - "node_modules/clone-deep": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-0.2.4.tgz", - "integrity": "sha512-we+NuQo2DHhSl+DP6jlUiAhyAjBQrYnpOk15rN6c6JSPScjiCLh8IbSU+VTcph6YS3o7mASE8a0+gbZ7ChLpgg==", + "node_modules/cloakbrowser": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/cloakbrowser/-/cloakbrowser-0.3.31.tgz", + "integrity": "sha512-HtwlpwFKd5FJh4jjCS5awsScWIKB9ofNSTkHLSEh/9JH0Hv1L79zu0MExSMOCP1nzOsgrohbacwwWocuKloQ1A==", "dev": true, "license": "MIT", "dependencies": { - "for-own": "^0.1.3", - "is-plain-object": "^2.0.1", - "kind-of": "^3.0.2", - "lazy-cache": "^1.0.3", - "shallow-clone": "^0.1.2" + "tar": "^7.0.0" + }, + "bin": { + "cloakbrowser": "dist/cli.js" }, "engines": { - "node": ">=0.10.0" + "node": ">=20.0.0" + }, + "peerDependencies": { + "mmdb-lib": ">=2.0.0", + "playwright-core": ">=1.53.0", + "puppeteer-core": ">=21.0.0", + "socks-proxy-agent": ">=10.0.0" + }, + "peerDependenciesMeta": { + "mmdb-lib": { + "optional": true + }, + "playwright-core": { + "optional": true + }, + "puppeteer-core": { + "optional": true + }, + "socks-proxy-agent": { + "optional": true + } } }, - "node_modules/clone-deep/node_modules/is-plain-object": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", - "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "node_modules/clone": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz", + "integrity": "sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==", "dev": true, "license": "MIT", - "dependencies": { - "isobject": "^3.0.1" - }, "engines": { - "node": ">=0.10.0" + "node": ">=0.8" } }, "node_modules/color-convert": { @@ -2882,52 +2873,6 @@ "dev": true, "license": "MIT" }, - "node_modules/cosmiconfig": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-9.0.0.tgz", - "integrity": "sha512-itvL5h8RETACmOTFc4UfIyB2RfEHi71Ax6E/PivVxq9NseKbOWpeyHEOIbmAw1rs8Ak0VursQNww7lf7YtUwzg==", - "dev": true, - "license": "MIT", - "dependencies": { - "env-paths": "^2.2.1", - "import-fresh": "^3.3.0", - "js-yaml": "^4.1.0", - "parse-json": "^5.2.0" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/d-fischer" - }, - "peerDependencies": { - "typescript": ">=4.9.5" - }, - "peerDependenciesMeta": { - "typescript": { - "optional": true - } - } - }, - "node_modules/cosmiconfig/node_modules/parse-json": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", - "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.0.0", - "error-ex": "^1.3.1", - "json-parse-even-better-errors": "^2.3.0", - "lines-and-columns": "^1.1.6" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -3050,16 +2995,6 @@ "node": ">=4.0.0" } }, - "node_modules/deepmerge": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", - "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/default-browser": { "version": "5.4.0", "resolved": "https://registry.npmjs.org/default-browser/-/default-browser-5.4.0.tgz", @@ -3354,16 +3289,6 @@ "url": "https://github.com/fb55/entities?sponsor=1" } }, - "node_modules/env-paths": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", - "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "node_modules/environment": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/environment/-/environment-1.1.0.tgz", @@ -3773,29 +3698,6 @@ "flat": "cli.js" } }, - "node_modules/for-in": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", - "integrity": "sha512-7EwmXrOjyL+ChxMhmG5lnW9MPt1aIeZEwKhQzoBUdTV0N3zuwWDZYVJatDvZ2OyzPUvdIAZDsCetk3coyMfcnQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/for-own": { - "version": "0.1.5", - "resolved": "https://registry.npmjs.org/for-own/-/for-own-0.1.5.tgz", - "integrity": "sha512-SKmowqGTJoPzLO1T0BBJpkfp3EMacCMOuH40hOUbrbzElVktk4DioXVM99QkLCyKoiuOmyjgcWMpVz2xjE7LZw==", - "dev": true, - "license": "MIT", - "dependencies": { - "for-in": "^1.0.1" - }, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/form-data": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", @@ -4414,33 +4316,6 @@ "dev": true, "license": "MIT" }, - "node_modules/import-fresh": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", - "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "parent-module": "^1.0.0", - "resolve-from": "^4.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/import-fresh/node_modules/resolve-from": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", - "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, "node_modules/import-meta-resolve": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-4.2.0.tgz", @@ -4523,13 +4398,6 @@ "node": ">=8" } }, - "node_modules/is-buffer": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", - "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", - "dev": true, - "license": "MIT" - }, "node_modules/is-docker": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz", @@ -4546,16 +4414,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/is-extendable": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", - "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -4976,13 +4834,6 @@ } } }, - "node_modules/json-parse-even-better-errors": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", - "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", - "dev": true, - "license": "MIT" - }, "node_modules/json5": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", @@ -5101,19 +4952,6 @@ "safe-buffer": "^5.0.1" } }, - "node_modules/kind-of": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", - "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-buffer": "^1.1.5" - }, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/kleur": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", @@ -5153,16 +4991,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/lazy-cache": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-1.0.4.tgz", - "integrity": "sha512-RE2g0b5VGZsOCFOCgP7omTRYFqydmZkBwl5oNnQ1lDYC57uyO9KqNnNVxT7COSHTxrRCWVcAVOcbjk+tvh/rgQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/lie": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", @@ -5445,13 +5273,6 @@ "url": "https://opencollective.com/parcel" } }, - "node_modules/lines-and-columns": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", - "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", - "dev": true, - "license": "MIT" - }, "node_modules/linkedom": { "version": "0.18.12", "resolved": "https://registry.npmjs.org/linkedom/-/linkedom-0.18.12.tgz", @@ -5998,21 +5819,6 @@ "node": ">= 0.4" } }, - "node_modules/merge-deep": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/merge-deep/-/merge-deep-3.0.3.tgz", - "integrity": "sha512-qtmzAS6t6grwEkNrunqTBdn0qKwFgNWvlxUbAV8es9M7Ot1EbyApytCnvE0jALPa46ZpKDUo527kKiaWplmlFA==", - "dev": true, - "license": "MIT", - "dependencies": { - "arr-union": "^3.1.0", - "clone-deep": "^0.2.4", - "kind-of": "^3.0.2" - }, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/micromatch": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", @@ -6115,36 +5921,25 @@ "node": ">=16 || 14 >=14.17" } }, - "node_modules/mitt": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz", - "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==", - "dev": true, - "license": "MIT" - }, - "node_modules/mixin-object": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/mixin-object/-/mixin-object-2.0.1.tgz", - "integrity": "sha512-ALGF1Jt9ouehcaXaHhn6t1yGWRqGaHkPFndtFVHfZXOvkIZ/yoGaSi0AHVTafb3ZBGg4dr/bDwnaEKqCXzchMA==", + "node_modules/minizlib": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", + "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", "dev": true, "license": "MIT", "dependencies": { - "for-in": "^0.1.3", - "is-extendable": "^0.1.1" + "minipass": "^7.1.2" }, "engines": { - "node": ">=0.10.0" + "node": ">= 18" } }, - "node_modules/mixin-object/node_modules/for-in": { - "version": "0.1.8", - "resolved": "https://registry.npmjs.org/for-in/-/for-in-0.1.8.tgz", - "integrity": "sha512-F0to7vbBSHP8E3l6dCjxNOLuSFAACIxFy3UehTUlG7svlXi37HHsDkyVcHo0Pq8QwrE+pXvWSVX3ZT1T9wAZ9g==", + "node_modules/mitt": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz", + "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==", "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } + "license": "MIT" }, "node_modules/mlly": { "version": "1.8.2", @@ -6671,6 +6466,21 @@ "node": ">= 14" } }, + "node_modules/pac-proxy-agent/node_modules/socks-proxy-agent": { + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", + "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", + "dev": true, + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "socks": "^2.8.3" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/pac-resolver": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz", @@ -6731,19 +6541,6 @@ "dev": true, "license": "(MIT AND Zlib)" }, - "node_modules/parent-module": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", - "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", - "dev": true, - "license": "MIT", - "dependencies": { - "callsites": "^3.0.0" - }, - "engines": { - "node": ">=6" - } - }, "node_modules/parse5": { "version": "7.3.0", "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", @@ -6780,16 +6577,6 @@ "node": ">=8" } }, - "node_modules/path-is-absolute": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", @@ -7071,6 +6858,21 @@ "node": ">=12" } }, + "node_modules/proxy-agent/node_modules/socks-proxy-agent": { + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", + "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", + "dev": true, + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "socks": "^2.8.3" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/proxy-from-env": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", @@ -7162,28 +6964,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/puppeteer": { - "version": "24.30.0", - "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-24.30.0.tgz", - "integrity": "sha512-A5OtCi9WpiXBQgJ2vQiZHSyrAzQmO/WDsvghqlN4kgw21PhxA5knHUaUQq/N3EMt8CcvSS0RM+kmYLJmedR3TQ==", - "dev": true, - "hasInstallScript": true, - "license": "Apache-2.0", - "dependencies": { - "@puppeteer/browsers": "2.10.13", - "chromium-bidi": "11.0.0", - "cosmiconfig": "^9.0.0", - "devtools-protocol": "0.0.1521046", - "puppeteer-core": "24.30.0", - "typed-query-selector": "^2.12.0" - }, - "bin": { - "puppeteer": "lib/cjs/puppeteer/node/cli.js" - }, - "engines": { - "node": ">=18" - } - }, "node_modules/puppeteer-core": { "version": "24.30.0", "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-24.30.0.tgz", @@ -7203,185 +6983,6 @@ "node": ">=18" } }, - "node_modules/puppeteer-extra": { - "version": "3.3.6", - "resolved": "https://registry.npmjs.org/puppeteer-extra/-/puppeteer-extra-3.3.6.tgz", - "integrity": "sha512-rsLBE/6mMxAjlLd06LuGacrukP2bqbzKCLzV1vrhHFavqQE/taQ2UXv3H5P0Ls7nsrASa+6x3bDbXHpqMwq+7A==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/debug": "^4.1.0", - "debug": "^4.1.1", - "deepmerge": "^4.2.2" - }, - "engines": { - "node": ">=8" - }, - "peerDependencies": { - "@types/puppeteer": "*", - "puppeteer": "*", - "puppeteer-core": "*" - }, - "peerDependenciesMeta": { - "@types/puppeteer": { - "optional": true - }, - "puppeteer": { - "optional": true - }, - "puppeteer-core": { - "optional": true - } - } - }, - "node_modules/puppeteer-extra-plugin": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin/-/puppeteer-extra-plugin-3.2.3.tgz", - "integrity": "sha512-6RNy0e6pH8vaS3akPIKGg28xcryKscczt4wIl0ePciZENGE2yoaQJNd17UiEbdmh5/6WW6dPcfRWT9lxBwCi2Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/debug": "^4.1.0", - "debug": "^4.1.1", - "merge-deep": "^3.0.1" - }, - "engines": { - "node": ">=9.11.2" - }, - "peerDependencies": { - "playwright-extra": "*", - "puppeteer-extra": "*" - }, - "peerDependenciesMeta": { - "playwright-extra": { - "optional": true - }, - "puppeteer-extra": { - "optional": true - } - } - }, - "node_modules/puppeteer-extra-plugin-stealth": { - "version": "2.11.2", - "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-stealth/-/puppeteer-extra-plugin-stealth-2.11.2.tgz", - "integrity": "sha512-bUemM5XmTj9i2ZerBzsk2AN5is0wHMNE6K0hXBzBXOzP5m5G3Wl0RHhiqKeHToe/uIH8AoZiGhc1tCkLZQPKTQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "debug": "^4.1.1", - "puppeteer-extra-plugin": "^3.2.3", - "puppeteer-extra-plugin-user-preferences": "^2.4.1" - }, - "engines": { - "node": ">=8" - }, - "peerDependencies": { - "playwright-extra": "*", - "puppeteer-extra": "*" - }, - "peerDependenciesMeta": { - "playwright-extra": { - "optional": true - }, - "puppeteer-extra": { - "optional": true - } - } - }, - "node_modules/puppeteer-extra-plugin-user-data-dir": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-user-data-dir/-/puppeteer-extra-plugin-user-data-dir-2.4.1.tgz", - "integrity": "sha512-kH1GnCcqEDoBXO7epAse4TBPJh9tEpVEK/vkedKfjOVOhZAvLkHGc9swMs5ChrJbRnf8Hdpug6TJlEuimXNQ+g==", - "dev": true, - "license": "MIT", - "dependencies": { - "debug": "^4.1.1", - "fs-extra": "^10.0.0", - "puppeteer-extra-plugin": "^3.2.3", - "rimraf": "^3.0.2" - }, - "engines": { - "node": ">=8" - }, - "peerDependencies": { - "playwright-extra": "*", - "puppeteer-extra": "*" - }, - "peerDependenciesMeta": { - "playwright-extra": { - "optional": true - }, - "puppeteer-extra": { - "optional": true - } - } - }, - "node_modules/puppeteer-extra-plugin-user-data-dir/node_modules/fs-extra": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", - "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "graceful-fs": "^4.2.0", - "jsonfile": "^6.0.1", - "universalify": "^2.0.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/puppeteer-extra-plugin-user-data-dir/node_modules/jsonfile": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.0.tgz", - "integrity": "sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==", - "dev": true, - "license": "MIT", - "dependencies": { - "universalify": "^2.0.0" - }, - "optionalDependencies": { - "graceful-fs": "^4.1.6" - } - }, - "node_modules/puppeteer-extra-plugin-user-data-dir/node_modules/universalify": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", - "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/puppeteer-extra-plugin-user-preferences": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-user-preferences/-/puppeteer-extra-plugin-user-preferences-2.4.1.tgz", - "integrity": "sha512-i1oAZxRbc1bk8MZufKCruCEC3CCafO9RKMkkodZltI4OqibLFXF3tj6HZ4LZ9C5vCXZjYcDWazgtY69mnmrQ9A==", - "dev": true, - "license": "MIT", - "dependencies": { - "debug": "^4.1.1", - "deepmerge": "^4.2.2", - "puppeteer-extra-plugin": "^3.2.3", - "puppeteer-extra-plugin-user-data-dir": "^2.4.1" - }, - "engines": { - "node": ">=8" - }, - "peerDependencies": { - "playwright-extra": "*", - "puppeteer-extra": "*" - }, - "peerDependenciesMeta": { - "playwright-extra": { - "optional": true - }, - "puppeteer-extra": { - "optional": true - } - } - }, "node_modules/quansync": { "version": "0.2.11", "resolved": "https://registry.npmjs.org/quansync/-/quansync-0.2.11.tgz", @@ -7599,43 +7200,6 @@ "dev": true, "license": "MIT" }, - "node_modules/rimraf": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", - "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", - "dev": true, - "license": "ISC", - "dependencies": { - "glob": "^7.1.3" - }, - "bin": { - "rimraf": "bin.js" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/rimraf/node_modules/glob": { - "version": "7.2.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", - "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "dev": true, - "license": "ISC", - "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.1.1", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - }, - "engines": { - "node": "*" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/rolldown": { "version": "1.0.0-rc.15", "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.0-rc.15.tgz", @@ -7804,45 +7368,6 @@ "dev": true, "license": "MIT" }, - "node_modules/shallow-clone": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-0.1.2.tgz", - "integrity": "sha512-J1zdXCky5GmNnuauESROVu31MQSnLoYvlyEn6j2Ztk6Q5EHFIhxkMhYcv6vuDzl2XEzoRr856QwzMgWM/TmZgw==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-extendable": "^0.1.1", - "kind-of": "^2.0.1", - "lazy-cache": "^0.2.3", - "mixin-object": "^2.0.1" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/shallow-clone/node_modules/kind-of": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-2.0.1.tgz", - "integrity": "sha512-0u8i1NZ/mg0b+W3MGGw5I7+6Eib2nx72S/QvXa0hYjEkjTknYmEYQJwGu3mLC0BrhtJjtQafTkyRUQ75Kx0LVg==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-buffer": "^1.0.2" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/shallow-clone/node_modules/lazy-cache": { - "version": "0.2.7", - "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-0.2.7.tgz", - "integrity": "sha512-gkX52wvU/R8DVMMt78ATVPFMJqfW8FPz1GZ1sVHBVQHmu/WvhIWE4cE1GBzhJNFicDeYhnwp6Rl35BcAIM3YOQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -7976,21 +7501,6 @@ "npm": ">= 3.0.0" } }, - "node_modules/socks-proxy-agent": { - "version": "8.0.5", - "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", - "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", - "dev": true, - "license": "MIT", - "dependencies": { - "agent-base": "^7.1.2", - "debug": "^4.3.4", - "socks": "^2.8.3" - }, - "engines": { - "node": ">= 14" - } - }, "node_modules/sonic-boom": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.2.0.tgz", @@ -8253,6 +7763,23 @@ "dev": true, "license": "MIT" }, + "node_modules/tar": { + "version": "7.5.15", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.15.tgz", + "integrity": "sha512-dzGK0boVlC4W5QFuQN1EFSl3bIDYsk7Tj40U6eIBnK2k/8ml7TZ5agbI5j5+qnoVcAA+rNtBml8SEiLxZpNqRQ==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/fs-minipass": "^4.0.0", + "chownr": "^3.0.0", + "minipass": "^7.1.2", + "minizlib": "^3.1.0", + "yallist": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/tar-fs": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.1.1.tgz", @@ -9465,6 +8992,16 @@ "node": ">=10" } }, + "node_modules/yallist": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", + "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, "node_modules/yaml": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.7.0.tgz", diff --git a/package.json b/package.json index a357da46..abe52c54 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "pretest": "npm run build:chrome", "test": "node --import ./register.mjs ./node_modules/.bin/mocha test/test-*.js --exclude test/test-sync.js --exit", "test:file": "node --import ./register.mjs ./node_modules/.bin/mocha --exit", + "browser": "npm run build:chrome && openBrowser=true node test/tmpBrowser.js", "format": "prettier --write \"**/*.{js,mjs,json,yaml,yml,html,css,md}\"" }, "repository": { @@ -49,9 +50,8 @@ "ora": "^5.4.1", "postcss": "^8.5.6", "prettier": "^3.8.3", - "puppeteer": "^24.16.0", - "puppeteer-extra": "^3.3.6", - "puppeteer-extra-plugin-stealth": "^2.11.2", + "cloakbrowser": "^0.3.31", + "puppeteer-core": "^24.16.0", "readline-sync": "^1.4.10", "uuid": "^9.0.1", "wxt": "^0.20.25", diff --git a/src/shared/js/utils/urls.js b/src/shared/js/utils/urls.js index f5b70f18..33f92b3a 100644 --- a/src/shared/js/utils/urls.js +++ b/src/shared/js/utils/urls.js @@ -31,50 +31,56 @@ export const isSourceURL = async (url, noStored) => * @returns {string} The id of the paper found. */ export const parseIdFromUrl = async (url, tab = null) => { - if (tab) { - return urlToWebsiteId(url); - } - let idForUrl; + try { + if (tab) { + return urlToWebsiteId(url); + } + let idForUrl; - const hashedUrl = miniHash(url); - const hashedId = state.urlHashToId[hashedUrl]; - if (hashedId) { - return hashedId; - } + const hashedUrl = miniHash(url); + const hashedId = state.urlHashToId[hashedUrl]; + if (hashedId) { + return hashedId; + } - const is = await isPaper(url, true); - const papers = Object.values(cleanPapers(state.papers)); - const ctx = { - papers, - titleHashToIds: state.titleHashToIds, - findPaperForProperty, - }; + const is = await isPaper(url, true); + const papers = Object.values(cleanPapers(state.papers)); + const ctx = { + papers, + titleHashToIds: state.titleHashToIds, + findPaperForProperty, + }; - let matchedSource = false; - for (const name of SOURCE_DISPATCH_ORDER) { - if (is[name]) { - matchedSource = true; - idForUrl = await getSource(name).urlToId(url, ctx); - break; + let matchedSource = false; + for (const name of SOURCE_DISPATCH_ORDER) { + if (is[name]) { + matchedSource = true; + idForUrl = await getSource(name).urlToId(url, ctx); + break; + } } - } - // A matched source returning undefined means "not in storage yet"; let the - // caller (addOrUpdatePaper) take the makePaper path. Only throw when the URL - // was not recognized as any known source. - if (!matchedSource && idForUrl === undefined) { - if (is.localFile) { - idForUrl = is.localFile; - } else if (is.parsedWebsite) { - idForUrl = is.parsedWebsite.id; - } else { - throw new Error( - "`parseIdFromUrl` failed, unknown paper url. Is: " + JSON.stringify(is), - ); + // A matched source returning undefined means "not in storage yet"; let the + // caller (addOrUpdatePaper) take the makePaper path. Only throw when the URL + // was not recognized as any known source. + if (!matchedSource && idForUrl === undefined) { + if (is.localFile) { + idForUrl = is.localFile; + } else if (is.parsedWebsite) { + idForUrl = is.parsedWebsite.id; + } else { + throw new Error( + "`parseIdFromUrl` failed, unknown paper url. Is: " + + JSON.stringify(is), + ); + } } - } - return idForUrl; + return idForUrl; + } catch (err) { + console.error("Error in parseIdFromUrl:", err); + return null; + } }; export const isArxivAbstractUrl = (url) => url.startsWith("https://arxiv.org/abs/"); diff --git a/system-architecture.md b/system-architecture.md index e9ae501c..5638707d 100644 --- a/system-architecture.md +++ b/system-architecture.md @@ -332,9 +332,11 @@ the `downloads` API), `functions.js` (49 misc helpers incl. `miniHash`, - **Runtime libs:** jQuery 4, Tom Select 2 (tag inputs), `@octokit/request` (Gist). - **Language:** plain ES modules (`"type": "module"`), JSDoc-typed, no TypeScript; `jsconfig.json` + `@pm`/`@pmu` path aliases. -- **Tooling:** Prettier; Mocha + jsdom + Puppeteer (stealth) for the - `test/test-*.js` suite (unit + headless-browser integration). `npm test` builds - Chrome first (`pretest`) then runs Mocha. +- **Tooling:** Prettier; Mocha + jsdom + [cloakbrowser](https://github.com/CloakHQ/cloakbrowser) + (source-level stealth Chromium, Puppeteer-core API) for the `test/test-*.js` + suite (unit + headless-browser integration). `npm test` builds Chrome first + (`pretest`) then runs Mocha. The cloakbrowser binary (~200MB) auto-downloads + to `~/.cloakbrowser` on first use; CI caches it via `actions/cache`. - **External APIs (no project backend):** arXiv, OpenReview, GitHub Gist, CrossRef, DBLP, Semantic Scholar, Unpaywall, Google Scholar, (historically PapersWithCode — its API is currently stubbed out in `background.js`). diff --git a/test/browser.js b/test/browser.js index 4fce6021..55192be3 100644 --- a/test/browser.js +++ b/test/browser.js @@ -1,26 +1,20 @@ import { expect } from "expect"; -import puppeteer from "puppeteer-extra"; -import StealthPlugin from "puppeteer-extra-plugin-stealth"; +import { launch } from "cloakbrowser/puppeteer"; import { sleep, root } from "./utilsForTests.js"; import fs from "fs"; -puppeteer.use(StealthPlugin()); - export const makeBrowser = async (headless = false, windowSize = "1200,900") => { - const browser = await puppeteer.launch({ + const browser = await launch({ headless, - ignoreHTTPSErrors: true, - ignoreDefaultArgs: ["--disable-extensions"], + extensionPaths: [`${root}/dist/chrome-mv3`], args: [ - `--load-extension=${root}/dist/chrome-mv3`, `--window-size=${windowSize}`, "--no-sandbox", "--disable-setuid-sandbox", - "--disable-web-security", "--disable-dev-shm-usage", - "--disable-gpu", ], + launchOptions: { acceptInsecureCerts: true }, }); return browser; }; diff --git a/test/citations-scraper.js b/test/citations-scraper.js index 9a4c1a99..9972f036 100644 --- a/test/citations-scraper.js +++ b/test/citations-scraper.js @@ -1,8 +1,5 @@ import Heap from "heap"; -import puppeteer from "puppeteer-extra"; -import StealthPlugin from "puppeteer-extra-plugin-stealth"; - -puppeteer.use(StealthPlugin()); +import { launch } from "cloakbrowser/puppeteer"; import fs from "fs"; import { glob } from "glob"; @@ -110,7 +107,7 @@ const getCoauthors = (page, max = 4) => { nextId.push({ id: knownIds.pop(), order: 0, level: 0 }); knownIds = new Set(knownIds ?? []); - const browser = await puppeteer.launch({ headless: false }); + const browser = await launch({ headless: false }); const page = await browser.newPage(); await page.setViewport({ width: 1200, height: 900 }); console.log("knownIds: ", knownIds); From 525fda93db2fabe666c14118fa90b7aa647614eb Mon Sep 17 00:00:00 2001 From: vict0rsch Date: Sat, 30 May 2026 23:57:35 +0200 Subject: [PATCH 02/19] refactor: fix agentic infra --- ...new-features.md => grill-with-docs-for-all-changes.md} | 2 +- .claude/rules/mirror-providers.md | 8 ++++++++ ...w-features.mdc => grill-with-docs-for-all-changes.mdc} | 4 ++-- .cursor/rules/keep-system-architecture-up-to-date.mdc | 5 +++++ .cursor/rules/mirror-providers.mdc | 6 ++++++ AGENTS.md | 5 +---- CLAUDE.md | 1 + 7 files changed, 24 insertions(+), 7 deletions(-) rename .claude/rules/{grill-with-docs-for-new-features.md => grill-with-docs-for-all-changes.md} (76%) rename .cursor/rules/{grill-with-docs-for-new-features.mdc => grill-with-docs-for-all-changes.mdc} (69%) create mode 100644 CLAUDE.md diff --git a/.claude/rules/grill-with-docs-for-new-features.md b/.claude/rules/grill-with-docs-for-all-changes.md similarity index 76% rename from .claude/rules/grill-with-docs-for-new-features.md rename to .claude/rules/grill-with-docs-for-all-changes.md index 8b9775c9..8e67b8db 100644 --- a/.claude/rules/grill-with-docs-for-new-features.md +++ b/.claude/rules/grill-with-docs-for-all-changes.md @@ -1,6 +1,6 @@ # Use /grill-with-docs For Development -When a user asks to develop this codebase, use the `/grill-with-docs` skill before implementation. +When a user asks to change anything in this codebase, use the `/grill-with-docs` skill before implementation. ## Required behavior diff --git a/.claude/rules/mirror-providers.md b/.claude/rules/mirror-providers.md index a289117e..be4f6453 100644 --- a/.claude/rules/mirror-providers.md +++ b/.claude/rules/mirror-providers.md @@ -1,3 +1,11 @@ +--- +paths: + - ".cursor/rules/**" + - ".cursor/skills/**" + - ".claude/rules/**" + - ".claude/skills/**" +--- + # Mirror Providers Rule ## Policy diff --git a/.cursor/rules/grill-with-docs-for-new-features.mdc b/.cursor/rules/grill-with-docs-for-all-changes.mdc similarity index 69% rename from .cursor/rules/grill-with-docs-for-new-features.mdc rename to .cursor/rules/grill-with-docs-for-all-changes.mdc index c6d92e36..a7a72181 100644 --- a/.cursor/rules/grill-with-docs-for-new-features.mdc +++ b/.cursor/rules/grill-with-docs-for-all-changes.mdc @@ -1,11 +1,11 @@ --- -description: Use grill-with-docs for development work +description: Use grill-with-docs for any development work alwaysApply: true --- # Use /grill-with-docs For Development -When a user asks to develop this codebase, use the `/grill-with-docs` skill before implementation. +When a user asks to change anything in this codebase, use the `/grill-with-docs` skill before implementation. ## Required behavior diff --git a/.cursor/rules/keep-system-architecture-up-to-date.mdc b/.cursor/rules/keep-system-architecture-up-to-date.mdc index 4244659f..279b80ff 100644 --- a/.cursor/rules/keep-system-architecture-up-to-date.mdc +++ b/.cursor/rules/keep-system-architecture-up-to-date.mdc @@ -1,3 +1,8 @@ +--- +description: When a change affects system architecture (components, boundaries, data flow, key abstractions, integrations, or deployment topology), update system-architecture.md in the same task +alwaysApply: false +--- + # Keep System Architecture Doc Up To Date ## Policy diff --git a/.cursor/rules/mirror-providers.mdc b/.cursor/rules/mirror-providers.mdc index 3b67f533..fc0f27c9 100644 --- a/.cursor/rules/mirror-providers.mdc +++ b/.cursor/rules/mirror-providers.mdc @@ -1,3 +1,9 @@ +--- +description: Keep .cursor and .claude skills/rules mirrored when adding, editing, renaming, or deleting any rule or skill +globs: .cursor/rules/**,.cursor/skills/**,.claude/rules/**,.claude/skills/** +alwaysApply: false +--- + # Mirror Providers Rule ## Policy diff --git a/AGENTS.md b/AGENTS.md index fba70d90..cbd2479d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -43,7 +43,7 @@ The test: Every changed line should trace directly to the user's request. **Every plan must be clear, exhaustive, imperative, and verifiable.** -When writing a plan: +Hard rules when writing a plan so a small model can follow literally without guessing intent: - Use numbered imperative steps (`Do X`, `Then do Y`), never vague goals. - Make the plan exhaustive for the requested scope: no hidden steps, no implicit work. - State assumptions and prerequisites explicitly before execution steps. @@ -51,9 +51,6 @@ When writing a plan: - Define concrete completion criteria so an agent can truthfully decide "done" vs "not done." - Use language that a small model can follow literally without guessing intent. -Hard rule: -- If a small-capability agent could not implement the plan truthfully from the plan text alone, the plan is invalid and must be rewritten. - # 5. Agent Entry Point (Shared Across Providers) Provider-specific rule files are the source of truth for cross-provider mirroring: diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..43c994c2 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md From 4f54f7df1268e5c62068a43dc2640c73aa952600 Mon Sep 17 00:00:00 2001 From: vict0rsch Date: Sun, 31 May 2026 00:28:13 +0200 Subject: [PATCH 03/19] fix: chemrxiv parsing --- src/shared/js/utils/sources/chemrxiv.js | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/shared/js/utils/sources/chemrxiv.js b/src/shared/js/utils/sources/chemrxiv.js index f20e5a1f..c77fcf33 100644 --- a/src/shared/js/utils/sources/chemrxiv.js +++ b/src/shared/js/utils/sources/chemrxiv.js @@ -1,5 +1,5 @@ import { BasePaperSource } from "./base.js"; -import { miniHash, noParamUrl, isPdfUrl } from "@pmu/functions.js"; +import { miniHash, noParamUrl } from "@pmu/functions.js"; import { extractDataFromDCMetaTags, fetchDom } from "@pmu/parsers.js"; export class ChemrxivSource extends BasePaperSource { @@ -14,7 +14,7 @@ export class ChemrxivSource extends BasePaperSource { ]; static async urlToId(url, ctx) { - let chemRxivId = isPdfUrl(url) + let chemRxivId = url.includes("/item/") ? url.split("/item/")[1].split("/")[0] : noParamUrl(url).split("/").last(); return ctx.findPaperForProperty(ctx.papers, "chemrxiv", miniHash(chemRxivId)); @@ -23,12 +23,15 @@ export class ChemrxivSource extends BasePaperSource { static async parse(url) { let chemRxivId; let absUrl = url; - if (isPdfUrl(url)) { + if (url.includes("/item/")) { chemRxivId = url.split("/item/")[1].split("/")[0]; absUrl = "https://chemrxiv.org/engage/chemrxiv/article-details/" + chemRxivId; } else { chemRxivId = noParamUrl(url).split("/").last(); + if (url.includes("/doi/pdf/")) { + absUrl = url.replace("/doi/pdf/", "/doi/full/"); + } } const dom = await fetchDom(absUrl); const { @@ -49,9 +52,12 @@ export class ChemrxivSource extends BasePaperSource { static toAbs(paper) { const pdf = paper.pdfLink; - return `https://chemrxiv.org/engage/chemrxiv/article-details/${ - pdf.split("/item/")[1].split("/")[0] - }`; + if (pdf.includes("/item/")) { + return `https://chemrxiv.org/engage/chemrxiv/article-details/${ + pdf.split("/item/")[1].split("/")[0] + }`; + } + return pdf.replace("/doi/pdf/", "/doi/full/"); } static toPDF(paper) { From b75b595362f4714a8ec175f8fe12c22a5affdf4f Mon Sep 17 00:00:00 2001 From: vict0rsch Date: Sun, 31 May 2026 00:28:23 +0200 Subject: [PATCH 04/19] tests: handle redirections --- test/browser.js | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/test/browser.js b/test/browser.js index 55192be3..a1eb8374 100644 --- a/test/browser.js +++ b/test/browser.js @@ -51,7 +51,23 @@ export const visitPaperPage = async (browser, target, options = {}) => { const opts = { ...defaults, ...options }; const p = opts.page || (await browser.pages())[0] || (await browser.newPage()); - await p.goto(target); + // A leftover page from the previous source (e.g. a Cloudflare "Just a moment..." + // challenge) can perform a client-side navigation that aborts this goto with + // net::ERR_ABORTED. Retry a few times so the competing navigation can settle. + const maxNavAttempts = 3; + let gotoError = null; + for (let attempt = 1; attempt <= maxNavAttempts; attempt++) { + try { + await p.goto(target); + gotoError = null; + break; + } catch (e) { + gotoError = e; + if (!String(e.message).includes("net::ERR_ABORTED")) break; + if (attempt < maxNavAttempts) await sleep(1000); + } + } + if (gotoError) throw gotoError; const paperIsStored = new Promise((resolve, reject) => { let screenshotTimeout; p.waitForSelector("meta[name='pm-complete-secret-html']") From 3fb42e48247352690183d815f102cd791b945e97 Mon Sep 17 00:00:00 2001 From: vict0rsch Date: Sun, 31 May 2026 13:50:47 +0200 Subject: [PATCH 05/19] fix: resolve sciencedirect PII from signed PDF redirect URLs Add a sciencedirectPii helper that extracts the article PII from abstract, stable /pdfft, and signed pdf.sciencedirectassets.com redirect URLs, and emit a stable sciencedirect.com PDF link. Only the PII is read from the signed URL; its security tokens are never stored. Co-authored-by: Cursor --- src/shared/js/utils/sources/sciencedirect.js | 35 +++++++++++++++++--- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/src/shared/js/utils/sources/sciencedirect.js b/src/shared/js/utils/sources/sciencedirect.js index d2724a0f..462fb638 100644 --- a/src/shared/js/utils/sources/sciencedirect.js +++ b/src/shared/js/utils/sources/sciencedirect.js @@ -2,6 +2,31 @@ import { BasePaperSource } from "./base.js"; import { miniHash } from "@pmu/functions.js"; import { fetchBibtexToPaper } from "@pmu/parsers.js"; +/** + * Extract the article PII from any known ScienceDirect URL shape: + * - abstract page: .../science/article/pii/{pii} + * - stable PDF link: .../science/article/pii/{pii}/pdfft + * - signed redirect: pdf.sciencedirectassets.com/.../1-s2.0-{pii}/main.pdf?...&pii={pii} + * The signed redirect (the actual S3 asset) carries the PII as a `pii` query + * param, with the `1-s2.0-{pii}` path segment as a fallback. We never store the + * signed URL or its security tokens — only the PII is read from it. + * @param {string} url + * @returns {string|undefined} + */ +const sciencedirectPii = (url) => { + if (url.includes("/pii/")) { + return url.split("/pii/")[1].split("/")[0].split("#")[0].split("?")[0]; + } + const param = new URL(url).searchParams.get("pii"); + if (param) return param; + const segment = url + .split("?")[0] + .split("/") + .reverse() + .find((s) => /^1-s2\.0-S/.test(s)); + return segment ? segment.replace("1-s2.0-", "") : undefined; +}; + export class SciencedirectSource extends BasePaperSource { static name = "sciencedirect"; static displayName = "ScienceDirect"; @@ -9,15 +34,16 @@ export class SciencedirectSource extends BasePaperSource { "sciencedirect.com/science/article/pii/", "sciencedirect.com/science/article/abs/pii/", "reader.elsevier.com/reader/sd/pii/", + "pdf.sciencedirectassets.com/", ]; static async urlToId(url, ctx) { - const pii = url.split("/pii/")[1].split("/")[0].split("#")[0].split("?")[0]; + const pii = sciencedirectPii(url); return ctx.findPaperForProperty(ctx.papers, "sciencedirect", miniHash(pii)); } static async parse(url) { - const pii = url.split("/pii/")[1].split("/")[0].split("#")[0].split("?")[0]; + const pii = sciencedirectPii(url); const data = await fetchBibtexToPaper({ url: `https://www.sciencedirect.com/sdfe/arp/cite?pii=${pii}&format=text%2Fx-bibtex&withabstract=false`, }); @@ -25,7 +51,7 @@ export class SciencedirectSource extends BasePaperSource { const note = `Published @ ${journal} (${year})`; const id = `ScienceDirect-${year}_${miniHash(pii)}`; const venue = journal ?? "Science Direct"; - const pdfLink = `https://reader.elsevier.com/reader/sd/pii/${pii}`; + const pdfLink = `https://www.sciencedirect.com/science/article/pii/${pii}/pdfft`; return { author, @@ -41,8 +67,7 @@ export class SciencedirectSource extends BasePaperSource { } static toAbs(paper) { - const pdf = paper.pdfLink; - const pii = pdf.split("/pii/")[1].split("/")[0].split("#")[0].split("?")[0]; + const pii = sciencedirectPii(paper.pdfLink); return `https://www.sciencedirect.com/science/article/pii/${pii}`; } From efe4c390cd1e639c5efd368a3b0591c77cd94015 Mon Sep 17 00:00:00 2001 From: vict0rsch Date: Sun, 31 May 2026 13:50:53 +0200 Subject: [PATCH 06/19] fix: guard cell parsing against missing journal data Co-authored-by: Cursor --- src/shared/js/utils/sources/cell.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/shared/js/utils/sources/cell.js b/src/shared/js/utils/sources/cell.js index 7a0a2c27..3a187a9d 100644 --- a/src/shared/js/utils/sources/cell.js +++ b/src/shared/js/utils/sources/cell.js @@ -8,7 +8,7 @@ const findCellPii = async (url) => { const isPdfExtended = url.toLowerCase().includes("pdfextended"); let pii; if (isPdf || isPdfExtended) { - const cellData = state.cellJournalData; + const cellData = state.cellJournalData ?? {}; pii = isPdf ? new URL(url).searchParams.get("pii") : url.split("/").last(); const issn = pii.match(/\d{4}-\d{3}[0-9X]/g)[0]; let venue; From 0ac47e7c62a17bac0f7efe13b774f9da9e4ca545 Mon Sep 17 00:00:00 2001 From: vict0rsch Date: Sun, 31 May 2026 13:50:53 +0200 Subject: [PATCH 07/19] fix: build pmlr abstract URL without duplicating proceedings key Co-authored-by: Cursor --- src/shared/js/utils/sources/pmlr.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/shared/js/utils/sources/pmlr.js b/src/shared/js/utils/sources/pmlr.js index 514df961..ccd356cf 100644 --- a/src/shared/js/utils/sources/pmlr.js +++ b/src/shared/js/utils/sources/pmlr.js @@ -44,7 +44,7 @@ export class PMLRSource extends BasePaperSource { const absURL = url.includes(".html") ? url - : url.split("/").slice(0, -1).join("/") + `/${key}.html`; + : url.split("/").slice(0, -1).join("/") + ".html"; const pdfLink = absURL.replace(".html", "") + `/${key}.pdf`; From c07ef41d8c5f81297ab55fc9196c5c07b9d9012a Mon Sep 17 00:00:00 2001 From: vict0rsch Date: Sun, 31 May 2026 13:51:02 +0200 Subject: [PATCH 08/19] fix: return null for unrecognized paper URLs instead of throwing Co-authored-by: Cursor --- src/shared/js/utils/urls.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/shared/js/utils/urls.js b/src/shared/js/utils/urls.js index 33f92b3a..bae4b75d 100644 --- a/src/shared/js/utils/urls.js +++ b/src/shared/js/utils/urls.js @@ -69,10 +69,9 @@ export const parseIdFromUrl = async (url, tab = null) => { } else if (is.parsedWebsite) { idForUrl = is.parsedWebsite.id; } else { - throw new Error( - "`parseIdFromUrl` failed, unknown paper url. Is: " + - JSON.stringify(is), - ); + // Not a recognized paper source: this is an expected outcome + // (e.g. arbitrary URLs), so return null rather than throwing. + return null; } } From 45743c276708935f9d317fe4defe26384ff60928 Mon Sep 17 00:00:00 2001 From: vict0rsch Date: Sun, 31 May 2026 13:51:02 +0200 Subject: [PATCH 09/19] test: refresh source URL fixtures and bot-prevention flags Co-authored-by: Cursor --- test/data/urls.json | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/test/data/urls.json b/test/data/urls.json index b381baf7..f2e3b303 100644 --- a/test/data/urls.json +++ b/test/data/urls.json @@ -50,10 +50,7 @@ ], "cell": [ "https://www.cell.com/cell/fulltext/S0092-8674(25)01089-X", - "https://www.cell.com/action/showPdf?pii=S0092-8674%2825%2901089-X", - { - "botPrevention": true - } + "https://www.cell.com/action/showPdf?pii=S0092-8674%2825%2901089-X" ], "chemrxiv": [ "https://chemrxiv.org/engage/chemrxiv/article-details/65957d349138d231611ad8f7", @@ -105,8 +102,7 @@ "https://iopscience.iop.org/article/10.1149/2.1051908jes", "https://iopscience.iop.org/article/10.1149/2.1051908jes/pdf", { - "venue": "Journal of The Electrochemical Society", - "botPrevention": true + "venue": "Journal of The Electrochemical Society" } ], "jmlr": [ @@ -152,7 +148,8 @@ "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7249434", "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7249434/pdf/12961_2020_Article_544.pdf", { - "venue": "Health research policy and systems" + "venue": "Health research policy and systems", + "botPrevention": true } ], "pmlr": [ @@ -179,10 +176,7 @@ ], "sciencedirect": [ "https://www.sciencedirect.com/science/article/pii/S2589721721000349", - "https://reader.elsevier.com/reader/sd/pii/S2589721721000349", - { - "botPrevention": true - } + "https://www.sciencedirect.com/science/article/pii/S2589721721000349/pdfft" ], "springer": [ "https://link.springer.com/article/10.1007/s41095-022-0271-y", From 42f67b5171a59381fd952e77c16dba8f68613a8e Mon Sep 17 00:00:00 2001 From: vict0rsch Date: Sun, 31 May 2026 13:51:02 +0200 Subject: [PATCH 10/19] test: add sciencedirect redirect tests and cell journal fixture Co-authored-by: Cursor --- test/test-utils.js | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test/test-utils.js b/test/test-utils.js index 2b1b5e26..1c6b6e16 100644 --- a/test/test-utils.js +++ b/test/test-utils.js @@ -6,6 +6,9 @@ import { loadPaperMemoryUtils, range, readJSON } from "./utilsForTests.js"; await loadPaperMemoryUtils(); // create fake `document`, parseUrl() will need it for instance global.document = new JSDOM(``).window.document; +// initState() (which loads the Cell journal data) does not run in unit tests, +// so populate it directly from the same source the extension fetches at runtime. +PMUtils.config.state.cellJournalData = readJSON("./public/data/cell.json"); describe("Bibtex parser", function () { var bdata = readJSON("./test/data/bibtexs.json"); @@ -165,4 +168,24 @@ describe("paper.js", () => { }); } }); + + describe("#sciencedirect signed PDF redirect", () => { + // Sanitised stand-in for the pdf.sciencedirectassets.com redirect: the + // real one carries an AWS X-Amz-Security-Token and expires in minutes, so + // we only keep the PII-bearing parts and drop every credential param. + const signedUrl = + "https://pdf.sciencedirectassets.com/271664/1-s2.0-S0001457526X20048/1-s2.0-S0001457526002083/main.pdf?pii=S0001457526002083&type=client"; + + it("is recognised as a sciencedirect page", async () => { + const isp = await PMUtils.paper.isPaper(signedUrl); + expect(isp.sciencedirect).toBe(true); + }); + + it("resolves to the abstract page from its PII", () => { + const paper = { source: "sciencedirect", pdfLink: signedUrl }; + expect(PMUtils.paper.paperToAbs(paper)).toEqual( + "https://www.sciencedirect.com/science/article/pii/S0001457526002083", + ); + }); + }); }); From 054088739ff04b0713ef5ee839e3818b2e1fba3f Mon Sep 17 00:00:00 2001 From: vict0rsch Date: Sun, 31 May 2026 13:51:13 +0200 Subject: [PATCH 11/19] test: surface page-visit timeouts and improve visit logging Reject (instead of silently resolving) when the completion meta never appears, raise the wait to 15s, and add indentation/source context to the storage and duplicates test logs. Co-authored-by: Cursor --- test/browser.js | 23 ++++++++++++++--------- test/test-duplicates.js | 3 ++- test/test-storage.js | 4 +++- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/test/browser.js b/test/browser.js index a1eb8374..e6af6b18 100644 --- a/test/browser.js +++ b/test/browser.js @@ -1,7 +1,7 @@ import { expect } from "expect"; import { launch } from "cloakbrowser/puppeteer"; -import { sleep, root } from "./utilsForTests.js"; +import { sleep, root, indent } from "./utilsForTests.js"; import fs from "fs"; export const makeBrowser = async (headless = false, windowSize = "1200,900") => { @@ -47,6 +47,7 @@ export const visitPaperPage = async (browser, target, options = {}) => { timeout: null, keepOpen: false, dontScreenshot: false, + indents: 0, }; const opts = { ...defaults, ...options }; @@ -83,28 +84,32 @@ export const visitPaperPage = async (browser, target, options = {}) => { return document.querySelector("meta[name='pm-complete-secret-html']"); }); if (!element && !opts.dontScreenshot) { - console.log(`No element found: taking a screenshot`); if (!fs.existsSync(`${root}/tmp`)) { - console.log(`Creating tmp directory in ${root}/tmp`); + console.log( + `${indent(opts.indents)}Creating tmp directory in ${root}/tmp`, + ); fs.mkdirSync(`${root}/tmp`); } let screenshotName = `screenshot_${Date.now()}_${target .replaceAll("https://", "") - .replaceAll("/", "__")}.jpg`; - screenshotName = screenshotName + .replaceAll("/", "__") .replace(/[^a-zA-Z0-9\-_\.]/g, "") - .slice(0, 100); + .slice(0, 50)}.jpg`; const screenshotPath = `${root}/tmp/${screenshotName}`; await p.screenshot({ path: screenshotPath, fullPage: true, }); - console.log(`Screenshot taken and saved to ${screenshotPath}\n\n`); + reject( + new Error( + `Timeout for ${target} -> Screenshot taken and saved to ${screenshotPath}`, + ), + ); } resolve(); - }, 5000); + }, 15000); }); - opts.timeout > 0 && (await paperIsStored); + await paperIsStored; opts.timeout > 0 && (await sleep(opts.timeout)); !opts.keepOpen && (await p.close()); }; diff --git a/test/test-duplicates.js b/test/test-duplicates.js index b668f18e..6c8f50a3 100644 --- a/test/test-duplicates.js +++ b/test/test-duplicates.js @@ -157,11 +157,12 @@ describe("Paper de-duplication", function () { // visit the paper urls for (const dup of duplicates) { console.log( - `${indent(2)}(${n + 1}/${nUrls}) visiting ${dup.url}`, + `${indent(2)}(${n + 1}/${nUrls}) visiting ${dup.name || "pre-duplicate"} (${dup.url.slice(0, 30)}[...])`, ); await visitPaperPage(browser, dup.url, { keepOpen, timeout: pageTimeout, + indents: 3, }); n += 1; } diff --git a/test/test-storage.js b/test/test-storage.js index b24d00a1..a4897e9e 100644 --- a/test/test-storage.js +++ b/test/test-storage.js @@ -167,6 +167,7 @@ describe("Test paper detection and storage", function () { for (const sourceOrderIdx of sourceOrder) { for (const [targetIdx, targets] of Object.values(urls).entries()) { + const currentSource = Object.keys(urls)[targetIdx]; // for each target url (abstract, pdf), visit the url // and wait a little for it to load const isPDF = sourceOrderIdx === 1; @@ -193,12 +194,13 @@ describe("Test paper detection and storage", function () { nUrls + 1; console.log( - `${indent(2)}(${n}/${nUrls * 2}) Going to: ${target}`, + `${indent(2)}(${n}/${nUrls * 2}) Going to: ${currentSource} (${target.slice(0, 30)}[...])`, ); await visitPaperPage(browser, target, { timeout: pageTimeout, keepOpen, + indents: 3, }); } } From 99295080fbb0906f546fd66010de3c4054bbf38b Mon Sep 17 00:00:00 2001 From: vict0rsch Date: Sun, 31 May 2026 13:51:13 +0200 Subject: [PATCH 12/19] ci: run duplicates test in storage workflow with retry Co-authored-by: Cursor --- .github/workflows/_test-matrix.yml | 2 -- .github/workflows/test.yml | 8 ++++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_test-matrix.yml b/.github/workflows/_test-matrix.yml index 28d44e19..9865cc57 100644 --- a/.github/workflows/_test-matrix.yml +++ b/.github/workflows/_test-matrix.yml @@ -28,8 +28,6 @@ jobs: files: test/test-menu.js - name: Sync files: test/test-sync.js - - name: Duplicates - files: test/test-duplicates.js - name: All test scripts are listed in a workflow files: test/test-meta.js diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2d2c8238..8d009e72 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,6 +35,14 @@ jobs: retry_on: error command: npm run test:file test/test-storage.js + - uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 + name: Run Storage Test + with: + timeout_minutes: 20 + max_attempts: 2 + retry_on: error + command: npm run test:file test/test-duplicates.js + - uses: actions/upload-artifact@v4 if: always() # Upload screenshots even on failure for debugging with: From 7c79025f9ea7c52f98d588b2cc82b87c6f882f9b Mon Sep 17 00:00:00 2001 From: vict0rsch Date: Sun, 31 May 2026 13:51:24 +0200 Subject: [PATCH 13/19] docs: add ingestion glossary and source-debugging skill Add CONTEXT.md fixing the detect/parse/store vocabulary and a mirrored debug-source-parsing skill (.cursor + .claude) for diagnosing source failures. Co-authored-by: Cursor --- .claude/skills/debug-source-parsing/SKILL.md | 112 +++++++++++++++++++ .cursor/skills/debug-source-parsing/SKILL.md | 112 +++++++++++++++++++ CONTEXT.md | 57 ++++++++++ 3 files changed, 281 insertions(+) create mode 100644 .claude/skills/debug-source-parsing/SKILL.md create mode 100644 .cursor/skills/debug-source-parsing/SKILL.md create mode 100644 CONTEXT.md diff --git a/.claude/skills/debug-source-parsing/SKILL.md b/.claude/skills/debug-source-parsing/SKILL.md new file mode 100644 index 00000000..2bff8d99 --- /dev/null +++ b/.claude/skills/debug-source-parsing/SKILL.md @@ -0,0 +1,112 @@ +--- +name: debug-source-parsing +description: Diagnose why a paper source fails to detect, parse, or store in PaperMemory by reproducing the failure inside cloakbrowser, isolating which pipeline stage broke, fixing the source, and verifying with the targeted storage test. Use when a source's storage test times out, a paper is not stored, or `parse` returns wrong/undefined fields. +--- + +# Debug Source Parsing + +Workflow for debugging a single PaperMemory source (`src/shared/js/utils/sources/.js`) when its detection, parsing, or storage fails. The driver is a real browser launched via cloakbrowser, because most sources block plain `node` `fetch` (Cloudflare, bot prevention). + +Vocabulary (detect / parse / store, abstract URL, pdf URL, parsing order, completion meta) is defined in the repo-root `CONTEXT.md`. + +## Background you need + +- A visited page becomes a stored paper through a strict three-stage pipeline; each stage runs only if the previous succeeded: + 1. **detect** — `isSourceURL(url)` / `isPaper(url)` decide the URL belongs to a source. If this fails, `parse` never runs. + 2. **parse** — the source's `parse(url)` builds metadata, usually via `fetchDom(absURL)`. + 3. **store** — `addOrUpdatePaper({ store: true })` persists it, then the content script injects the **completion meta** `meta[name='pm-complete-secret-html']`. +- Source storage tests live in `test/test-storage.js`. They visit each source's **abstract** and **pdf** URLs (from `test/data/urls.json`) and assert the paper was stored. `test/browser.js` → `visitPaperPage` waits ~15s for the completion meta; on timeout it screenshots the page to `tmp/` and throws `Timeout for -> Screenshot ...`. +- **A timeout only means "not stored" — it does not pinpoint the stage.** The failure could be detect, parse, or store. Don't assume `parse`. +- Each source is visited in **both parsing orders**: `abs;pdf` (abstract first) and `pdf;abs` (pdf first). +- `fetchDom` (`src/shared/js/utils/parsers.js`) returns an **empty DOM on a non-OK response** (no throw). The throw then happens later on `dom.getElementById(...).innerText`, so a wrong/blocked URL surfaces as a timeout, not a fetch error. +- Always `npm run build:chrome` before running browser tests — they load the built `dist/chrome-mv3`. + +## Workflow + +``` +- [ ] 1. Reproduce: run the targeted storage test for the one source +- [ ] 2. Read clues: order(s) failed + tmp/ screenshot + console 404s + left-open page +- [ ] 3. Hypothesize: map clue -> failing stage/cause +- [ ] 4. Confirm in cloakbrowser: reproduce the offending fetch/DOM step +- [ ] 5. Fix the source (minimal, surgical) +- [ ] 6. Verify: re-run the targeted test in both orders, then clean up +``` + +### 1. Reproduce + +Scope the run to the failing source so it takes seconds, not minutes: + +```bash +onlySources= npm run test:file -- test/test-storage.js +``` + +Note which order fails (`abs;pdf` vs `pdf;abs`) and the exact URL in the timeout message. The test runs with `keepOpen`, so the failing page stays open for inspection. + +### 2. Read clues + +- **Which order(s) failed** — one order vs both is the strongest signal (see step 3). +- **The screenshot** referenced in the timeout (`tmp/screenshot_*.jpg`): a normally rendered page, a bot-wall/challenge, or a blank page are three different stories. +- **Console output**: a `Failed to load resource: ... status of 404` means a fetch inside `parse` hit a wrong URL. +- **The left-open page**: open its DevTools and inspect how far the pipeline got — was the source even detected, did `parse` throw, did a `fetch` fail. Inspect extension state via `PMDebug` (see `test/browser.js` helpers for the shape). + +### 3. Hypothesize (clue -> cause) + +| Clue | Likely stage & cause | +| --- | --- | +| Fails in **one** order only | **parse** — the URL conversion for the *other* page is wrong. Inspect `absURL` / `pdfLink` / `toAbs` / `toPDF`. | +| Fails in **both** orders, page **rendered fine** | **parse** — a selector/field the publisher changed, so `dom.getElementById(...)`/`getElementsByTagName(...)` returns null and throws (or returns a wrong/`undefined` field). | +| Fails in **both** orders, page is a **challenge/blank** | **detect or fetch blocked** — bot-wall. Confirm with a visible browser; the source may need the `botPrevention` flag in `urls.json`. | +| Stored, but a field is wrong/`undefined` | **parse** — the extraction logic for that field; check against the live DOM. | + +### 4. Confirm in cloakbrowser + +Reproduce the suspect fetch from a real browser page. `parse`/`fetchDom` run in the content script, which shares the page's **origin**, so a `page.evaluate(fetch)` reproduces the same status/CORS behavior. Write a throwaway script and run it with the repo's loader: + +```js +// test/tmp-debug-.js +import { makeBrowser } from "./browser.js"; + +const url = ""; +const candidates = { + current: /* buggy derivation copied from parse() */ "", + fixed: /* proposed derivation */ "", +}; + +const browser = await makeBrowser(true); // headless; pass false to watch +const page = (await browser.pages())[0]; +await page.goto(url).catch(() => {}); + +for (const [label, u] of Object.entries(candidates)) { + const res = await page.evaluate(async (u) => { + const r = await fetch(u); + const text = r.ok ? await r.text() : ""; + const dom = new DOMParser().parseFromString(text, "text/html"); + return { u, status: r.status, ok: r.ok, hasKeyEl: !!dom.getElementById("bibtex") }; + }, u); + console.log(label, res); +} +await browser.close(); +``` + +```bash +node --import ./register.mjs test/tmp-debug-.js +``` + +Adjust `hasKeyEl` to whatever element/field `parse` depends on. Confirm the buggy candidate 404s / lacks the element and the fixed one returns 200 with it. + +### 5. Fix + +Make the **smallest** change to the failing stage — usually a derivation in `parse` (or `toAbs`/`toPDF`) or a single selector. Prefer mirroring an existing correct helper in the same file over inventing new logic. Touch only the broken line(s). + +### 6. Verify and clean up + +```bash +npm run build:chrome && onlySources= npm run test:file -- test/test-storage.js +``` + +All cases must pass in **both** orders. Then delete the throwaway script (`test/tmp-debug-.js`). Leave `tmp/` screenshots alone (test artifacts). + +## Notes + +- Set `headless` to `false` (e.g. `HEADLESS=0` for `test/manual-open-urls-browser.js`, or `makeBrowser(false)`) when you need to watch the page interactively. +- Sources flagged `botPrevention` in `urls.json` are skipped by the automated test — debug those manually with a visible browser. diff --git a/.cursor/skills/debug-source-parsing/SKILL.md b/.cursor/skills/debug-source-parsing/SKILL.md new file mode 100644 index 00000000..2bff8d99 --- /dev/null +++ b/.cursor/skills/debug-source-parsing/SKILL.md @@ -0,0 +1,112 @@ +--- +name: debug-source-parsing +description: Diagnose why a paper source fails to detect, parse, or store in PaperMemory by reproducing the failure inside cloakbrowser, isolating which pipeline stage broke, fixing the source, and verifying with the targeted storage test. Use when a source's storage test times out, a paper is not stored, or `parse` returns wrong/undefined fields. +--- + +# Debug Source Parsing + +Workflow for debugging a single PaperMemory source (`src/shared/js/utils/sources/.js`) when its detection, parsing, or storage fails. The driver is a real browser launched via cloakbrowser, because most sources block plain `node` `fetch` (Cloudflare, bot prevention). + +Vocabulary (detect / parse / store, abstract URL, pdf URL, parsing order, completion meta) is defined in the repo-root `CONTEXT.md`. + +## Background you need + +- A visited page becomes a stored paper through a strict three-stage pipeline; each stage runs only if the previous succeeded: + 1. **detect** — `isSourceURL(url)` / `isPaper(url)` decide the URL belongs to a source. If this fails, `parse` never runs. + 2. **parse** — the source's `parse(url)` builds metadata, usually via `fetchDom(absURL)`. + 3. **store** — `addOrUpdatePaper({ store: true })` persists it, then the content script injects the **completion meta** `meta[name='pm-complete-secret-html']`. +- Source storage tests live in `test/test-storage.js`. They visit each source's **abstract** and **pdf** URLs (from `test/data/urls.json`) and assert the paper was stored. `test/browser.js` → `visitPaperPage` waits ~15s for the completion meta; on timeout it screenshots the page to `tmp/` and throws `Timeout for -> Screenshot ...`. +- **A timeout only means "not stored" — it does not pinpoint the stage.** The failure could be detect, parse, or store. Don't assume `parse`. +- Each source is visited in **both parsing orders**: `abs;pdf` (abstract first) and `pdf;abs` (pdf first). +- `fetchDom` (`src/shared/js/utils/parsers.js`) returns an **empty DOM on a non-OK response** (no throw). The throw then happens later on `dom.getElementById(...).innerText`, so a wrong/blocked URL surfaces as a timeout, not a fetch error. +- Always `npm run build:chrome` before running browser tests — they load the built `dist/chrome-mv3`. + +## Workflow + +``` +- [ ] 1. Reproduce: run the targeted storage test for the one source +- [ ] 2. Read clues: order(s) failed + tmp/ screenshot + console 404s + left-open page +- [ ] 3. Hypothesize: map clue -> failing stage/cause +- [ ] 4. Confirm in cloakbrowser: reproduce the offending fetch/DOM step +- [ ] 5. Fix the source (minimal, surgical) +- [ ] 6. Verify: re-run the targeted test in both orders, then clean up +``` + +### 1. Reproduce + +Scope the run to the failing source so it takes seconds, not minutes: + +```bash +onlySources= npm run test:file -- test/test-storage.js +``` + +Note which order fails (`abs;pdf` vs `pdf;abs`) and the exact URL in the timeout message. The test runs with `keepOpen`, so the failing page stays open for inspection. + +### 2. Read clues + +- **Which order(s) failed** — one order vs both is the strongest signal (see step 3). +- **The screenshot** referenced in the timeout (`tmp/screenshot_*.jpg`): a normally rendered page, a bot-wall/challenge, or a blank page are three different stories. +- **Console output**: a `Failed to load resource: ... status of 404` means a fetch inside `parse` hit a wrong URL. +- **The left-open page**: open its DevTools and inspect how far the pipeline got — was the source even detected, did `parse` throw, did a `fetch` fail. Inspect extension state via `PMDebug` (see `test/browser.js` helpers for the shape). + +### 3. Hypothesize (clue -> cause) + +| Clue | Likely stage & cause | +| --- | --- | +| Fails in **one** order only | **parse** — the URL conversion for the *other* page is wrong. Inspect `absURL` / `pdfLink` / `toAbs` / `toPDF`. | +| Fails in **both** orders, page **rendered fine** | **parse** — a selector/field the publisher changed, so `dom.getElementById(...)`/`getElementsByTagName(...)` returns null and throws (or returns a wrong/`undefined` field). | +| Fails in **both** orders, page is a **challenge/blank** | **detect or fetch blocked** — bot-wall. Confirm with a visible browser; the source may need the `botPrevention` flag in `urls.json`. | +| Stored, but a field is wrong/`undefined` | **parse** — the extraction logic for that field; check against the live DOM. | + +### 4. Confirm in cloakbrowser + +Reproduce the suspect fetch from a real browser page. `parse`/`fetchDom` run in the content script, which shares the page's **origin**, so a `page.evaluate(fetch)` reproduces the same status/CORS behavior. Write a throwaway script and run it with the repo's loader: + +```js +// test/tmp-debug-.js +import { makeBrowser } from "./browser.js"; + +const url = ""; +const candidates = { + current: /* buggy derivation copied from parse() */ "", + fixed: /* proposed derivation */ "", +}; + +const browser = await makeBrowser(true); // headless; pass false to watch +const page = (await browser.pages())[0]; +await page.goto(url).catch(() => {}); + +for (const [label, u] of Object.entries(candidates)) { + const res = await page.evaluate(async (u) => { + const r = await fetch(u); + const text = r.ok ? await r.text() : ""; + const dom = new DOMParser().parseFromString(text, "text/html"); + return { u, status: r.status, ok: r.ok, hasKeyEl: !!dom.getElementById("bibtex") }; + }, u); + console.log(label, res); +} +await browser.close(); +``` + +```bash +node --import ./register.mjs test/tmp-debug-.js +``` + +Adjust `hasKeyEl` to whatever element/field `parse` depends on. Confirm the buggy candidate 404s / lacks the element and the fixed one returns 200 with it. + +### 5. Fix + +Make the **smallest** change to the failing stage — usually a derivation in `parse` (or `toAbs`/`toPDF`) or a single selector. Prefer mirroring an existing correct helper in the same file over inventing new logic. Touch only the broken line(s). + +### 6. Verify and clean up + +```bash +npm run build:chrome && onlySources= npm run test:file -- test/test-storage.js +``` + +All cases must pass in **both** orders. Then delete the throwaway script (`test/tmp-debug-.js`). Leave `tmp/` screenshots alone (test artifacts). + +## Notes + +- Set `headless` to `false` (e.g. `HEADLESS=0` for `test/manual-open-urls-browser.js`, or `makeBrowser(false)`) when you need to watch the page interactively. +- Sources flagged `botPrevention` in `urls.json` are skipped by the automated test — debug those manually with a visible browser. diff --git a/CONTEXT.md b/CONTEXT.md new file mode 100644 index 00000000..f018d1f8 --- /dev/null +++ b/CONTEXT.md @@ -0,0 +1,57 @@ +# PaperMemory + +PaperMemory is a browser extension that recognizes academic papers on publisher pages and stores their metadata. This glossary fixes the vocabulary for how a visited page becomes a stored paper. + +## Language + +### Ingestion pipeline + +**Source**: +A supported publisher whose pages PaperMemory understands, implemented as a class in `src/shared/js/utils/sources/`. +_Avoid_: provider, site, publisher (when referring to the code object) + +**Detect**: +Deciding that a visited URL belongs to a **Source** (`isSourceURL` / `isPaper`), which gates whether parsing runs at all. +_Avoid_: match, recognize, identify + +**Parse**: +A **Source**'s `parse(url)` building a paper's metadata, typically by fetching the **Abstract URL**'s DOM or fetching a Bibtex citation file. +_Avoid_: scrape, extract, read + +**Store**: +Persisting the parsed paper (`addOrUpdatePaper` with `store: true`); the final stage, after which the **Completion meta** is injected. +_Avoid_: save, commit, persist (informally) + +### Page shapes & signals + +**Abstract URL**: +A **Source** HTML page describing the paper (metadata, bibtex), as opposed to the PDF file. +_Avoid_: landing page, html page + +**PDF URL**: +The direct link to a paper's PDF file. +_Avoid_: download link, file url + +**Parsing order**: +The sequence in which a paper's two URLs are visited during the storage test — `abs;pdf` (abstract first) or `pdf;abs` (pdf first). +_Avoid_: direction, mode + +**Completion meta**: +The `meta[name='pm-complete-secret-html']` element the content script injects once a paper is **Store**d, used by the test harness to detect success. +_Avoid_: done flag, marker + +## Relationships + +- A **Source** owns the logic to **Detect**, **Parse**, and convert between an **Abstract URL** and a **PDF URL** (`toAbs` / `toPDF`). +- **Detect** → **Parse** → **Store** is a strict pipeline: each stage only runs if the previous succeeded. +- A **Store** triggers exactly one **Completion meta**; its absence (a test timeout) means the pipeline stopped at **Detect**, **Parse**, or **Store** — not necessarily **Parse**. +- The storage test visits both **Abstract URL** and **PDF URL** in each **Parsing order**; an order-dependent failure implicates the **Abstract URL**↔**PDF URL** conversion. + +## Example dialogue + +> **Dev:** "The PMLR storage test times out — is `parse` broken?" +> **Maintainer:** "A timeout only tells you the **Completion meta** never appeared, i.e. the paper wasn't **Store**d. It could have failed to **Detect** or **Parse**. Here it failed only in `pdf;abs` **Parsing order**, so `parse` derived a bad **Abstract URL** from the **PDF URL**." + +## Flagged ambiguities + +- "parse or detect" was used as if interchangeable — resolved: **Detect**, **Parse**, and **Store** are distinct pipeline stages, and a storage-test timeout does not pinpoint which one failed. From db50d371fb35a4e26b81344710c0bef34319629b Mon Sep 17 00:00:00 2001 From: vict0rsch Date: Sun, 31 May 2026 13:51:24 +0200 Subject: [PATCH 14/19] test: add cloakbrowser-driven manual url opener Co-authored-by: Cursor --- test/manual-open-urls-browser.js | 235 +++++++++++++++++++++++++++++++ 1 file changed, 235 insertions(+) create mode 100644 test/manual-open-urls-browser.js diff --git a/test/manual-open-urls-browser.js b/test/manual-open-urls-browser.js new file mode 100644 index 00000000..36829947 --- /dev/null +++ b/test/manual-open-urls-browser.js @@ -0,0 +1,235 @@ +// ============================================================================ +// MANUAL TEST (cloakbrowser): Reset memory, open every urls.json source so +// PaperMemory can parse it inside a launched cloakbrowser instance, then +// verify what was actually parsed. +// +// This is the cloakbrowser-driven sibling of `manual-open-urls.js`: instead of +// being pasted into the extension's service-worker console (chrome.* APIs), it +// drives a real browser via the helpers in `browser.js` (puppeteer/cloakbrowser). +// +// HOW TO USE: +// 1. Build the extension: `npm run build:chrome` +// 2. Run: `node test/manual-open-urls-browser.js` +// +// WHAT IT DOES: +// Phase 0 (reset): +// - Launches a cloakbrowser instance with PaperMemory loaded +// - Opens the extension popup page and wipes storage `papers` +// (safe: this is a fresh, isolated browser profile, not your real memory) +// Phase 1 (open): +// - For each source, opens the URL in a new page and waits (via +// `visitPaperPage`) for PaperMemory to store the paper +// - On success: closes the page +// - On timeout: LEAVES THE PAGE OPEN so you can inspect the failure +// Phase 2 (verify): +// - Reads `papers` from the extension popup page +// - Logs found/missing tables +// - Does NOT reopen missing URLs — their pages from phase 1 are still open +// ============================================================================ + +import { + makeBrowser, + visitPaperPage, + getMemoryPapers, + setStorage, + findExtensionId, + getPMURLs, +} from "./browser.js"; +import { sleep } from "./utilsForTests.js"; + +// Restrict the run to specific sources (e.g. `new Set(["iop", "acm"])`). +// Leave empty to test every source in the `urls` list below. +const testOnly = new Set([]); + +const headless = process.env.HEADLESS === "0" ? false : true; // default to true +console.log("headless :", headless); + +const urls = [ + { source: "acl", url: "https://aclanthology.org/2020.acl-main.405" }, + { + source: "aip", + url: "https://pubs.aip.org/aip/aml/article/1/1/010901/2878738/Deep-language-models-for-interpretative-and?searchresult=1", + }, + { source: "acm", url: "https://dl.acm.org/doi/10.5555/3491440.3491756" }, + { source: "acs", url: "https://pubs.acs.org/doi/10.1021/acs.jpca.9b00311" }, + { + source: "aps", + url: "https://journals.aps.org/prl/abstract/10.1103/PhysRevLett.111.111101", + }, + { source: "arxiv", url: "https://arxiv.org/abs/1703.10593" }, + { + source: "biorxiv", + url: "https://www.biorxiv.org/content/10.1101/2021.11.08.467690v2", + }, + { + source: "cell", + url: "https://www.cell.com/cell/fulltext/S0092-8674(25)01089-X", + }, + { + source: "chemrxiv", + url: "https://chemrxiv.org/engage/chemrxiv/article-details/65957d349138d231611ad8f7", + }, + { + source: "cvf", + url: "https://openaccess.thecvf.com/content_CVPR_2020/html/Bhattacharjee_DUNIT_Detection-Based_Unsupervised_Image-to-Image_Translation_CVPR_2020_paper.html", + idPrefix: "cvpr", + }, + { + source: "frontiers", + url: "https://www.frontiersin.org/articles/10.3389/fpace.2022.892330/full", + }, + { source: "hal", url: "https://hal.science/hal-03171076" }, + { source: "ijcai", url: "https://www.ijcai.org/proceedings/2020/1" }, + { source: "ieee", url: "https://ieeexplore.ieee.org/document/9090146" }, + { source: "ihep", url: "https://inspirehep.net/literature/2095720" }, + { + source: "iop", + url: "https://iopscience.iop.org/article/10.1149/2.1051908jes", + idPrefix: "IOPscience", + }, + { source: "jmlr", url: "https://www.jmlr.org/papers/v13/bergstra12a.html" }, + { source: "mdpi", url: "https://www.mdpi.com/2076-328X/13/12/961" }, + { source: "nature", url: "https://www.nature.com/articles/s41467-018-07210-0" }, + { + source: "neurips", + url: "https://proceedings.neurips.cc/paper/2019/hash/0118a063b4aae95277f0bc1752c75abf-Abstract.html", + }, + { + source: "openreview", + url: "https://openreview.net/forum?id=xQUe1pOKPam", + idPrefix: "or", + }, + { + source: "oup", + url: "https://academic.oup.com/brain/article/147/3/743/7617466", + }, + { source: "pmc", url: "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7249434" }, + { source: "pmlr", url: "https://proceedings.mlr.press/v130/husain21a.html" }, + { + source: "wiley", + url: "https://onlinelibrary.wiley.com/doi/abs/10.1002/acr2.11440", + }, + { + source: "pnas", + url: "https://www.pnas.org/doi/full/10.1073/pnas.2114679118", + }, + { + source: "science", + url: "https://science.org/doi/full/10.1126/scirobotics.abm6597", + }, + { + source: "sciencedirect", + url: "https://www.sciencedirect.com/science/article/pii/S2589721721000349", + }, + { + source: "springer", + url: "https://link.springer.com/article/10.1007/s41095-022-0271-y", + }, + { + source: "plos", + url: "https://journals.plos.org/climate/article?id=10.1371/journal.pclm.0000068", + }, + { + source: "rsc", + url: "https://pubs.rsc.org/en/content/articlelanding/2022/dd/d2dd00066k", + }, +].filter(({ source }) => testOnly.size === 0 || testOnly.has(source)); + +// Extra delay (ms) after `visitPaperPage` reports the page settled, before we +// re-read storage to decide success/failure. +const SETTLE_MS = 1000; + +// Some sources tag papers with an id prefix that differs from the source +// name (e.g. openreview → `OR-...`). Everything else uses the source tag +// itself as the id prefix, case-insensitively. +const idPrefixFor = (target) => (target.idPrefix || target.source).toLowerCase(); + +const browser = await makeBrowser(true); +const extensionId = await findExtensionId(browser); +const { popupURL } = getPMURLs(extensionId); + +// A dedicated PaperMemory extension page used to reset and read storage. +const pmPage = await browser.newPage(); +await pmPage.goto(popupURL); + +const getPaperCount = async () => { + const papers = await getMemoryPapers(pmPage); + return Object.keys(papers || {}).length; +}; + +const openOne = async (target, index, total) => { + const { source, url } = target; + const prevCount = await getPaperCount(); + console.log(`[${index + 1}/${total}] Opening ${source} (papers: ${prevCount})`); + const page = await browser.newPage(); + + const started = Date.now(); + await visitPaperPage(browser, url, { page, keepOpen: true, timeout: SETTLE_MS }); + const elapsed = ((Date.now() - started) / 1000).toFixed(1); + + const newCount = await getPaperCount(); + if (newCount > prevCount) { + console.log(` ✓ ${source} parsed in ${elapsed}s`); + await page.close(); + } else { + console.log(` ✗ ${source} timed out after ${elapsed}s — leaving page open`); + } +}; + +const openAll = async () => { + console.log(`\n===== Phase 1: Opening ${urls.length} URLs =====`); + for (let i = 0; i < urls.length; i++) { + await openOne(urls[i], i, urls.length); + } + console.log("Open phase complete.\n"); +}; + +const verify = async () => { + const papers = await getMemoryPapers(pmPage); + const papersList = Object.values(papers || {}).filter( + (p) => p && typeof p === "object", + ); + const found = []; + const missing = []; + + for (const target of urls) { + const { source, url } = target; + const prefix = idPrefixFor(target); + const match = papersList.find((p) => { + const pid = (p.id || "").toLowerCase(); + return pid.startsWith(`${prefix}-`) || pid.startsWith(`${prefix}_`); + }); + if (match) { + found.push({ source, title: match.title.slice(0, 30), id: match.id }); + } else { + missing.push({ source, url }); + } + } + + console.log(`===== Phase 2: Verification =====`); + console.log(`Total papers in storage: ${papersList.length}`); + console.log(`Expected sources: ${urls.length}`); + console.log(`Found: ${found.length}`); + console.log(`Missing: ${missing.length}\n`); + + if (found.length > 0) { + console.log(`✓ Found papers (${found.length}):`); + console.table(found); + } + + if (missing.length > 0) { + console.log(`✗ Missing papers (${missing.length}):`); + console.table(missing); + console.log( + "Pages for missing sources were left open in phase 1 — inspect them directly.", + ); + } else { + console.log(`✓ All sources were successfully parsed!`); + } +}; + +await setStorage(pmPage, "papers", { __dataVersion: 1 }); +console.log("Memory reset: extension storage `papers` cleared."); +await openAll(); +await sleep(1000); +await verify(); From 4204b812a0ab6c25bc74672bbec40142c97026c1 Mon Sep 17 00:00:00 2001 From: vict0rsch Date: Fri, 5 Jun 2026 00:14:14 +0200 Subject: [PATCH 15/19] chore: botprevention for AIP and ACM --- test/data/urls.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/data/urls.json b/test/data/urls.json index f2e3b303..af779de8 100644 --- a/test/data/urls.json +++ b/test/data/urls.json @@ -11,7 +11,8 @@ "https://watermark02.silverchair.com/010901_1_5.0134317.pdf?token=AQECAHi208BE49Ooan9kkhW_Ercy7Dm3ZL_9Cf3qfKAc485ysgAAAy8wggMrBgkqhkiG9w0BBwagggMcMIIDGAIBADCCAxEGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQM4RTwOlcL3rxxHpClAgEQgIIC4n-V-uQcDI2YfQlmdHOaw_Dn7s7RiWP2Eh70_z9G4Sis8c9LslR-jTagm5adDBWKBvFeZVv2L3zU-5ZTRhkMD4cfWiBJHLDmEqvz7uEOMqcbgntFRzMj17QCMYMwwQtQHnAcn4jARCldux8AEJfpmGR2Ay5l9hIwO21GyZY_vgf41bm4tBk7fNiUJfitFXqY8Rzmm7WMsOlUn_iCfwueNG0HbaZQHErpOv2VnzEFaX7rQddmWXpCpZmKOZy8bNXqtErupmMSef9pqLT0fSBqv0oZwL-e40FtzEf2Rzyg7670-7yRc7kaMSDNmPSTC8NaD5I7HccHzBFHD-2727GrutfeXvUOWO6y7BatXF6P0xxXJD4glt7aBQgEKg99odz0XHGoWYEJQCFXQf6gAYHQNncSji9FDTVpp_2wpM-VM6dFBEbPBF1ebRtphgkfPGbx8ZHR0qaclD9zAmLnBv7XdPba6Tc5HgsIdPVt3gToeaBRQxeyBn3ez0DgnyawVKtdLehKFD42Wn1FoVqfkg2E3n4dmkFItuCEtbMsZMrfrDUuwILpx-VHl-k14pjubbeTl-qcy_5STdXoZqUTuNANiS--Cpjg1FR3e-0hTjyqOzl7byL0oiVULgUEYs1P4jRG8DEGEYdsPhFSa7NnklmuQMn1Kc-v9jLI1zdilU67ggEmNPMCl6H_rN4lEGhWktAPIVFWNsOY3976M0g1nzDU6bHhYMfIznG3N1p1LrKrxBup0Mwg-ot8gPi6CX-2E3WTMdeEbRX_gw-hKYAvG9n86AjCVOUsX3cCPpq2sLE1rjI-brpePMSxktLVMM5vXWz-3no3f60kN8sgMcnq1KrYf8H2luIo30j-n6JqBx-FLJD9nT40y5VRI88-eE0ugfCJ8YJOjdW5UIfSFXD1ECoVCspGrfJgw7oNVv_YFx0v1SDP7FxALgEF4cwRWMhYpWlOL0qKwD6dHSe6x-zNpxc84_nA5Q", { "noPdf": true, - "venue": "APL Machine Learning" + "venue": "APL Machine Learning", + "botPrevention": true } ], "acm": [ @@ -19,7 +20,8 @@ "https://dl.acm.org/doi/pdf/10.5555/3491440.3491756", { "code": true, - "venue": "ijcai" + "venue": "ijcai", + "botPrevention": true } ], "acs": [ From 652a25d7b875cf2e4343b5c9393e0aae8ca73d66 Mon Sep 17 00:00:00 2001 From: vict0rsch Date: Fri, 5 Jun 2026 00:34:35 +0200 Subject: [PATCH 16/19] test: aggregate live visit failures Continue live publisher batches after individual visit failures and keep Gist sync manual-only so CI reports stable aggregate failures. Co-authored-by: Cursor --- .github/workflows/_test-matrix.yml | 2 -- contributing.md | 2 +- test/browser.js | 32 ++++++++++++++++++++++++++++++ test/test-duplicates.js | 8 ++++++-- test/test-storage.js | 8 ++++++-- test/test-sync.js | 2 ++ 6 files changed, 47 insertions(+), 7 deletions(-) diff --git a/.github/workflows/_test-matrix.yml b/.github/workflows/_test-matrix.yml index 9865cc57..3297289e 100644 --- a/.github/workflows/_test-matrix.yml +++ b/.github/workflows/_test-matrix.yml @@ -26,8 +26,6 @@ jobs: files: test/test-memory-item-actions.js test/test-memory-table-ui.js - name: Menu files: test/test-menu.js - - name: Sync - files: test/test-sync.js - name: All test scripts are listed in a workflow files: test/test-meta.js diff --git a/contributing.md b/contributing.md index 30386261..4443b15e 100644 --- a/contributing.md +++ b/contributing.md @@ -281,7 +281,7 @@ Sources are **stateless** `BasePaperSource` subclasses (the registry does a name Tests use [cloakbrowser](https://github.com/CloakHQ/cloakbrowser) (a source-level stealth Chromium with the Puppeteer-core API) to launch a real Chrome instance with the extension loaded. On first run, cloakbrowser auto-downloads its custom Chromium binary (~200MB) to `~/.cloakbrowser`; subsequent runs use the cached binary. `CHROME_PATH` is not used for the stealth browser. `npm test` builds the extension first (`pretest` runs `npm run build:chrome`), then runs all test suites. -The integration suites `test/test-duplicates.js` and `test/test-storage.js` hit many live publisher pages; they are valuable as smoke checks but are **not expected to pass entirely** (timeouts, bot walls, and DOM drift are common). `npm test` excludes `test/test-sync.js`; run it explicitly with `npm run test:file test/test-sync.js` when working on Gist sync — that file has the same “best effort” expectations. +The integration suites `test/test-duplicates.js` and `test/test-storage.js` hit many live publisher pages; they are valuable as smoke checks but are volatile (timeouts, bot walls, and DOM drift are common). Their visit phase is best-effort: one URL failure does not stop later URLs from being visited, but the suite still fails afterward with an aggregate visit-failure summary. `npm test` and CI exclude `test/test-sync.js`; run it explicitly with `npm run test:file test/test-sync.js` when working on Gist sync — that file requires a GitHub PAT and has the same live-service volatility. ```bash npm test # Build + run all tests diff --git a/test/browser.js b/test/browser.js index e6af6b18..ac53584f 100644 --- a/test/browser.js +++ b/test/browser.js @@ -114,6 +114,38 @@ export const visitPaperPage = async (browser, target, options = {}) => { !opts.keepOpen && (await p.close()); }; +export const makeVisitFailureCollector = (label = "paper page visits") => { + const failures = []; + + return { + failures, + visit: async (browser, target, options = {}) => { + try { + await visitPaperPage(browser, target, options); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + failures.push({ target, message }); + console.error( + `${indent(options.indents || 0)}Visit failed: ${message}`, + ); + } + }, + throwIfFailed: () => { + if (!failures.length) return; + + const summary = failures + .map( + ({ target, message }, index) => + `${index + 1}. ${target}\n ${message}`, + ) + .join("\n"); + throw new Error( + `${label}: ${failures.length} visit(s) failed after completing the batch:\n${summary}`, + ); + }, + }; +}; + // Helper function to find the actual extension ID from the Chrome extensions page export const findExtensionId = async (browser) => { const page = await browser.newPage(); diff --git a/test/test-duplicates.js b/test/test-duplicates.js index 6c8f50a3..9ad632e5 100644 --- a/test/test-duplicates.js +++ b/test/test-duplicates.js @@ -10,7 +10,7 @@ import { getPaperMemoryState, findExtensionId, getPMURLs, - visitPaperPage, + makeVisitFailureCollector, } from "./browser.js"; import { @@ -132,6 +132,9 @@ describe("Paper de-duplication", function () { before(async function () { // create browser browser = await makeBrowser(headless); + const visitFailures = makeVisitFailureCollector( + `Duplicate visits for order ${order}`, + ); // visit non-duplicated papers first, then known duplicates const allVisits = [...preDuplicates, ...allDuplicates]; // count total number of urls to visit @@ -159,7 +162,7 @@ describe("Paper de-duplication", function () { console.log( `${indent(2)}(${n + 1}/${nUrls}) visiting ${dup.name || "pre-duplicate"} (${dup.url.slice(0, 30)}[...])`, ); - await visitPaperPage(browser, dup.url, { + await visitFailures.visit(browser, dup.url, { keepOpen, timeout: pageTimeout, indents: 3, @@ -167,6 +170,7 @@ describe("Paper de-duplication", function () { n += 1; } } + visitFailures.throwIfFailed(); // go to the extension popup's page memoryPage = (await browser.pages())[0]; diff --git a/test/test-storage.js b/test/test-storage.js index a4897e9e..ff9818c4 100644 --- a/test/test-storage.js +++ b/test/test-storage.js @@ -12,7 +12,7 @@ import { getMemoryPapers, findExtensionId, getPMURLs, - visitPaperPage, + makeVisitFailureCollector, } from "./browser.js"; import { @@ -156,6 +156,9 @@ describe("Test paper detection and storage", function () { !Object.keys(urls).length && this.skip(); // create browser browser = await makeBrowser(headless); + const visitFailures = makeVisitFailureCollector( + `Storage visits for parsing order ${order}`, + ); // count total urls to visit depending on maxSources const nUrls = sources.length; @@ -197,13 +200,14 @@ describe("Test paper detection and storage", function () { `${indent(2)}(${n}/${nUrls * 2}) Going to: ${currentSource} (${target.slice(0, 30)}[...])`, ); - await visitPaperPage(browser, target, { + await visitFailures.visit(browser, target, { timeout: pageTimeout, keepOpen, indents: 3, }); } } + visitFailures.throwIfFailed(); // go to the extension's popup url const page = await browser.newPage(); diff --git a/test/test-sync.js b/test/test-sync.js index f14816be..20f69553 100644 --- a/test/test-sync.js +++ b/test/test-sync.js @@ -1,3 +1,5 @@ +// test-meta:ignore -- manual-only Gist sync test; requires a GitHub PAT. + import { makeBrowser, getPMURLs, From 6da34c7b1424c0f76e1a35c4c4950d20fac831f4 Mon Sep 17 00:00:00 2001 From: vict0rsch Date: Fri, 5 Jun 2026 10:55:39 +0200 Subject: [PATCH 17/19] test: replace manual url opener with mocha phases Co-authored-by: Cursor --- .claude/skills/debug-source-parsing/SKILL.md | 5 +- .cursor/skills/debug-source-parsing/SKILL.md | 5 +- .github/workflows/test.yml | 8 + test/data/urls.json | 6 +- test/manual-open-urls-browser.js | 235 ----------------- test/manual-open-urls.js | 252 ------------------- test/manual-test-urls.js | 231 +++++++++++++++-- test/test-duplicates.js | 3 +- test/test-storage.js | 3 +- test/testConfig.yaml | 12 +- test/utilsForTests.js | 8 + 11 files changed, 244 insertions(+), 524 deletions(-) delete mode 100644 test/manual-open-urls-browser.js delete mode 100644 test/manual-open-urls.js diff --git a/.claude/skills/debug-source-parsing/SKILL.md b/.claude/skills/debug-source-parsing/SKILL.md index 2bff8d99..bd24ff25 100644 --- a/.claude/skills/debug-source-parsing/SKILL.md +++ b/.claude/skills/debug-source-parsing/SKILL.md @@ -55,7 +55,7 @@ Note which order fails (`abs;pdf` vs `pdf;abs`) and the exact URL in the timeout | --- | --- | | Fails in **one** order only | **parse** — the URL conversion for the *other* page is wrong. Inspect `absURL` / `pdfLink` / `toAbs` / `toPDF`. | | Fails in **both** orders, page **rendered fine** | **parse** — a selector/field the publisher changed, so `dom.getElementById(...)`/`getElementsByTagName(...)` returns null and throws (or returns a wrong/`undefined` field). | -| Fails in **both** orders, page is a **challenge/blank** | **detect or fetch blocked** — bot-wall. Confirm with a visible browser; the source may need the `botPrevention` flag in `urls.json`. | +| Fails in **both** orders, page is a **challenge/blank** | **detect or fetch blocked** — bot-wall. Confirm with a visible browser; the source may need the `ciBotPrevention` flag in `urls.json`. | | Stored, but a field is wrong/`undefined` | **parse** — the extraction logic for that field; check against the live DOM. | ### 4. Confirm in cloakbrowser @@ -109,4 +109,5 @@ All cases must pass in **both** orders. Then delete the throwaway script (`test/ ## Notes - Set `headless` to `false` (e.g. `HEADLESS=0` for `test/manual-open-urls-browser.js`, or `makeBrowser(false)`) when you need to watch the page interactively. -- Sources flagged `botPrevention` in `urls.json` are skipped by the automated test — debug those manually with a visible browser. +- Sources flagged `ciBotPrevention` in `urls.json` are skipped by the automated test — debug those manually with a visible browser. +- Sources flagged `manualBotPrevention` in `urls.json` are also skipped by `test/manual-open-urls-browser.js`. diff --git a/.cursor/skills/debug-source-parsing/SKILL.md b/.cursor/skills/debug-source-parsing/SKILL.md index 2bff8d99..bd24ff25 100644 --- a/.cursor/skills/debug-source-parsing/SKILL.md +++ b/.cursor/skills/debug-source-parsing/SKILL.md @@ -55,7 +55,7 @@ Note which order fails (`abs;pdf` vs `pdf;abs`) and the exact URL in the timeout | --- | --- | | Fails in **one** order only | **parse** — the URL conversion for the *other* page is wrong. Inspect `absURL` / `pdfLink` / `toAbs` / `toPDF`. | | Fails in **both** orders, page **rendered fine** | **parse** — a selector/field the publisher changed, so `dom.getElementById(...)`/`getElementsByTagName(...)` returns null and throws (or returns a wrong/`undefined` field). | -| Fails in **both** orders, page is a **challenge/blank** | **detect or fetch blocked** — bot-wall. Confirm with a visible browser; the source may need the `botPrevention` flag in `urls.json`. | +| Fails in **both** orders, page is a **challenge/blank** | **detect or fetch blocked** — bot-wall. Confirm with a visible browser; the source may need the `ciBotPrevention` flag in `urls.json`. | | Stored, but a field is wrong/`undefined` | **parse** — the extraction logic for that field; check against the live DOM. | ### 4. Confirm in cloakbrowser @@ -109,4 +109,5 @@ All cases must pass in **both** orders. Then delete the throwaway script (`test/ ## Notes - Set `headless` to `false` (e.g. `HEADLESS=0` for `test/manual-open-urls-browser.js`, or `makeBrowser(false)`) when you need to watch the page interactively. -- Sources flagged `botPrevention` in `urls.json` are skipped by the automated test — debug those manually with a visible browser. +- Sources flagged `ciBotPrevention` in `urls.json` are skipped by the automated test — debug those manually with a visible browser. +- Sources flagged `manualBotPrevention` in `urls.json` are also skipped by `test/manual-open-urls-browser.js`. diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8d009e72..5b30cbc7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,6 +35,14 @@ jobs: retry_on: error command: npm run test:file test/test-storage.js + - uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 + name: Run Open URLs Test + with: + timeout_minutes: 30 + max_attempts: 2 + retry_on: error + command: npm run test:file test/manual-open-urls.js + - uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 name: Run Storage Test with: diff --git a/test/data/urls.json b/test/data/urls.json index af779de8..760a096f 100644 --- a/test/data/urls.json +++ b/test/data/urls.json @@ -12,7 +12,7 @@ { "noPdf": true, "venue": "APL Machine Learning", - "botPrevention": true + "ciBotPrevention": true } ], "acm": [ @@ -21,7 +21,7 @@ { "code": true, "venue": "ijcai", - "botPrevention": true + "ciBotPrevention": true } ], "acs": [ @@ -151,7 +151,7 @@ "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7249434/pdf/12961_2020_Article_544.pdf", { "venue": "Health research policy and systems", - "botPrevention": true + "ciBotPrevention": true } ], "pmlr": [ diff --git a/test/manual-open-urls-browser.js b/test/manual-open-urls-browser.js deleted file mode 100644 index 36829947..00000000 --- a/test/manual-open-urls-browser.js +++ /dev/null @@ -1,235 +0,0 @@ -// ============================================================================ -// MANUAL TEST (cloakbrowser): Reset memory, open every urls.json source so -// PaperMemory can parse it inside a launched cloakbrowser instance, then -// verify what was actually parsed. -// -// This is the cloakbrowser-driven sibling of `manual-open-urls.js`: instead of -// being pasted into the extension's service-worker console (chrome.* APIs), it -// drives a real browser via the helpers in `browser.js` (puppeteer/cloakbrowser). -// -// HOW TO USE: -// 1. Build the extension: `npm run build:chrome` -// 2. Run: `node test/manual-open-urls-browser.js` -// -// WHAT IT DOES: -// Phase 0 (reset): -// - Launches a cloakbrowser instance with PaperMemory loaded -// - Opens the extension popup page and wipes storage `papers` -// (safe: this is a fresh, isolated browser profile, not your real memory) -// Phase 1 (open): -// - For each source, opens the URL in a new page and waits (via -// `visitPaperPage`) for PaperMemory to store the paper -// - On success: closes the page -// - On timeout: LEAVES THE PAGE OPEN so you can inspect the failure -// Phase 2 (verify): -// - Reads `papers` from the extension popup page -// - Logs found/missing tables -// - Does NOT reopen missing URLs — their pages from phase 1 are still open -// ============================================================================ - -import { - makeBrowser, - visitPaperPage, - getMemoryPapers, - setStorage, - findExtensionId, - getPMURLs, -} from "./browser.js"; -import { sleep } from "./utilsForTests.js"; - -// Restrict the run to specific sources (e.g. `new Set(["iop", "acm"])`). -// Leave empty to test every source in the `urls` list below. -const testOnly = new Set([]); - -const headless = process.env.HEADLESS === "0" ? false : true; // default to true -console.log("headless :", headless); - -const urls = [ - { source: "acl", url: "https://aclanthology.org/2020.acl-main.405" }, - { - source: "aip", - url: "https://pubs.aip.org/aip/aml/article/1/1/010901/2878738/Deep-language-models-for-interpretative-and?searchresult=1", - }, - { source: "acm", url: "https://dl.acm.org/doi/10.5555/3491440.3491756" }, - { source: "acs", url: "https://pubs.acs.org/doi/10.1021/acs.jpca.9b00311" }, - { - source: "aps", - url: "https://journals.aps.org/prl/abstract/10.1103/PhysRevLett.111.111101", - }, - { source: "arxiv", url: "https://arxiv.org/abs/1703.10593" }, - { - source: "biorxiv", - url: "https://www.biorxiv.org/content/10.1101/2021.11.08.467690v2", - }, - { - source: "cell", - url: "https://www.cell.com/cell/fulltext/S0092-8674(25)01089-X", - }, - { - source: "chemrxiv", - url: "https://chemrxiv.org/engage/chemrxiv/article-details/65957d349138d231611ad8f7", - }, - { - source: "cvf", - url: "https://openaccess.thecvf.com/content_CVPR_2020/html/Bhattacharjee_DUNIT_Detection-Based_Unsupervised_Image-to-Image_Translation_CVPR_2020_paper.html", - idPrefix: "cvpr", - }, - { - source: "frontiers", - url: "https://www.frontiersin.org/articles/10.3389/fpace.2022.892330/full", - }, - { source: "hal", url: "https://hal.science/hal-03171076" }, - { source: "ijcai", url: "https://www.ijcai.org/proceedings/2020/1" }, - { source: "ieee", url: "https://ieeexplore.ieee.org/document/9090146" }, - { source: "ihep", url: "https://inspirehep.net/literature/2095720" }, - { - source: "iop", - url: "https://iopscience.iop.org/article/10.1149/2.1051908jes", - idPrefix: "IOPscience", - }, - { source: "jmlr", url: "https://www.jmlr.org/papers/v13/bergstra12a.html" }, - { source: "mdpi", url: "https://www.mdpi.com/2076-328X/13/12/961" }, - { source: "nature", url: "https://www.nature.com/articles/s41467-018-07210-0" }, - { - source: "neurips", - url: "https://proceedings.neurips.cc/paper/2019/hash/0118a063b4aae95277f0bc1752c75abf-Abstract.html", - }, - { - source: "openreview", - url: "https://openreview.net/forum?id=xQUe1pOKPam", - idPrefix: "or", - }, - { - source: "oup", - url: "https://academic.oup.com/brain/article/147/3/743/7617466", - }, - { source: "pmc", url: "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7249434" }, - { source: "pmlr", url: "https://proceedings.mlr.press/v130/husain21a.html" }, - { - source: "wiley", - url: "https://onlinelibrary.wiley.com/doi/abs/10.1002/acr2.11440", - }, - { - source: "pnas", - url: "https://www.pnas.org/doi/full/10.1073/pnas.2114679118", - }, - { - source: "science", - url: "https://science.org/doi/full/10.1126/scirobotics.abm6597", - }, - { - source: "sciencedirect", - url: "https://www.sciencedirect.com/science/article/pii/S2589721721000349", - }, - { - source: "springer", - url: "https://link.springer.com/article/10.1007/s41095-022-0271-y", - }, - { - source: "plos", - url: "https://journals.plos.org/climate/article?id=10.1371/journal.pclm.0000068", - }, - { - source: "rsc", - url: "https://pubs.rsc.org/en/content/articlelanding/2022/dd/d2dd00066k", - }, -].filter(({ source }) => testOnly.size === 0 || testOnly.has(source)); - -// Extra delay (ms) after `visitPaperPage` reports the page settled, before we -// re-read storage to decide success/failure. -const SETTLE_MS = 1000; - -// Some sources tag papers with an id prefix that differs from the source -// name (e.g. openreview → `OR-...`). Everything else uses the source tag -// itself as the id prefix, case-insensitively. -const idPrefixFor = (target) => (target.idPrefix || target.source).toLowerCase(); - -const browser = await makeBrowser(true); -const extensionId = await findExtensionId(browser); -const { popupURL } = getPMURLs(extensionId); - -// A dedicated PaperMemory extension page used to reset and read storage. -const pmPage = await browser.newPage(); -await pmPage.goto(popupURL); - -const getPaperCount = async () => { - const papers = await getMemoryPapers(pmPage); - return Object.keys(papers || {}).length; -}; - -const openOne = async (target, index, total) => { - const { source, url } = target; - const prevCount = await getPaperCount(); - console.log(`[${index + 1}/${total}] Opening ${source} (papers: ${prevCount})`); - const page = await browser.newPage(); - - const started = Date.now(); - await visitPaperPage(browser, url, { page, keepOpen: true, timeout: SETTLE_MS }); - const elapsed = ((Date.now() - started) / 1000).toFixed(1); - - const newCount = await getPaperCount(); - if (newCount > prevCount) { - console.log(` ✓ ${source} parsed in ${elapsed}s`); - await page.close(); - } else { - console.log(` ✗ ${source} timed out after ${elapsed}s — leaving page open`); - } -}; - -const openAll = async () => { - console.log(`\n===== Phase 1: Opening ${urls.length} URLs =====`); - for (let i = 0; i < urls.length; i++) { - await openOne(urls[i], i, urls.length); - } - console.log("Open phase complete.\n"); -}; - -const verify = async () => { - const papers = await getMemoryPapers(pmPage); - const papersList = Object.values(papers || {}).filter( - (p) => p && typeof p === "object", - ); - const found = []; - const missing = []; - - for (const target of urls) { - const { source, url } = target; - const prefix = idPrefixFor(target); - const match = papersList.find((p) => { - const pid = (p.id || "").toLowerCase(); - return pid.startsWith(`${prefix}-`) || pid.startsWith(`${prefix}_`); - }); - if (match) { - found.push({ source, title: match.title.slice(0, 30), id: match.id }); - } else { - missing.push({ source, url }); - } - } - - console.log(`===== Phase 2: Verification =====`); - console.log(`Total papers in storage: ${papersList.length}`); - console.log(`Expected sources: ${urls.length}`); - console.log(`Found: ${found.length}`); - console.log(`Missing: ${missing.length}\n`); - - if (found.length > 0) { - console.log(`✓ Found papers (${found.length}):`); - console.table(found); - } - - if (missing.length > 0) { - console.log(`✗ Missing papers (${missing.length}):`); - console.table(missing); - console.log( - "Pages for missing sources were left open in phase 1 — inspect them directly.", - ); - } else { - console.log(`✓ All sources were successfully parsed!`); - } -}; - -await setStorage(pmPage, "papers", { __dataVersion: 1 }); -console.log("Memory reset: extension storage `papers` cleared."); -await openAll(); -await sleep(1000); -await verify(); diff --git a/test/manual-open-urls.js b/test/manual-open-urls.js deleted file mode 100644 index cfe7585d..00000000 --- a/test/manual-open-urls.js +++ /dev/null @@ -1,252 +0,0 @@ -// ============================================================================ -// MANUAL TEST: Reset memory, open every urls.json source so PaperMemory can -// parse it, then verify what was actually parsed. -// -// HOW TO USE: -// 1. Run `npm run dev` to start the dev browser with PaperMemory loaded -// 2. Open the extension's service worker DevTools (chrome://extensions → -// PaperMemory → "Inspect views: service worker"). The options page -// console also works. -// 3. Set `deleteMemory = true` at the top of this script (safety gate) -// 4. Copy-paste this entire script into the console and press Enter -// -// WHAT IT DOES: -// Phase 1 (open): -// - Wipes chrome.storage.local.papers (guarded by `deleteMemory` flag) -// - For each source, opens the URL in a background tab and polls -// chrome.storage.local until a new paper appears (or times out) -// - On success: closes the tab -// - On timeout: LEAVES THE TAB OPEN so you can inspect the failure -// Phase 2 (verify): -// - Reads chrome.storage.local.papers -// - Logs found/missing tables -// - Does NOT reopen missing URLs — their tabs from phase 1 are still open -// -// Safe in a service-worker console: no window-only APIs (no confirm/alert). -// ============================================================================ - -// Safety gate: set to true manually before pasting to confirm you really -// want to wipe chrome.storage.local.papers. Left false by default so an -// accidental paste never destroys your memory. -const deleteMemory = false; - -// Restrict the run to specific sources (e.g. `new Set(["iop", "acm"])`). -// Leave empty to test every source in the `urls` list below. -const testOnly = new Set([]); - -if (!deleteMemory) { - console.warn( - "Aborted: `deleteMemory` is false. This script will reset ALL papers in chrome.storage.local. Set `const deleteMemory = true;` at the top of the script to proceed.", - ); - throw new Error("Aborted: `deleteMemory` is false."); -} - -const urls = [ - { source: "acl", url: "https://aclanthology.org/2020.acl-main.405" }, - { - source: "aip", - url: "https://pubs.aip.org/aip/aml/article/1/1/010901/2878738/Deep-language-models-for-interpretative-and?searchresult=1", - }, - { source: "acm", url: "https://dl.acm.org/doi/10.5555/3491440.3491756" }, - { source: "acs", url: "https://pubs.acs.org/doi/10.1021/acs.jpca.9b00311" }, - { - source: "aps", - url: "https://journals.aps.org/prl/abstract/10.1103/PhysRevLett.111.111101", - }, - { source: "arxiv", url: "https://arxiv.org/abs/1703.10593" }, - { - source: "biorxiv", - url: "https://www.biorxiv.org/content/10.1101/2021.11.08.467690v2", - }, - { - source: "cell", - url: "https://www.cell.com/cell/fulltext/S0092-8674(25)01089-X", - }, - { - source: "chemrxiv", - url: "https://chemrxiv.org/engage/chemrxiv/article-details/65957d349138d231611ad8f7", - }, - { - source: "cvf", - url: "https://openaccess.thecvf.com/content_CVPR_2020/html/Bhattacharjee_DUNIT_Detection-Based_Unsupervised_Image-to-Image_Translation_CVPR_2020_paper.html", - idPrefix: "cvpr", - }, - { - source: "frontiers", - url: "https://www.frontiersin.org/articles/10.3389/fpace.2022.892330/full", - }, - { source: "hal", url: "https://hal.science/hal-03171076" }, - { source: "ijcai", url: "https://www.ijcai.org/proceedings/2020/1" }, - { source: "ieee", url: "https://ieeexplore.ieee.org/document/9090146" }, - { source: "ihep", url: "https://inspirehep.net/literature/2095720" }, - { - source: "iop", - url: "https://iopscience.iop.org/article/10.1149/2.1051908jes", - idPrefix: "IOPscience", - }, - { source: "jmlr", url: "https://www.jmlr.org/papers/v13/bergstra12a.html" }, - { source: "mdpi", url: "https://www.mdpi.com/2076-328X/13/12/961" }, - { source: "nature", url: "https://www.nature.com/articles/s41467-018-07210-0" }, - { - source: "neurips", - url: "https://proceedings.neurips.cc/paper/2019/hash/0118a063b4aae95277f0bc1752c75abf-Abstract.html", - }, - { - source: "openreview", - url: "https://openreview.net/forum?id=xQUe1pOKPam", - idPrefix: "or", - }, - { - source: "oup", - url: "https://academic.oup.com/brain/article/147/3/743/7617466", - }, - { source: "pmc", url: "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7249434" }, - { source: "pmlr", url: "https://proceedings.mlr.press/v130/husain21a.html" }, - { - source: "wiley", - url: "https://onlinelibrary.wiley.com/doi/abs/10.1002/acr2.11440", - }, - { - source: "pnas", - url: "https://www.pnas.org/doi/full/10.1073/pnas.2114679118", - }, - { - source: "science", - url: "https://science.org/doi/full/10.1126/scirobotics.abm6597", - }, - { - source: "sciencedirect", - url: "https://www.sciencedirect.com/science/article/pii/S2589721721000349", - }, - { - source: "springer", - url: "https://link.springer.com/article/10.1007/s41095-022-0271-y", - }, - { - source: "plos", - url: "https://journals.plos.org/climate/article?id=10.1371/journal.pclm.0000068", - }, - { - source: "rsc", - url: "https://pubs.rsc.org/en/content/articlelanding/2022/dd/d2dd00066k", - }, -].filter(({ source }) => testOnly.size === 0 || testOnly.has(source)); - -const POLL_INTERVAL = 100; -const MAX_WAIT = 10_000; - -// Some sources tag papers with an id prefix that differs from the source -// name (e.g. openreview → `OR-...`). Everything else uses the source tag -// itself as the id prefix, case-insensitively. -const idPrefixFor = (target) => (target.idPrefix || target.source).toLowerCase(); - -const getPaperCount = async () => { - const data = await chrome.storage.local.get("papers"); - return Object.keys(data.papers || {}).length; -}; - -const waitForNewPaper = (prevCount, elapsed) => - new Promise((resolve) => { - if (elapsed >= MAX_WAIT) { - resolve(false); - return; - } - setTimeout(() => { - getPaperCount().then((count) => { - if (count > prevCount) { - resolve(true); - } else { - resolve(waitForNewPaper(prevCount, elapsed + POLL_INTERVAL)); - } - }); - }, POLL_INTERVAL); - }); - -const openOne = async (source, url, index, total) => { - const prevCount = await getPaperCount(); - console.log(`[${index + 1}/${total}] Opening ${source} (papers: ${prevCount})`); - const tab = await chrome.tabs.create({ url, active: false }); - - const started = Date.now(); - const found = await waitForNewPaper(prevCount, 0); - const elapsed = ((Date.now() - started) / 1000).toFixed(1); - if (found) { - console.log(` ✓ ${source} parsed in ${elapsed}s`); - await chrome.tabs.remove(tab.id); - } else { - console.log( - ` ✗ ${source} timed out after ${elapsed}s — leaving tab ${tab.id} open`, - ); - } -}; - -const openAll = async () => { - console.log(`\n===== Phase 1: Opening ${urls.length} URLs =====`); - for (let i = 0; i < urls.length; i++) { - const target = urls[i]; - const { source, url } = target; - await openOne(source, url, i, urls.length); - } - console.log("Open phase complete.\n"); -}; - -const verify = async () => { - const data = await chrome.storage.local.get("papers"); - const papersList = Object.values(data.papers || {}); - const found = []; - const missing = []; - - for (const target of urls) { - const { source, url } = target; - const prefix = idPrefixFor(target); - const match = papersList.find((p) => { - const pid = (p.id || "").toLowerCase(); - return pid.startsWith(`${prefix}-`) || pid.startsWith(`${prefix}_`); - }); - if (match) { - found.push({ source, title: match.title, id: match.id }); - } else { - missing.push({ source, url }); - } - } - - console.log(`===== Phase 2: Verification =====`); - console.log(`Total papers in storage: ${papersList.length}`); - console.log(`Expected sources: ${urls.length}`); - console.log(`Found: ${found.length}`); - console.log(`Missing: ${missing.length}\n`); - - if (found.length > 0) { - console.log( - `%c✓ Found papers (${found.length}):`, - "color: green; font-weight: bold", - ); - console.table(found); - } - - if (missing.length > 0) { - console.log( - `%c✗ Missing papers (${missing.length}):`, - "color: red; font-weight: bold", - ); - console.table(missing); - console.log( - "Tabs for missing sources were left open in phase 1 — inspect them directly.", - ); - } else { - console.log( - `%c✓ All sources were successfully parsed!`, - "color: green; font-weight: bold; font-size: 14px", - ); - } -}; - -const main = async () => { - await chrome.storage.local.set({ papers: { __dataVersion: 1 } }); - console.log("Memory reset: chrome.storage.local.papers cleared."); - await openAll(); - await new Promise((resolve) => setTimeout(resolve, 1000)); - await verify(); -}; - -main(); diff --git a/test/manual-test-urls.js b/test/manual-test-urls.js index 730c6e58..591daa33 100644 --- a/test/manual-test-urls.js +++ b/test/manual-test-urls.js @@ -1,35 +1,222 @@ -import { makeBrowser, visitPaperPage } from "./browser.js"; -import { readJSON, sleep, root } from "./utilsForTests.js"; +import { expect } from "expect"; +import fs from "fs"; +import { basename } from "node:path"; +import { + findExtensionId, + getMemoryPapers, + getPMURLs, + makeBrowser, + makeVisitFailureCollector, + setStorage, + visitPaperPage, +} from "./browser.js"; +import { + hasManualBotPrevention, + indent, + loadConfig, + readURLs, + root, + sleep, +} from "./utilsForTests.js"; -const testUrls = readJSON(`${root}/test/data/urls.json`); +let { dump, headless, ignoreSources, keepOpen, maxSources, onlySources, pageTimeout } = + loadConfig(); -const browser = await makeBrowser(); +if (typeof ignoreSources === "string") { + ignoreSources = ignoreSources.split(",").map((source) => source.trim()); +} + +console.log(`\n${basename(import.meta.url)} args:`); +console.log(" pageTimeout : ", pageTimeout); +console.log(" maxSources : ", maxSources); +console.log(" onlySources : ", onlySources); +console.log(" keepOpen : ", keepOpen); +console.log(" dump : ", dump); +console.log(" ignoreSources : ", ignoreSources); +console.log(" headless : ", headless); + +const targetUrls = (entries) => entries.filter((entry) => typeof entry === "string"); + +const buildPaperTargets = () => { + let urls = Object.entries(readURLs()); + + if (maxSources > 0) { + console.info("Truncating urls to maxSources: ", maxSources); + urls = urls.slice(0, maxSources); + } else if (onlySources && onlySources.length > 0) { + urls = urls.filter(([source]) => onlySources.includes(source)); + } + + if (ignoreSources && ignoreSources.length > 0) { + urls = urls.filter(([source]) => !ignoreSources.includes(source)); + } + + const skippedSources = []; + const targets = urls.flatMap(([source, entries]) => { + if (hasManualBotPrevention(entries)) { + skippedSources.push(source); + return []; + } + + const urls = targetUrls(entries); + return { + source, + abstractUrl: urls[0], + pdfUrl: urls[1], + noPdf: Boolean(entries[2]?.noPdf), + }; + }); + + if (skippedSources.length) { + console.log( + `\n>>> Skipping manual open-url tests for sources with manual bot prevention: ${skippedSources.join(", ")}`, + ); + } + + return targets; +}; + +const paperForSource = (source, memoryPapers) => + Object.values(memoryPapers).find((paper) => paper.source === source); + +const memorySources = (memoryPapers) => + Object.values(memoryPapers) + .map((paper) => paper.source) + .sort(); + +const withoutDataVersion = (memoryPapers) => { + const papers = { ...memoryPapers }; + delete papers.__dataVersion; + return papers; +}; + +const createMemoryContext = async () => { + const browser = await makeBrowser(headless); + const extensionId = await findExtensionId(browser); + const { popupURL } = getPMURLs(extensionId); + const memoryPage = await browser.newPage(); + await memoryPage.goto(popupURL); + await setStorage(memoryPage, "papers", { __dataVersion: 1 }); + return { browser, memoryPage }; +}; + +const readMemory = async ({ memoryPage }) => + withoutDataVersion(await getMemoryPapers(memoryPage)); + +const dumpMemory = (label, memoryPapers) => { + if (!dump) return; -const options = { - keepOpen: true, - timeout: -1, + fs.mkdirSync(`${root}/test/tmp`, { recursive: true }); + fs.writeFileSync( + `${root}/test/tmp/open-urls-${label}-${Date.now()}.json`, + JSON.stringify(memoryPapers, null, 2), + ); }; -const gotToPaperPage = async (url) => { - try { +const visitTargets = async ({ browser }, targets, phaseLabel) => { + const visitFailures = makeVisitFailureCollector(`${phaseLabel} open-url visits`); + + for (const [index, target] of targets.entries()) { + console.log( + `${indent(2)}(${index + 1}/${targets.length}) visiting ${phaseLabel} for ${target.source} (${target.url.slice(0, 30)}[...])`, + ); const page = await browser.newPage(); - await visitPaperPage(browser, url, { - ...options, + await visitFailures.visit(browser, target.url, { page, + keepOpen, + timeout: pageTimeout, + indents: 3, }); - } catch (error) { - console.error(`Error visiting paper page ${url}:`, error); } + + visitFailures.throwIfFailed(); + await sleep(1000); }; -for (const [source, urls] of Object.entries(testUrls)) { - if (urls[2]?.botPrevention) { - console.log( - `\nManual test for ${source} because it has bot prevention:\n ${urls[0]}\n ${urls[1]}`, - ); - await gotToPaperPage(urls[0]); - await gotToPaperPage(urls[1]); +const assertStoredSources = (memoryPapers, targets) => { + const expectedSources = targets.map(({ source }) => source).sort(); + const storedSources = memorySources(memoryPapers); + expect(storedSources).toEqual(expectedSources); +}; + +const assertVisitCounts = (memoryPapers, targets) => { + const undercountedSources = []; + + for (const target of targets) { + const paper = paperForSource(target.source, memoryPapers); + const expectedMinCount = target.pdfUrl && !target.noPdf ? 2 : 1; + if (!paper || paper.count < expectedMinCount) { + undercountedSources.push({ + source: target.source, + expectedMinCount, + actualCount: paper?.count, + }); + } } -} -console.log("Done"); + expect(undercountedSources).toEqual([]); +}; + +describe("Manual open URL parsing phases", function () { + const paperTargets = buildPaperTargets(); + const abstractTargets = paperTargets.map((target) => ({ + ...target, + url: target.abstractUrl, + })); + const pdfTargets = paperTargets + .filter((target) => target.pdfUrl && !target.noPdf) + .map((target) => ({ + ...target, + url: target.pdfUrl, + })); + const timeout = (abstractTargets.length * 2 + pdfTargets.length + 2) * 20 * 5000; + + let abstractContext; + let pdfContext; + + this.timeout(timeout); + this.slow(timeout); + + after(async function () { + if (!keepOpen) { + await abstractContext?.browser.close(); + await pdfContext?.browser.close(); + } + }); + + it("Phase 1: stores every paper from Abstract URLs", async function () { + abstractContext = await createMemoryContext(); + + await visitTargets(abstractContext, abstractTargets, "Abstract URL"); + const memoryPapers = await readMemory(abstractContext); + dumpMemory("abstract", memoryPapers); + + assertStoredSources(memoryPapers, abstractTargets); + for (const target of abstractTargets) { + expect( + paperForSource(target.source, memoryPapers)?.count, + ).toBeGreaterThanOrEqual(1); + } + }); + + it("Phase 2: stores every paper from PDF URLs in a fresh context", async function () { + pdfContext = await createMemoryContext(); + + await visitTargets(pdfContext, pdfTargets, "PDF URL"); + const memoryPapers = await readMemory(pdfContext); + dumpMemory("pdf", memoryPapers); + + assertStoredSources(memoryPapers, pdfTargets); + }); + + it("Phase 3: increments PDF-backed papers when Abstract URLs reuse the PDF context", async function () { + if (!pdfContext) this.skip(); + + await visitTargets(pdfContext, abstractTargets, "Abstract URL after PDF URL"); + const memoryPapers = await readMemory(pdfContext); + dumpMemory("pdf-then-abstract", memoryPapers); + + assertStoredSources(memoryPapers, abstractTargets); + assertVisitCounts(memoryPapers, abstractTargets); + }); +}); diff --git a/test/test-duplicates.js b/test/test-duplicates.js index 9ad632e5..bb061342 100644 --- a/test/test-duplicates.js +++ b/test/test-duplicates.js @@ -21,6 +21,7 @@ import { root, loadConfig, indent, + hasCiBotPrevention, } from "./utilsForTests.js"; // make all functions in utils.min.js available in the `global` scope @@ -84,7 +85,7 @@ if (maxSources > 0) { let preDuplicates = Object.entries(urls) .filter(([source, urls]) => !ignoreSources.includes(source)) .map(([source, urls]) => urls) - .filter((urls) => urls.length < 3 || !urls[2].botPrevention) + .filter((urls) => !hasCiBotPrevention(urls)) .map((urls) => [{ url: urls[0] }]) .map((value) => ({ value, sort: Math.random() })) // shuffle .sort((a, b) => a.sort - b.sort) diff --git a/test/test-storage.js b/test/test-storage.js index ff9818c4..aaadca58 100644 --- a/test/test-storage.js +++ b/test/test-storage.js @@ -21,6 +21,7 @@ import { loadConfig, loadPaperMemoryUtils, indent, + hasCiBotPrevention, } from "./utilsForTests.js"; import { allAttributes } from "./processMemory.js"; @@ -106,7 +107,7 @@ describe("Test paper detection and storage", function () { for (const source in urls) { const targets = urls[source]; - if (targets.length === 3 && targets[2].botPrevention) { + if (hasCiBotPrevention(targets)) { console.log( `\n>>> Skipping test for "${source}" because its website ` + `prevents automated browsing. Remember to test manually:` + diff --git a/test/testConfig.yaml b/test/testConfig.yaml index 6e490767..d8350923 100644 --- a/test/testConfig.yaml +++ b/test/testConfig.yaml @@ -3,31 +3,31 @@ # --------------------- # Keep tab open after paper is parsed? -# |> test-storage, test-duplicates +# |> test-storage, manual-open-urls, test-duplicates keepOpen: type: bool defaultValue: true # Timeout between paper pages (ms) -# |> test-storage, test-duplicates +# |> test-storage, manual-open-urls, test-duplicates pageTimeout: type: int defaultValue: 200 # Max number of sources to iterate through for debug. -1 is all sources. -# |> test-storage, test-duplicates +# |> test-storage, manual-open-urls, test-duplicates maxSources: type: int defaultValue: -1 # Whether or not to dump the parsed memory to a JSON file in test/tmp -# |> test-storage, test-duplicates +# |> test-storage, manual-open-urls, test-duplicates dump: type: bool defaultValue: true # Which sources to test (,-separated) -# |> test-storage, test-duplicates +# |> test-storage, manual-open-urls, test-duplicates onlySources: type: comma-separated-str defaultValue: "" @@ -46,7 +46,7 @@ singleName: defaultValue: "" # Ignore these sources (,-separated) -# |> test-storage, test-duplicates +# |> test-storage, manual-open-urls, test-duplicates ignoreSources: type: comma-separated-str defaultValue: "" diff --git a/test/utilsForTests.js b/test/utilsForTests.js index 2642aac6..2211cf83 100644 --- a/test/utilsForTests.js +++ b/test/utilsForTests.js @@ -132,6 +132,14 @@ export const asyncMap = (arr, func) => Promise.all(arr.map(func)); */ export const readURLs = () => readJSON(`${root}/test/data/urls.json`); +export const urlTestConfig = (entries) => entries[2] || {}; + +export const hasCiBotPrevention = (entries) => + Boolean(urlTestConfig(entries).ciBotPrevention); + +export const hasManualBotPrevention = (entries) => + Boolean(urlTestConfig(entries).manualBotPrevention); + /** * Read the duplicates data file. */ From 66431e7da9163b7b7fe24accab17fba5f578e6b1 Mon Sep 17 00:00:00 2001 From: vict0rsch Date: Fri, 5 Jun 2026 10:58:44 +0200 Subject: [PATCH 18/19] test: restore manual open urls file Co-authored-by: Cursor --- test/manual-open-urls.js | 222 +++++++++++++++++++++++++++++++++++++ test/manual-test-urls.js | 231 ++++----------------------------------- 2 files changed, 244 insertions(+), 209 deletions(-) create mode 100644 test/manual-open-urls.js diff --git a/test/manual-open-urls.js b/test/manual-open-urls.js new file mode 100644 index 00000000..591daa33 --- /dev/null +++ b/test/manual-open-urls.js @@ -0,0 +1,222 @@ +import { expect } from "expect"; +import fs from "fs"; +import { basename } from "node:path"; +import { + findExtensionId, + getMemoryPapers, + getPMURLs, + makeBrowser, + makeVisitFailureCollector, + setStorage, + visitPaperPage, +} from "./browser.js"; +import { + hasManualBotPrevention, + indent, + loadConfig, + readURLs, + root, + sleep, +} from "./utilsForTests.js"; + +let { dump, headless, ignoreSources, keepOpen, maxSources, onlySources, pageTimeout } = + loadConfig(); + +if (typeof ignoreSources === "string") { + ignoreSources = ignoreSources.split(",").map((source) => source.trim()); +} + +console.log(`\n${basename(import.meta.url)} args:`); +console.log(" pageTimeout : ", pageTimeout); +console.log(" maxSources : ", maxSources); +console.log(" onlySources : ", onlySources); +console.log(" keepOpen : ", keepOpen); +console.log(" dump : ", dump); +console.log(" ignoreSources : ", ignoreSources); +console.log(" headless : ", headless); + +const targetUrls = (entries) => entries.filter((entry) => typeof entry === "string"); + +const buildPaperTargets = () => { + let urls = Object.entries(readURLs()); + + if (maxSources > 0) { + console.info("Truncating urls to maxSources: ", maxSources); + urls = urls.slice(0, maxSources); + } else if (onlySources && onlySources.length > 0) { + urls = urls.filter(([source]) => onlySources.includes(source)); + } + + if (ignoreSources && ignoreSources.length > 0) { + urls = urls.filter(([source]) => !ignoreSources.includes(source)); + } + + const skippedSources = []; + const targets = urls.flatMap(([source, entries]) => { + if (hasManualBotPrevention(entries)) { + skippedSources.push(source); + return []; + } + + const urls = targetUrls(entries); + return { + source, + abstractUrl: urls[0], + pdfUrl: urls[1], + noPdf: Boolean(entries[2]?.noPdf), + }; + }); + + if (skippedSources.length) { + console.log( + `\n>>> Skipping manual open-url tests for sources with manual bot prevention: ${skippedSources.join(", ")}`, + ); + } + + return targets; +}; + +const paperForSource = (source, memoryPapers) => + Object.values(memoryPapers).find((paper) => paper.source === source); + +const memorySources = (memoryPapers) => + Object.values(memoryPapers) + .map((paper) => paper.source) + .sort(); + +const withoutDataVersion = (memoryPapers) => { + const papers = { ...memoryPapers }; + delete papers.__dataVersion; + return papers; +}; + +const createMemoryContext = async () => { + const browser = await makeBrowser(headless); + const extensionId = await findExtensionId(browser); + const { popupURL } = getPMURLs(extensionId); + const memoryPage = await browser.newPage(); + await memoryPage.goto(popupURL); + await setStorage(memoryPage, "papers", { __dataVersion: 1 }); + return { browser, memoryPage }; +}; + +const readMemory = async ({ memoryPage }) => + withoutDataVersion(await getMemoryPapers(memoryPage)); + +const dumpMemory = (label, memoryPapers) => { + if (!dump) return; + + fs.mkdirSync(`${root}/test/tmp`, { recursive: true }); + fs.writeFileSync( + `${root}/test/tmp/open-urls-${label}-${Date.now()}.json`, + JSON.stringify(memoryPapers, null, 2), + ); +}; + +const visitTargets = async ({ browser }, targets, phaseLabel) => { + const visitFailures = makeVisitFailureCollector(`${phaseLabel} open-url visits`); + + for (const [index, target] of targets.entries()) { + console.log( + `${indent(2)}(${index + 1}/${targets.length}) visiting ${phaseLabel} for ${target.source} (${target.url.slice(0, 30)}[...])`, + ); + const page = await browser.newPage(); + await visitFailures.visit(browser, target.url, { + page, + keepOpen, + timeout: pageTimeout, + indents: 3, + }); + } + + visitFailures.throwIfFailed(); + await sleep(1000); +}; + +const assertStoredSources = (memoryPapers, targets) => { + const expectedSources = targets.map(({ source }) => source).sort(); + const storedSources = memorySources(memoryPapers); + expect(storedSources).toEqual(expectedSources); +}; + +const assertVisitCounts = (memoryPapers, targets) => { + const undercountedSources = []; + + for (const target of targets) { + const paper = paperForSource(target.source, memoryPapers); + const expectedMinCount = target.pdfUrl && !target.noPdf ? 2 : 1; + if (!paper || paper.count < expectedMinCount) { + undercountedSources.push({ + source: target.source, + expectedMinCount, + actualCount: paper?.count, + }); + } + } + + expect(undercountedSources).toEqual([]); +}; + +describe("Manual open URL parsing phases", function () { + const paperTargets = buildPaperTargets(); + const abstractTargets = paperTargets.map((target) => ({ + ...target, + url: target.abstractUrl, + })); + const pdfTargets = paperTargets + .filter((target) => target.pdfUrl && !target.noPdf) + .map((target) => ({ + ...target, + url: target.pdfUrl, + })); + const timeout = (abstractTargets.length * 2 + pdfTargets.length + 2) * 20 * 5000; + + let abstractContext; + let pdfContext; + + this.timeout(timeout); + this.slow(timeout); + + after(async function () { + if (!keepOpen) { + await abstractContext?.browser.close(); + await pdfContext?.browser.close(); + } + }); + + it("Phase 1: stores every paper from Abstract URLs", async function () { + abstractContext = await createMemoryContext(); + + await visitTargets(abstractContext, abstractTargets, "Abstract URL"); + const memoryPapers = await readMemory(abstractContext); + dumpMemory("abstract", memoryPapers); + + assertStoredSources(memoryPapers, abstractTargets); + for (const target of abstractTargets) { + expect( + paperForSource(target.source, memoryPapers)?.count, + ).toBeGreaterThanOrEqual(1); + } + }); + + it("Phase 2: stores every paper from PDF URLs in a fresh context", async function () { + pdfContext = await createMemoryContext(); + + await visitTargets(pdfContext, pdfTargets, "PDF URL"); + const memoryPapers = await readMemory(pdfContext); + dumpMemory("pdf", memoryPapers); + + assertStoredSources(memoryPapers, pdfTargets); + }); + + it("Phase 3: increments PDF-backed papers when Abstract URLs reuse the PDF context", async function () { + if (!pdfContext) this.skip(); + + await visitTargets(pdfContext, abstractTargets, "Abstract URL after PDF URL"); + const memoryPapers = await readMemory(pdfContext); + dumpMemory("pdf-then-abstract", memoryPapers); + + assertStoredSources(memoryPapers, abstractTargets); + assertVisitCounts(memoryPapers, abstractTargets); + }); +}); diff --git a/test/manual-test-urls.js b/test/manual-test-urls.js index 591daa33..7cf03250 100644 --- a/test/manual-test-urls.js +++ b/test/manual-test-urls.js @@ -1,222 +1,35 @@ -import { expect } from "expect"; -import fs from "fs"; -import { basename } from "node:path"; -import { - findExtensionId, - getMemoryPapers, - getPMURLs, - makeBrowser, - makeVisitFailureCollector, - setStorage, - visitPaperPage, -} from "./browser.js"; -import { - hasManualBotPrevention, - indent, - loadConfig, - readURLs, - root, - sleep, -} from "./utilsForTests.js"; +import { makeBrowser, visitPaperPage } from "./browser.js"; +import { hasManualBotPrevention, readURLs } from "./utilsForTests.js"; -let { dump, headless, ignoreSources, keepOpen, maxSources, onlySources, pageTimeout } = - loadConfig(); +const testUrls = readURLs(); -if (typeof ignoreSources === "string") { - ignoreSources = ignoreSources.split(",").map((source) => source.trim()); -} - -console.log(`\n${basename(import.meta.url)} args:`); -console.log(" pageTimeout : ", pageTimeout); -console.log(" maxSources : ", maxSources); -console.log(" onlySources : ", onlySources); -console.log(" keepOpen : ", keepOpen); -console.log(" dump : ", dump); -console.log(" ignoreSources : ", ignoreSources); -console.log(" headless : ", headless); - -const targetUrls = (entries) => entries.filter((entry) => typeof entry === "string"); - -const buildPaperTargets = () => { - let urls = Object.entries(readURLs()); - - if (maxSources > 0) { - console.info("Truncating urls to maxSources: ", maxSources); - urls = urls.slice(0, maxSources); - } else if (onlySources && onlySources.length > 0) { - urls = urls.filter(([source]) => onlySources.includes(source)); - } - - if (ignoreSources && ignoreSources.length > 0) { - urls = urls.filter(([source]) => !ignoreSources.includes(source)); - } - - const skippedSources = []; - const targets = urls.flatMap(([source, entries]) => { - if (hasManualBotPrevention(entries)) { - skippedSources.push(source); - return []; - } - - const urls = targetUrls(entries); - return { - source, - abstractUrl: urls[0], - pdfUrl: urls[1], - noPdf: Boolean(entries[2]?.noPdf), - }; - }); - - if (skippedSources.length) { - console.log( - `\n>>> Skipping manual open-url tests for sources with manual bot prevention: ${skippedSources.join(", ")}`, - ); - } - - return targets; -}; - -const paperForSource = (source, memoryPapers) => - Object.values(memoryPapers).find((paper) => paper.source === source); - -const memorySources = (memoryPapers) => - Object.values(memoryPapers) - .map((paper) => paper.source) - .sort(); - -const withoutDataVersion = (memoryPapers) => { - const papers = { ...memoryPapers }; - delete papers.__dataVersion; - return papers; -}; - -const createMemoryContext = async () => { - const browser = await makeBrowser(headless); - const extensionId = await findExtensionId(browser); - const { popupURL } = getPMURLs(extensionId); - const memoryPage = await browser.newPage(); - await memoryPage.goto(popupURL); - await setStorage(memoryPage, "papers", { __dataVersion: 1 }); - return { browser, memoryPage }; -}; - -const readMemory = async ({ memoryPage }) => - withoutDataVersion(await getMemoryPapers(memoryPage)); - -const dumpMemory = (label, memoryPapers) => { - if (!dump) return; +const browser = await makeBrowser(); - fs.mkdirSync(`${root}/test/tmp`, { recursive: true }); - fs.writeFileSync( - `${root}/test/tmp/open-urls-${label}-${Date.now()}.json`, - JSON.stringify(memoryPapers, null, 2), - ); +const options = { + keepOpen: true, + timeout: -1, }; -const visitTargets = async ({ browser }, targets, phaseLabel) => { - const visitFailures = makeVisitFailureCollector(`${phaseLabel} open-url visits`); - - for (const [index, target] of targets.entries()) { - console.log( - `${indent(2)}(${index + 1}/${targets.length}) visiting ${phaseLabel} for ${target.source} (${target.url.slice(0, 30)}[...])`, - ); +const gotToPaperPage = async (url) => { + try { const page = await browser.newPage(); - await visitFailures.visit(browser, target.url, { + await visitPaperPage(browser, url, { + ...options, page, - keepOpen, - timeout: pageTimeout, - indents: 3, }); + } catch (error) { + console.error(`Error visiting paper page ${url}:`, error); } - - visitFailures.throwIfFailed(); - await sleep(1000); -}; - -const assertStoredSources = (memoryPapers, targets) => { - const expectedSources = targets.map(({ source }) => source).sort(); - const storedSources = memorySources(memoryPapers); - expect(storedSources).toEqual(expectedSources); }; -const assertVisitCounts = (memoryPapers, targets) => { - const undercountedSources = []; - - for (const target of targets) { - const paper = paperForSource(target.source, memoryPapers); - const expectedMinCount = target.pdfUrl && !target.noPdf ? 2 : 1; - if (!paper || paper.count < expectedMinCount) { - undercountedSources.push({ - source: target.source, - expectedMinCount, - actualCount: paper?.count, - }); - } +for (const [source, urls] of Object.entries(testUrls)) { + if (hasManualBotPrevention(urls)) { + console.log( + `\nManual test for ${source} because it has manual bot prevention:\n ${urls[0]}\n ${urls[1]}`, + ); + await gotToPaperPage(urls[0]); + await gotToPaperPage(urls[1]); } +} - expect(undercountedSources).toEqual([]); -}; - -describe("Manual open URL parsing phases", function () { - const paperTargets = buildPaperTargets(); - const abstractTargets = paperTargets.map((target) => ({ - ...target, - url: target.abstractUrl, - })); - const pdfTargets = paperTargets - .filter((target) => target.pdfUrl && !target.noPdf) - .map((target) => ({ - ...target, - url: target.pdfUrl, - })); - const timeout = (abstractTargets.length * 2 + pdfTargets.length + 2) * 20 * 5000; - - let abstractContext; - let pdfContext; - - this.timeout(timeout); - this.slow(timeout); - - after(async function () { - if (!keepOpen) { - await abstractContext?.browser.close(); - await pdfContext?.browser.close(); - } - }); - - it("Phase 1: stores every paper from Abstract URLs", async function () { - abstractContext = await createMemoryContext(); - - await visitTargets(abstractContext, abstractTargets, "Abstract URL"); - const memoryPapers = await readMemory(abstractContext); - dumpMemory("abstract", memoryPapers); - - assertStoredSources(memoryPapers, abstractTargets); - for (const target of abstractTargets) { - expect( - paperForSource(target.source, memoryPapers)?.count, - ).toBeGreaterThanOrEqual(1); - } - }); - - it("Phase 2: stores every paper from PDF URLs in a fresh context", async function () { - pdfContext = await createMemoryContext(); - - await visitTargets(pdfContext, pdfTargets, "PDF URL"); - const memoryPapers = await readMemory(pdfContext); - dumpMemory("pdf", memoryPapers); - - assertStoredSources(memoryPapers, pdfTargets); - }); - - it("Phase 3: increments PDF-backed papers when Abstract URLs reuse the PDF context", async function () { - if (!pdfContext) this.skip(); - - await visitTargets(pdfContext, abstractTargets, "Abstract URL after PDF URL"); - const memoryPapers = await readMemory(pdfContext); - dumpMemory("pdf-then-abstract", memoryPapers); - - assertStoredSources(memoryPapers, abstractTargets); - assertVisitCounts(memoryPapers, abstractTargets); - }); -}); +console.log("Done"); From 22d5f816e3b259670d5f62f56a8f97661074192f Mon Sep 17 00:00:00 2001 From: vict0rsch Date: Tue, 9 Jun 2026 22:45:53 +0200 Subject: [PATCH 19/19] docs: better manual-open-urls documentation --- .claude/skills/debug-source-parsing/SKILL.md | 4 ++-- .cursor/skills/debug-source-parsing/SKILL.md | 4 ++-- .github/workflows/test.yml | 8 -------- contributing.md | 11 ++++++++++- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/.claude/skills/debug-source-parsing/SKILL.md b/.claude/skills/debug-source-parsing/SKILL.md index bd24ff25..85dfd183 100644 --- a/.claude/skills/debug-source-parsing/SKILL.md +++ b/.claude/skills/debug-source-parsing/SKILL.md @@ -108,6 +108,6 @@ All cases must pass in **both** orders. Then delete the throwaway script (`test/ ## Notes -- Set `headless` to `false` (e.g. `HEADLESS=0` for `test/manual-open-urls-browser.js`, or `makeBrowser(false)`) when you need to watch the page interactively. +- Set `headless` to `false` (e.g. `headless=false npm run test:file test/manual-open-urls.js`, or `makeBrowser(false)`) when you need to watch the page interactively. - Sources flagged `ciBotPrevention` in `urls.json` are skipped by the automated test — debug those manually with a visible browser. -- Sources flagged `manualBotPrevention` in `urls.json` are also skipped by `test/manual-open-urls-browser.js`. +- Sources flagged `manualBotPrevention` in `urls.json` are also skipped by `test/manual-open-urls.js`. diff --git a/.cursor/skills/debug-source-parsing/SKILL.md b/.cursor/skills/debug-source-parsing/SKILL.md index bd24ff25..85dfd183 100644 --- a/.cursor/skills/debug-source-parsing/SKILL.md +++ b/.cursor/skills/debug-source-parsing/SKILL.md @@ -108,6 +108,6 @@ All cases must pass in **both** orders. Then delete the throwaway script (`test/ ## Notes -- Set `headless` to `false` (e.g. `HEADLESS=0` for `test/manual-open-urls-browser.js`, or `makeBrowser(false)`) when you need to watch the page interactively. +- Set `headless` to `false` (e.g. `headless=false npm run test:file test/manual-open-urls.js`, or `makeBrowser(false)`) when you need to watch the page interactively. - Sources flagged `ciBotPrevention` in `urls.json` are skipped by the automated test — debug those manually with a visible browser. -- Sources flagged `manualBotPrevention` in `urls.json` are also skipped by `test/manual-open-urls-browser.js`. +- Sources flagged `manualBotPrevention` in `urls.json` are also skipped by `test/manual-open-urls.js`. diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5b30cbc7..8d009e72 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,14 +35,6 @@ jobs: retry_on: error command: npm run test:file test/test-storage.js - - uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 - name: Run Open URLs Test - with: - timeout_minutes: 30 - max_attempts: 2 - retry_on: error - command: npm run test:file test/manual-open-urls.js - - uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 name: Run Storage Test with: diff --git a/contributing.md b/contributing.md index 4443b15e..4f62d54f 100644 --- a/contributing.md +++ b/contributing.md @@ -266,7 +266,7 @@ Most editors can auto-format on save with the Prettier extension. For VS Code, i 2. **Register** the class in `sources/index.js`: add its import and insert the class reference into the `ALL_SOURCES` array (once). `BY_NAME`, `SOURCE_DISPATCH_ORDER`, `knownPaperPages`, `preprintSources`, and `allSources()` are all derived automatically. The mutual-exclusion test in `test/test-meta.js` enforces that your patterns don't overlap with any existing source. The settings page renders sources alphabetically by display name. 3. **Reuse** shared HTTP/BibTeX/DOM helpers from `parsers.js` instead of duplicating fetches. 4. **Optional overrides** on the class: `displayId(paper, baseId)`, `venue(paper)`, `static isPreprint = true` (only for preprint servers used in fuzzy deduplication). -5. **Add** URLs under `test/data/urls.json` and into `test/test-storage.js` to keep test coverage. +5. **Add** URLs under `test/data/urls.json` and verify parsing locally with `test/manual-open-urls.js`. ### Why stateless handlers, not class instances @@ -283,6 +283,15 @@ Tests use [cloakbrowser](https://github.com/CloakHQ/cloakbrowser) (a source-leve The integration suites `test/test-duplicates.js` and `test/test-storage.js` hit many live publisher pages; they are valuable as smoke checks but are volatile (timeouts, bot walls, and DOM drift are common). Their visit phase is best-effort: one URL failure does not stop later URLs from being visited, but the suite still fails afterward with an aggregate visit-failure summary. `npm test` and CI exclude `test/test-sync.js`; run it explicitly with `npm run test:file test/test-sync.js` when working on Gist sync — that file requires a GitHub PAT and has the same live-service volatility. +`test/manual-open-urls.js` is the actual end-to-end parsing correctness check for `test/data/urls.json`. It visits each **Abstract URL**, then each **PDF URL** in a fresh browser context, then reopens the **Abstract URLs** in the PDF context and asserts the expected visit counts. Run it locally on a developer laptop after adding or changing a source: + +```bash +npm run build:chrome +headless=false npm run test:file test/manual-open-urls.js +``` + +Do not rely on CI for this check. Publisher bot-prevention systems make cloud runners unreliable even when parsing is correct, so this test is intentionally a local validation step. + ```bash npm test # Build + run all tests npm run test:file test/test-utils.js # Run a single test file (no auto-build)