Skip to content

fix(1688): extract detail images from shadow DOM with lazy-render scrolling #2272

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
HARRY-BEAR wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(1688): extract detail images from shadow DOM with lazy-render scrolling #2272

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 58 additions & 3 deletions clis/1688/assets.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,26 @@
import { cli, Strategy } from '@jackwener/opencli/registry';
import { assertAuthenticatedState, buildDetailUrl, buildProvenance, cleanText, extractOfferId, gotoAndReadState, uniqueMediaSources, } from './shared.js';
// 1688 商品详情区位于自定义元素 v-detail-e 的 shadow DOM 内（懒渲染），
// 普通 CSS selector 无法穿透 shadowRoot，需沿 shadow host 链判断归属。
export const DETAIL_CONTAINER_SELECTOR = '.de-description-detail, #detailContentContainer, .html-description, .desc-lazyload-container';
export function inDetailContainer(el) {
let node = el;
while (node) {
const rootNode = node.getRootNode ? node.getRootNode() : null;
if (rootNode && rootNode.host) {
const host = rootNode.host;
if (host && host.matches && host.matches(DETAIL_CONTAINER_SELECTOR))
return true;
node = host;
}
else {
if (node.closest && node.closest(DETAIL_CONTAINER_SELECTOR))
return true;
node = null;
}
}
return false;
}
function scriptToReadAssets() {
return `
(() => {
Expand All @@ -11,8 +32,24 @@ function scriptToReadAssets() {
{ key: 'main', type: 'image', selectors: ['#dt-tab img', '.detail-gallery-turn img.detail-gallery-img', '.img-list-wrapper img.od-gallery-img', '.od-scroller-item span'] },
{ key: 'video', type: 'video', selectors: ['.lib-video video', 'video[src]', 'video source[src]'] },
{ key: 'sku', type: 'image', selectors: ['.pc-sku-wrapper .prop-item-inner-wrapper', '.sku-item-wrapper', '.specification-cell', '.sku-filter-button', '.expand-view-item', '.feature-item img'], srcProps: ['backgroundImage'] },
{ key: 'detail', type: 'image', selectors: ['.de-description-detail img', '#detailContentContainer img', '.html-description img', '.html-description source', '.desc-lazyload-container img'] },
];
const detailContainerSelector = ${JSON.stringify(DETAIL_CONTAINER_SELECTOR)};
// 页面上下文内定义的局部版本（闭包携带常量），与模块级 inDetailContainer 逻辑一致
const inDetailContainer = (el) => {
let node = el;
while (node) {
const rootNode = node.getRootNode ? node.getRootNode() : null;
if (rootNode && rootNode.host) {
const host = rootNode.host;
if (host && host.matches && host.matches(detailContainerSelector)) return true;
node = host;
} else {
if (node.closest && node.closest(detailContainerSelector)) return true;
node = null;
}
}
return false;
};
const assets = [];
const seen = new Set();

Expand Down Expand Up @@ -108,6 +145,14 @@ function scriptToReadAssets() {
}
}

// 详情区素材：全量收集 img/source（穿透 shadowRoot）+ host 链归属判断
for (const element of [...queryAllDeep('img'), ...queryAllDeep('source')]) {
if (!inDetailContainer(element)) continue;
for (const value of valuesFromElement(element)) {
push('image', 'detail', value, 'shadow:html-description');
}
}

const scriptTexts = Array.from(document.scripts).map((script) => script.textContent || '');
const videoRegex = /https?:\\/\\/[^"'\\s]+\\.(?:mp4|m3u8)(?:\\?[^"'\\s]*)?/gi;
for (const scriptText of scriptTexts) {
Expand Down Expand Up @@ -171,8 +216,16 @@ function normalizeAssets(payload) {
async function readAssetsPayload(page, itemUrl) {
const state = await gotoAndReadState(page, itemUrl, 2500, 'assets');
assertAuthenticatedState(state, 'assets');
await page.autoScroll({ times: 3, delayMs: 400 });
await page.wait(1);
// 详情区懒渲染：多次滚动到底触发 shadow DOM 内容与懒加载图片。
// 首次滚动 6 次保证长页面到底，再滚 4 次等待加载完成后二次确认。
await page.autoScroll({ times: 6, delayMs: 500 });
await page.autoScroll({ times: 4, delayMs: 500 });
// 若详情容器存在，定位到它并等待 shadow 内容渲染（图片 src 填充）
await page.evaluate(`(() => {
const el = document.querySelector('.html-description, v-detail-e, .de-description-detail, #detailContentContainer');
if (el) el.scrollIntoView({ behavior: 'instant', block: 'start' });
})()`);
await page.wait(3);
return await page.evaluate(scriptToReadAssets());
}
export async function extractAssetsForInput(page, input) {
Expand Down Expand Up @@ -202,4 +255,6 @@ cli({
});
export const __test__ = {
normalizeAssets,
inDetailContainer,
DETAIL_CONTAINER_SELECTOR,
};
77 changes: 43 additions & 34 deletions clis/1688/assets.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,39 +1,48 @@
import { describe, expect, it } from 'vitest';
import { JSDOM } from 'jsdom';
import { __test__ } from './assets.js';
import { __test__ as sharedTest } from './shared.js';
describe('1688 assets normalization', () => {
it('normalizes gallery and scanned assets into grouped media lists', () => {
const result = __test__.normalizeAssets({
href: 'https://detail.1688.com/offer/887904326744.html',
title: '测试商品 - 阿里巴巴',
offerTitle: '测试商品',
offerId: 887904326744,
gallery: {
mainImage: ['//img.example.com/main-1.jpg'],
offerImgList: ['https://img.example.com/main-2.jpg'],
wlImageInfos: [{ fullPathImageURI: 'https://img.example.com/main-3.jpg' }],
},
scannedAssets: [
{ type: 'image', group: 'sku', url: 'https://img.example.com/sku-1.png', source: 'dom:.sku' },
{ type: 'image', group: 'detail', url: 'https://img.example.com/detail-1.jpg', source: 'dom:.detail' },
{ type: 'video', group: 'video', url: 'https://video.example.com/demo.mp4', source: 'script' },
{ type: 'image', group: 'detail', url: 'blob:https://detail.1688.com/1', source: 'ignore' },
],
});
expect(result.offer_id).toBe('887904326744');
expect(result.main_images).toEqual([
'https://img.example.com/main-1.jpg',
'https://img.example.com/main-2.jpg',
'https://img.example.com/main-3.jpg',
]);
expect(result.sku_images).toEqual(['https://img.example.com/sku-1.png']);
expect(result.detail_images).toEqual(['https://img.example.com/detail-1.jpg']);
expect(result.videos).toEqual(['https://video.example.com/demo.mp4']);
expect(result.main_count).toBe(3);
expect(result.video_count).toBe(1);

function makeDetailHostDom() {
// 模拟 1688 详情页：详情图片位于 v-detail-e（class=html-description）
// 的 shadow DOM 内，普通 CSS selector 无法穿透 shadowRoot。
const dom = new JSDOM(
`<html><body>
<div class="detail-gallery-turn"><img src="https://img.example.com/main-1.jpg"></div>
<v-detail-e class="html-description"></v-detail-e>
<div class="de-description-detail"><img src="https://img.example.com/light-1.jpg"></div>
</body></html>`,
{ url: 'https://detail.1688.com/offer/887904326744.html' },
);
const { window } = dom;
const host = window.document.querySelector('v-detail-e');
const shadow = host.attachShadow({ mode: 'open' });
shadow.innerHTML = `
<img src="https://img.example.com/detail-1.jpg">
<img data-lazyload-src="https://img.example.com/detail-2.jpg">
<source srcset="https://img.example.com/detail-3.webp">
`;
return { window, host, shadow };
}

describe('1688 assets shadow-DOM detail container detection', () => {
it('detects images inside the v-detail-e shadow root as detail assets', () => {
const { window, shadow } = makeDetailHostDom();
const shadowImgs = [...shadow.querySelectorAll('img, source')];
expect(shadowImgs.length).toBe(3);
for (const el of shadowImgs) {
expect(__test__.inDetailContainer(el)).toBe(true);
}
});
it('normalizes media urls from style syntax and protocol-relative URLs', () => {
expect(sharedTest.normalizeMediaUrl('url("//img.example.com/1.jpg")')).toBe('https://img.example.com/1.jpg');
expect(sharedTest.normalizeMediaUrl('blob:https://detail.1688.com/1')).toBe('');

it('does not match light-DOM main gallery images', () => {
const { window } = makeDetailHostDom();
const mainImg = window.document.querySelector('.detail-gallery-turn img');
expect(__test__.inDetailContainer(mainImg)).toBe(false);
});

it('matches light-DOM detail containers that use plain classes', () => {
const { window } = makeDetailHostDom();
const lightDetail = window.document.querySelector('.de-description-detail img');
expect(__test__.inDetailContainer(lightDetail)).toBe(true);
});
});