Optimize review collection for beginners
This commit is contained in:
喻长发
+16
-6
@@ -56,11 +56,11 @@ chrome.runtime.onMessage.addListener((m,s,send)=>{
|
||||
return true;
|
||||
}
|
||||
if(m?.type==='STOP_COLLECTION'){(async()=>{try{await chrome.scripting.executeScript({target:{tabId:m.tabId},world:'MAIN',func:()=>{window.__TMALL_AI_STOP_REVIEW__=true;}});send({ok:true})}catch(e){send({ok:false,error:String(e)})}})();return true;}
|
||||
if(m?.type==='RUN_CURRENT'){ run(m.tabId,true).then(x=>send(x)).catch(e=>send({ok:false,error:String(e)})); return true; }
|
||||
if(m?.type==='RUN_CURRENT'){ run(m.tabId,true,m.collectionMode||'standard').then(x=>send(x)).catch(e=>send({ok:false,error:String(e)})); return true; }
|
||||
if(m?.type==='HEALTH'){ health().then(send); return true; }
|
||||
});
|
||||
|
||||
async function run(tabId,manual){
|
||||
async function run(tabId,manual,collectionMode='standard'){
|
||||
const tab=await chrome.tabs.get(tabId);
|
||||
if(!isProduct(tab.url)) throw new Error('当前标签页不是天猫/淘宝商品详情页');
|
||||
const h=await health();
|
||||
@@ -83,7 +83,7 @@ async function run(tabId,manual){
|
||||
await setStatus(tabId,{state:'collecting',message:'正在采集当前商品：主图、SKU、详情、参数、问答与评论。不会采集推荐商品…',progress:10});
|
||||
await chrome.scripting.executeScript({target:{tabId},world:'MAIN',func:()=>{window.__TMALL_AI_STOP_REVIEW__=false;window.__TMALL_AI_REVIEW_PROGRESS__={count:0,round:0,stagnant:0,mode:'启动',elapsedMs:0,preview:[]};}});
|
||||
let collectorDone=false,collectorError=null;const startedAt=Date.now();
|
||||
const collectorPromise=chrome.scripting.executeScript({target:{tabId},world:'MAIN',func:collector,args:[manual]}).then(v=>v).catch(e=>{collectorError=e;return null}).finally(()=>{collectorDone=true});
|
||||
const collectorPromise=chrome.scripting.executeScript({target:{tabId},world:'MAIN',func:collector,args:[manual,collectionMode]}).then(v=>v).catch(e=>{collectorError=e;return null}).finally(()=>{collectorDone=true});
|
||||
while(!collectorDone){try{const rr=await chrome.scripting.executeScript({target:{tabId},world:'MAIN',func:()=>window.__TMALL_AI_REVIEW_PROGRESS__||null});const live=rr?.[0]?.result;if(live)await setStatus(tabId,{state:'collecting_reviews',message:`评论采集中：${live.count||0} 条 · ${live.mode||'处理中'}${live.stagnant?` · ${live.stagnant}轮无新增`:''}`,progress:Math.min(67,18+(Number(live.round||0)*2)),reviewLive:{...live,elapsedMs:Date.now()-startedAt}})}catch(e){}await sleep(600)}
|
||||
const collectorResult=await collectorPromise;
|
||||
if(collectorError)throw new Error('页面采集脚本执行失败：'+String(collectorError?.message||collectorError));
|
||||
@@ -121,7 +121,7 @@ async function run(tabId,manual){
|
||||
return {ok:true,taskId:j.taskId,rawStats:{title:raw.product?.title||'',reviews:raw.reviews?.length||0,questions:raw.questions?.length||0,sku:raw.sku?.length||0,attributes:raw.attributes?.length||0,promotions:raw.promotions?.length||0,images:raw.images?.all?.length||0}};
|
||||
}
|
||||
|
||||
async function collector(manual=false){
|
||||
async function collector(manual=false,collectionMode='standard'){
|
||||
const delay=ms=>new Promise(r=>setTimeout(r,ms));
|
||||
const text=(el)=>el?.innerText?.trim()||el?.textContent?.trim()||'';
|
||||
const clean=s=>(s||'').replace(/\s+/g,' ').trim();
|
||||
@@ -144,6 +144,7 @@ async function collector(manual=false){
|
||||
const bodyText=clean(bodyClone?.innerText||document.body?.innerText||'');
|
||||
const captured=[];
|
||||
const reviews=[]; const questions=[];
|
||||
const reviewCap=collectionMode==='quick'?300:(collectionMode==='full'?3000:800);
|
||||
const reviewNetMeta={totals:[],hasMoreFalse:false,lastPageFlags:[],pageHints:[],networkReviewObjects:0,networkResponses:0};
|
||||
const reviewStartedAt=Date.now();
|
||||
const processedCaptureKeys=new Set();
|
||||
@@ -330,6 +331,7 @@ async function collector(manual=false){
|
||||
// 每轮即时解析已捕获网络响应
|
||||
const capNow=(window.__TMALL_AI_CAPTURE__||[]).slice(-120);for(const c of capNow){try{const ck=String(c.__safeKey||[c.method||'',c.url||'',String(c.text||'').slice(0,240)].join('|'));if(processedCaptureKeys.has(ck))continue;processedCaptureKeys.add(ck);if(/review|rate|comment|feed/i.test(c.url||''))reviewNetMeta.networkResponses++;let z=(c.text||'').replace(/^\s*[\w$.]+\(/,'').replace(/\)\s*;?\s*$/,'');try{walk(JSON.parse(z),0,c.url||'')}catch(_){const m=z.match(/\{[\s\S]*\}/);if(m)try{walk(JSON.parse(m[0]),0,c.url||'')}catch(__){}}}catch(e){}}
|
||||
collectVisibleReviews();collectVisibleQuestions();const after=dedupReviewCount();stagnant=after<=before?stagnant+1:0;publishReviewProgress(round+1,stagnant,next?'翻页/加载更多':'短滚动',{lastDelta:after-before});
|
||||
if(collectionMode!=='full' && after>=reviewCap){collectionComplete=true;explicitEnd=true;stopReason='sample_cap';publishReviewProgress(round+1,stagnant,collectionMode==='quick'?'已采够快速样本':'已采够标准样本');break}
|
||||
const bodyNow=clean(document.body?.innerText||'');const noMoreText=/没有更多(?:评价|评论)|已显示全部(?:评价|评论)|到底了|暂无更多(?:评价|评论)/.test(bodyNow);const captcha=/验证码|滑块|安全验证|访问过于频繁|操作频繁/.test(bodyNow);
|
||||
if(captcha){stopReason='human_verification_required';publishReviewProgress(round+1,stagnant,'需要人工验证');break}
|
||||
if((disabledNext||noMoreText)&&stagnant>=1){explicitEnd=true;collectionComplete=true;stopReason='explicit_last_page';publishReviewProgress(round+1,stagnant,'页面明确结束');break}
|
||||
@@ -528,7 +530,15 @@ async function collector(manual=false){
|
||||
if(!deReview.has(k))deReview.set(k,{...r,images:uniq((r.images||[]).map(reviewImageUrl)).filter(validReviewImage)});
|
||||
else{const old=deReview.get(k);old.images=uniq([...(old.images||[]),...(r.images||[]).map(reviewImageUrl)]).filter(validReviewImage).slice(0,24)}
|
||||
}
|
||||
const reviewRows=[...deReview.values()];
|
||||
let reviewRows=[...deReview.values()];
|
||||
// 评论过多时保留最有决策价值的样本：差评、带图、最新评论优先，其余按原顺序补齐。
|
||||
let reviewSampled=false;
|
||||
if(reviewRows.length>reviewCap){
|
||||
const scored=reviewRows.map((r,i)=>{const rating=Number(String(r.rating||'').match(/[1-5]/)?.[0]||5);const bad=rating<=3?5:0;const media=(r.images||[]).length?3:0;const date=Date.parse(String(r.date||''));const rec=Number.isFinite(date)?Math.max(0,Math.min(2,(date-Date.now()+31536000000)/31536000000)):0;return {r,i,score:bad+media+rec}});
|
||||
scored.sort((a,b)=>b.score-a.score||a.i-b.i);
|
||||
const keep=new Set(scored.slice(0,reviewCap).map(x=>x.i));
|
||||
reviewRows=reviewRows.filter((_,i)=>keep.has(i)); reviewSampled=true;
|
||||
}
|
||||
const buyerCandidates=uniq(reviewRows.flatMap(r=>r.images||[])).filter(validReviewImage);
|
||||
const metaImage=assetUrl(meta(['meta[property="og:image"]','meta[name="twitter:image"]','meta[itemprop="image"]']));
|
||||
const isMainRegionImg=img=>{try{if(isRecImg(img)||img.closest('[class*=review],[class*=Review],[class*=rate],[class*=Rate],[class*=comment],[class*=Comment],[class*=detail],[class*=Detail],[class*=desc],[class*=Desc]'))return false;const r=img.getBoundingClientRect();return r.top<1800&&r.bottom>-80&&r.width>=34&&r.height>=34&&r.left<innerWidth*.76}catch(e){return false}};
|
||||
@@ -571,6 +581,6 @@ async function collector(manual=false){
|
||||
sales:{currentPrice:priceValue,sold:soldValue,ranking:rankMatch?.[1]||''},
|
||||
sku:skuTexts.map(name=>({name})),attributes:attrs,promotions:promoTexts.map(text=>({text})),
|
||||
images:{main,detail,sku:skuImages,buyerShow:buyer,all:allImgs,provenance,classification:{version:'strict-v1',classifiedCount:allImgs.length,unclassifiedCount:Math.max(0,allPageCandidates.length-allImgs.length),groupsAreDisjoint:true}},
|
||||
reviews:reviewRows,questions:[...deQ.values()],pageText:liveBodyText.slice(0,120000),collection:{reviewActual:deReview.size,questionActual:deQ.size,reviewDrawerOpened:reviewDrawerWasOpened,reviewCollectionComplete:collectionComplete,reviewCompleteness:collectionComplete?'confirmed':'unconfirmed',reviewStopReason:stopReason,reviewPagesVisited:pagesVisited,publicReviewCount:publicRaw,publicReviewCountNumeric:publicNumeric,foldedDefaultReviewCount:foldedDefaultMatch?parseCount(foldedDefaultMatch[1]):null,defaultPraiseBuyerCount:defaultPraiseMatch?parseCount(defaultPraiseMatch[1]):null,networkReviewTotal:networkTotal,networkReviewObjects:reviewNetMeta.networkReviewObjects,networkReviewResponses:reviewNetMeta.networkResponses,networkExplicitEnd:reviewNetMeta.hasMoreFalse||reviewNetMeta.lastPageFlags.length>0,explicitEnd,reviewLive:window.__TMALL_AI_REVIEW_PROGRESS__||null}
|
||||
reviews:reviewRows,questions:[...deQ.values()],pageText:liveBodyText.slice(0,120000),collection:{reviewActual:deReview.size,reviewSampled,reviewSampleCap:reviewCap,reviewCollectionMode:collectionMode,questionActual:deQ.size,reviewDrawerOpened:reviewDrawerWasOpened,reviewCollectionComplete:collectionComplete,reviewCompleteness:reviewSampled?'sampled':(collectionComplete?'confirmed':'unconfirmed'),reviewStopReason:reviewSampled?'sample_cap':stopReason,reviewPagesVisited:pagesVisited,publicReviewCount:publicRaw,publicReviewCountNumeric:publicNumeric,foldedDefaultReviewCount:foldedDefaultMatch?parseCount(foldedDefaultMatch[1]):null,defaultPraiseBuyerCount:defaultPraiseMatch?parseCount(defaultPraiseMatch[1]):null,networkReviewTotal:networkTotal,networkReviewObjects:reviewNetMeta.networkReviewObjects,networkReviewResponses:reviewNetMeta.networkResponses,networkExplicitEnd:reviewNetMeta.hasMoreFalse||reviewNetMeta.lastPageFlags.length>0,explicitEnd,reviewLive:window.__TMALL_AI_REVIEW_PROGRESS__||null}
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user