Jack
f1ce583fd1
同步家纺情报 v2 数据、59 个严格权威源、趋势图库、移动端交互、Google/Yandex 翻译门控及静态 SEO/GEO 构建。 补充面向 C# 研发的 README，明确当前无 LLM、多模态与生图模型，记录翻译请求参数、缓存与失败策略、数据处理边界、前端职责和无 GitHub Pages 部署流程。 新增按日期倒序维护的 CHANGELOG，记录影响、验证与代码/数据/dist 回滚方法。 验证：11 个 Python 契约测试、13 个 Node 前端测试通过；静态构建生成 1096 个 Sitemap URL。
925 lines
36 KiB
Python
925 lines
36 KiB
Python
#!/usr/bin/env python3
|
||
"""家纺流行趋势数据采集脚本 — 聚合多个行业数据源，生成 textile-trends.json。
|
||
|
||
数据源清单:
|
||
1. TrendHunter — hipfashion / furniture
|
||
2. 免费趋势博客 RSS — Dezeen / Design Milk / Fibre2Fashion 等
|
||
3. 时装趋势 RSS — Hypebeast / GQ Hong Kong / Citta Bella
|
||
4. The Met / V&A 开放博物馆图像
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import hashlib
|
||
import json
|
||
import re
|
||
import time
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
from dataclasses import dataclass, field
|
||
from datetime import datetime, timedelta, timezone
|
||
from pathlib import Path
|
||
from typing import Any
|
||
from urllib.parse import urljoin
|
||
|
||
import requests
|
||
from bs4 import BeautifulSoup
|
||
from requests.adapters import HTTPAdapter
|
||
from urllib3.util.retry import Retry
|
||
|
||
if __package__:
|
||
from .textile_sources import CURATED_TREND_PAGES, FREE_TREND_FEEDS, is_fashion_reference
|
||
from .intelligence_web_translation import translation_fields
|
||
else:
|
||
from textile_sources import CURATED_TREND_PAGES, FREE_TREND_FEEDS, is_fashion_reference
|
||
from intelligence_web_translation import translation_fields
|
||
|
||
try:
|
||
import feedparser # type: ignore[import-not-found]
|
||
except ModuleNotFoundError:
|
||
feedparser = None # type: ignore[assignment]
|
||
|
||
try:
|
||
from dateutil import parser as dtparser # type: ignore[import-not-found]
|
||
except ModuleNotFoundError:
|
||
dtparser = None # type: ignore[assignment]
|
||
|
||
# ────────────── 常量 ──────────────
|
||
|
||
UTC = timezone.utc
|
||
BROWSER_UA = (
|
||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
|
||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
|
||
)
|
||
|
||
# 指定的 Dezeen Showroom HTML 爬取源
|
||
DEZEEN_SHOWROOM_URLS: list[dict[str, str]] = [
|
||
{"name": "Dezeen Showroom - Fabrics", "url": "https://www.dezeen.com/showroom/textiles-soft-furnishings/fabrics/"},
|
||
{"name": "Dezeen Showroom - Textiles & Furnishings", "url": "https://www.dezeen.com/showroom/textiles-soft-furnishings/"},
|
||
]
|
||
|
||
BLOCKED_PAID_TREND_MARKERS = ("91jiafang", "91家纺", "pop-fashion", "pop趋势网", "pop家居")
|
||
|
||
|
||
# ────────────── 数据结构 ──────────────
|
||
|
||
@dataclass
|
||
class TrendItem:
|
||
"""一条趋势数据。"""
|
||
source: str # 数据源名称
|
||
category: str # 分类名
|
||
category_key: str # 分类键
|
||
title: str
|
||
url: str
|
||
thumbnail: str = ""
|
||
image_original: str = ""
|
||
summary: str = ""
|
||
published_at: str | None = None
|
||
meta: dict[str, Any] = field(default_factory=dict)
|
||
|
||
|
||
# ────────────── 翻译逻辑 ──────────────
|
||
|
||
def translate_to_zh_cn(session: requests.Session, text: str) -> str | None:
|
||
s = (text or "").strip()
|
||
if not s:
|
||
return None
|
||
try:
|
||
import os
|
||
proxies = {}
|
||
proxy_url = os.environ.get("HTTP_PROXY") or os.environ.get("HTTPS_PROXY") or os.environ.get("http_proxy") or os.environ.get("https_proxy")
|
||
if proxy_url:
|
||
proxies = {"http": proxy_url, "https": proxy_url}
|
||
|
||
r = session.get(
|
||
"https://translate.googleapis.com/translate_a/single",
|
||
params={
|
||
"client": "gtx",
|
||
"sl": "auto",
|
||
"tl": "zh-CN",
|
||
"dt": "t",
|
||
"q": s,
|
||
},
|
||
proxies=proxies if proxies else None,
|
||
timeout=12,
|
||
)
|
||
r.raise_for_status()
|
||
payload = r.json()
|
||
if not isinstance(payload, list) or not payload:
|
||
return None
|
||
segs = payload[0]
|
||
if not isinstance(segs, list):
|
||
return None
|
||
translated = "".join(str(seg[0]) for seg in segs if isinstance(seg, list) and seg and seg[0])
|
||
translated = translated.strip()
|
||
if translated and translated != s:
|
||
return translated
|
||
except Exception as exc:
|
||
print(f" [翻译警告] '{s[:25]}...' 汉化失败: {exc}")
|
||
return None
|
||
return None
|
||
|
||
|
||
def load_zh_cache(output_dir: Path) -> dict[str, str]:
|
||
path = output_dir / "textile-zh-cache.json"
|
||
if not path.exists():
|
||
return {}
|
||
try:
|
||
data = json.loads(path.read_text(encoding="utf-8"))
|
||
if isinstance(data, dict):
|
||
return {str(k): str(v) for k, v in data.items() if str(k).strip() and str(v).strip()}
|
||
except Exception:
|
||
pass
|
||
return {}
|
||
|
||
|
||
def save_zh_cache(output_dir: Path, cache: dict[str, str]) -> None:
|
||
path = output_dir / "textile-zh-cache.json"
|
||
try:
|
||
path.write_text(json.dumps(cache, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
# ────────────── 工具函数 ──────────────
|
||
|
||
def utc_now() -> datetime:
|
||
return datetime.now(tz=UTC)
|
||
|
||
|
||
def iso(dt: datetime | None) -> str | None:
|
||
if not dt:
|
||
return None
|
||
return dt.astimezone(UTC).isoformat().replace("+00:00", "Z")
|
||
|
||
|
||
def make_trend_id(source: str, title: str, url: str) -> str:
|
||
"""生成趋势条目唯一 ID。"""
|
||
key = "||".join([source.strip().lower(), title.strip().lower(), url.strip().lower()])
|
||
return hashlib.sha1(key.encode("utf-8")).hexdigest()
|
||
|
||
|
||
VAM_GENERIC_TITLE_ZH = {
|
||
"hanging": "壁挂织物",
|
||
"pattern": "纹样图稿",
|
||
"embroidery sample": "刺绣样本",
|
||
"textile": "纺织藏品",
|
||
"embroidery": "刺绣作品",
|
||
"machine embroidered textile": "机绣织物",
|
||
}
|
||
|
||
|
||
def curate_vam_title(title: str) -> str:
|
||
match = re.fullmatch(r"\s*(.+?)\s+\[(O\d+)\]\s*", title, re.IGNORECASE)
|
||
if not match:
|
||
return ""
|
||
localized = VAM_GENERIC_TITLE_ZH.get(match.group(1).strip().casefold(), "")
|
||
return f"{localized}（V&A {match.group(2).upper()}）" if localized else ""
|
||
|
||
|
||
def parse_date_str(value: Any) -> str | None:
|
||
"""尝试解析日期字符串为 ISO 格式。"""
|
||
if value is None:
|
||
return None
|
||
s = str(value).strip()
|
||
if not s:
|
||
return None
|
||
|
||
# 尝试 dateutil 解析
|
||
if dtparser is not None:
|
||
try:
|
||
dt = dtparser.parse(s, tzinfos={"UT": 0, "UTC": 0, "GMT": 0})
|
||
if not dt.tzinfo:
|
||
dt = dt.replace(tzinfo=UTC)
|
||
return iso(dt)
|
||
except Exception:
|
||
pass
|
||
|
||
# 尝试常见中文日期格式: 2026-06-01 / 2026.06.01 / 2026年06月01日
|
||
m = re.search(r"(\d{4})[\-\.年/](\d{1,2})[\-\.月/](\d{1,2})", s)
|
||
if m:
|
||
try:
|
||
dt = datetime(int(m.group(1)), int(m.group(2)), int(m.group(3)), tzinfo=UTC)
|
||
return iso(dt)
|
||
except Exception:
|
||
pass
|
||
|
||
return None
|
||
|
||
|
||
def create_session() -> requests.Session:
|
||
"""创建带重试策略的 HTTP 会话。"""
|
||
session = requests.Session()
|
||
retry = Retry(
|
||
total=3,
|
||
connect=2,
|
||
read=2,
|
||
backoff_factor=2.0,
|
||
backoff_max=30,
|
||
status_forcelist=[429, 500, 502, 503, 504],
|
||
allowed_methods=frozenset(["GET", "POST"]),
|
||
)
|
||
adapter = HTTPAdapter(max_retries=retry)
|
||
session.mount("http://", adapter)
|
||
session.mount("https://", adapter)
|
||
session.headers.update({
|
||
"User-Agent": BROWSER_UA,
|
||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||
})
|
||
return session
|
||
|
||
|
||
# ────────────── 免费流行趋势 RSS 聚合 ──────────────
|
||
|
||
def fetch_free_trend_rss(session: requests.Session, now: datetime) -> list[TrendItem]:
|
||
"""通过 RSS 聚合各种国际免费的 Textiles / Fashion / Home Design 精选趋势渠道。"""
|
||
items: list[TrendItem] = []
|
||
if feedparser is None:
|
||
print(" [免费趋势RSS] feedparser 未安装，跳过 RSS 抓取")
|
||
return items
|
||
|
||
for feed_info in FREE_TREND_FEEDS:
|
||
feed_name = feed_info["name"]
|
||
feed_url = feed_info["url"]
|
||
category_name = str(feed_info.get("category") or "趋势博客")
|
||
category_key = str(feed_info.get("category_key") or "trend_blogs")
|
||
|
||
try:
|
||
r = session.get(feed_url, timeout=20)
|
||
r.raise_for_status()
|
||
parsed = feedparser.parse(r.content)
|
||
|
||
count = 0
|
||
for entry in parsed.entries[:15]: # 每个源最多取 15 条
|
||
title = str(entry.get("title", "")).strip()
|
||
link = str(entry.get("link", "")).strip()
|
||
if not title or not link:
|
||
continue
|
||
|
||
# 解析发布日期
|
||
published = None
|
||
for date_field in ("published", "updated", "pubDate"):
|
||
if entry.get(date_field):
|
||
published = parse_date_str(entry.get(date_field))
|
||
if published:
|
||
break
|
||
|
||
content_html = "".join(str(c.get("value") or "") for c in entry.get("content", []))
|
||
|
||
# 提取摘要
|
||
summary = ""
|
||
if entry.get("summary"):
|
||
raw = str(entry.get("summary", ""))
|
||
summary_text = BeautifulSoup(raw, "html.parser").get_text(" ", strip=True)
|
||
summary = re.split(r"\bThe post\b", summary_text, maxsplit=1, flags=re.I)[0].strip()[:200]
|
||
|
||
reference_text = BeautifulSoup(
|
||
str(entry.get("summary") or ""),
|
||
"html.parser",
|
||
).get_text(" ", strip=True)
|
||
if category_key == "fashion_trends" and not is_fashion_reference(title, reference_text):
|
||
continue
|
||
|
||
# 提取缩略图
|
||
thumbnail = ""
|
||
if entry.get("media_content"):
|
||
for mc in entry.media_content:
|
||
if isinstance(mc, dict) and mc.get("url"):
|
||
thumbnail = mc["url"]
|
||
break
|
||
if not thumbnail and entry.get("links"):
|
||
for lk in entry.links:
|
||
if isinstance(lk, dict) and lk.get("type", "").startswith("image"):
|
||
thumbnail = lk.get("href", "")
|
||
break
|
||
|
||
# 正则容错提取：从正文 HTML 中提取第一张 <img> 作为缩略图
|
||
if not thumbnail:
|
||
html_content = content_html or str(entry.get("summary") or "")
|
||
if html_content:
|
||
m_img = re.search(r'<img[^>]+src=["\']([^"\']+)["\']', html_content, re.I)
|
||
if m_img:
|
||
thumbnail = m_img.group(1)
|
||
|
||
items.append(TrendItem(
|
||
source=feed_name,
|
||
category=category_name,
|
||
category_key=category_key,
|
||
title=title,
|
||
url=link,
|
||
thumbnail=thumbnail,
|
||
summary=summary,
|
||
published_at=published,
|
||
))
|
||
count += 1
|
||
|
||
print(f" [免费趋势RSS] {feed_name}: {count} 条")
|
||
|
||
except Exception as exc:
|
||
print(f" [免费趋势RSS] {feed_name} 抓取失败: {exc}")
|
||
continue
|
||
|
||
return items
|
||
|
||
|
||
def fetch_curated_trend_pages(session: requests.Session, now: datetime) -> list[TrendItem]:
|
||
"""抓取无稳定 RSS、但可公开验证的免费趋势页面。"""
|
||
items: list[TrendItem] = []
|
||
for page in CURATED_TREND_PAGES:
|
||
try:
|
||
response = session.get(page["url"], timeout=30)
|
||
response.raise_for_status()
|
||
soup = BeautifulSoup(response.text, "html.parser")
|
||
title = (soup.select_one('meta[property="og:title"]') or {}).get("content", "")
|
||
summary = (soup.select_one('meta[property="og:description"]') or {}).get("content", "")
|
||
if not title and soup.title:
|
||
title = soup.title.get_text(" ", strip=True)
|
||
image = (soup.select_one('meta[property="og:image"]') or {}).get("content", "")
|
||
published = ""
|
||
for selector in ('meta[property="article:published_time"]', 'meta[name="date"]'):
|
||
node = soup.select_one(selector)
|
||
if node and node.get("content"):
|
||
published = str(node.get("content"))
|
||
break
|
||
product_image = soup.select_one('img[src*="m.media-amazon.com"]')
|
||
if product_image:
|
||
image = str(product_image.get("src") or "")
|
||
elif not image or image.endswith("/fb.png"):
|
||
image_node = soup.select_one("img[src]")
|
||
image = image_node.get("src", "") if image_node else ""
|
||
if not title or not summary:
|
||
raise ValueError("缺少可验证的标题或摘要")
|
||
items.append(TrendItem(
|
||
source=page["name"],
|
||
category=page["category"],
|
||
category_key=page["category_key"],
|
||
title=title.strip(),
|
||
url=page["url"],
|
||
thumbnail=urljoin(page["url"], image),
|
||
summary=summary.strip()[:300],
|
||
published_at=parse_date_str(published),
|
||
))
|
||
print(f" [精选趋势页] {page['name']}: 1 条")
|
||
except Exception as exc:
|
||
print(f" [精选趋势页] {page['name']} 抓取失败: {exc}")
|
||
return items
|
||
|
||
|
||
# ────────────── Dezeen Showroom 网页抓取 ──────────────
|
||
|
||
def fetch_dezeen_showroom(session: requests.Session, now: datetime) -> list[TrendItem]:
|
||
"""爬取 Dezeen Showroom 中的纺织面料与软装产品页面。"""
|
||
import random
|
||
items: list[TrendItem] = []
|
||
category_name = "趋势博客"
|
||
category_key = "trend_blogs"
|
||
|
||
for site in DEZEEN_SHOWROOM_URLS:
|
||
name = site["name"]
|
||
url = site["url"]
|
||
|
||
# 引入随机等待机制避开 WAF 的并发频率检测
|
||
time.sleep(random.uniform(1.2, 3.0))
|
||
|
||
# 注入高仿真 Referer 和 Accept 头，拟合正常浏览器点击跳转行为
|
||
headers = {
|
||
"Referer": "https://www.dezeen.com/showroom/",
|
||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
|
||
}
|
||
|
||
html_text = ""
|
||
max_retries = 3
|
||
for attempt in range(max_retries):
|
||
try:
|
||
r = session.get(url, headers=headers, timeout=30)
|
||
r.raise_for_status()
|
||
html_text = r.text
|
||
break
|
||
except Exception as exc:
|
||
if attempt < max_retries - 1:
|
||
wait_time = (attempt + 1) * 4
|
||
print(f" [Dezeen Showroom] {name} 请求异常 (尝试 {attempt+1}/{max_retries}): {exc}，等待 {wait_time} 秒后重试...")
|
||
time.sleep(wait_time)
|
||
else:
|
||
print(f" [Dezeen Showroom] {name} 抓取彻底失败: {exc}")
|
||
|
||
if not html_text:
|
||
continue
|
||
|
||
try:
|
||
soup = BeautifulSoup(html_text, "html.parser")
|
||
articles = soup.find_all("article")
|
||
count = 0
|
||
for art in articles:
|
||
title_el = art.find("h3") or art.find("h2") or art.find("a")
|
||
title = title_el.get_text(" ", strip=True) if title_el else ""
|
||
|
||
link_el = art.find("a")
|
||
link = link_el.get("href") if link_el else ""
|
||
if link and not link.startswith("http"):
|
||
link = urljoin("https://www.dezeen.com", link)
|
||
|
||
if not title or not link:
|
||
continue
|
||
|
||
# 从链接解析时间：形如 https://www.dezeen.com/2026/04/27/...
|
||
published = None
|
||
date_match = re.search(r"/(\d{4})/(\d{2})/(\d{2})/", link)
|
||
if date_match:
|
||
try:
|
||
dt = datetime(int(date_match.group(1)), int(date_match.group(2)), int(date_match.group(3)), tzinfo=timezone.utc)
|
||
published = iso(dt)
|
||
except Exception:
|
||
pass
|
||
|
||
img_el = art.find("img")
|
||
thumbnail = ""
|
||
if img_el:
|
||
thumbnail = img_el.get("src") or img_el.get("data-src") or ""
|
||
|
||
summary_el = art.find("p") or art.find(class_="summary")
|
||
summary = summary_el.get_text(" ", strip=True) if summary_el else ""
|
||
# 清理尾部
|
||
if "More about" in summary:
|
||
summary = summary.split("More about")[0].strip()
|
||
|
||
items.append(TrendItem(
|
||
source=name,
|
||
category=category_name,
|
||
category_key=category_key,
|
||
title=title,
|
||
url=link,
|
||
thumbnail=thumbnail,
|
||
summary=summary,
|
||
published_at=published,
|
||
))
|
||
count += 1
|
||
|
||
print(f" [Dezeen Showroom] {name}: {count} 条")
|
||
except Exception as exc:
|
||
print(f" [Dezeen Showroom] {name} 抓取失败: {exc}")
|
||
continue
|
||
|
||
return items
|
||
|
||
|
||
# ────────────── TrendHunter 容错采集 ──────────────
|
||
|
||
def fetch_trendhunter(session: requests.Session, now: datetime) -> list[TrendItem]:
|
||
"""容错尝试抓取 TrendHunter 流行趋势。若因 Akamai WAF 403 被阻断，将默默跳过不抛出异常。"""
|
||
items: list[TrendItem] = []
|
||
category_name = "趋势博客"
|
||
category_key = "trend_blogs"
|
||
source_name = "TrendHunter"
|
||
|
||
urls = [
|
||
"https://www.trendhunter.com/hipfashion",
|
||
"https://www.trendhunter.com/trends/textile-design",
|
||
"https://www.trendhunter.com/trends/premium-home-textiles",
|
||
"https://www.trendhunter.com/trends/furnishing-fabrics"
|
||
]
|
||
|
||
headers = {
|
||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
|
||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||
"Referer": "https://www.google.com/"
|
||
}
|
||
|
||
for url in urls:
|
||
slug = url.split("/")[-1]
|
||
try:
|
||
r = session.get(url, headers=headers, timeout=15)
|
||
if r.status_code == 403:
|
||
print(f" [TrendHunter] {slug} 抓取受限 (HTTP 403)，已自动跳过")
|
||
continue
|
||
r.raise_for_status()
|
||
|
||
soup = BeautifulSoup(r.text, "html.parser")
|
||
# 查找卡片元素，采用多重 fallback
|
||
cards = soup.select("div[class*='trendUnit'], div[class*='post-item'], .trend-card")
|
||
if not cards:
|
||
cards = [a for a in soup.find_all("a", href=True) if "/trends/" in a["href"] and len(a["href"]) > 20]
|
||
|
||
count = 0
|
||
for card in cards:
|
||
title = ""
|
||
href = ""
|
||
thumbnail = ""
|
||
summary = ""
|
||
|
||
if card.name == "a":
|
||
href = card["href"]
|
||
title = card.get("title", "").strip() or card.get_text(" ", strip=True)
|
||
img = card.find("img")
|
||
if img:
|
||
thumbnail = img.get("src") or img.get("data-src") or img.get("data-original") or ""
|
||
if not title:
|
||
title = img.get("alt", "").strip()
|
||
else:
|
||
a_link = card.find("a", href=True)
|
||
if a_link:
|
||
href = a_link["href"]
|
||
title = a_link.get("title", "").strip() or a_link.get_text(" ", strip=True)
|
||
|
||
img = card.find("img")
|
||
if img:
|
||
thumbnail = img.get("src") or img.get("data-src") or img.get("data-original") or ""
|
||
if not title:
|
||
title = img.get("alt", "").strip()
|
||
|
||
p = card.find("p")
|
||
if p:
|
||
summary = p.get_text(" ", strip=True)
|
||
|
||
if not href:
|
||
continue
|
||
|
||
if not href.startswith("http"):
|
||
href = urljoin("https://www.trendhunter.com", href)
|
||
|
||
# 标题兜底
|
||
if not title:
|
||
m = re.search(r"/trends/([^/?#]+)", href)
|
||
if m:
|
||
title = m.group(1).replace("-", " ").title()
|
||
|
||
items.append(TrendItem(
|
||
source=source_name,
|
||
category=category_name,
|
||
category_key=category_key,
|
||
title=title,
|
||
url=href,
|
||
thumbnail=thumbnail,
|
||
summary=summary,
|
||
published_at=iso(now),
|
||
))
|
||
count += 1
|
||
print(f" [TrendHunter] {slug} 成功抓取: {count} 条")
|
||
|
||
except Exception as exc:
|
||
print(f" [TrendHunter] {slug} 抓取异常 (已自动跳过): {exc}")
|
||
continue
|
||
|
||
return items
|
||
|
||
|
||
# ────────────── 博物馆灵感图库采集 ──────────────
|
||
|
||
def fetch_museum_images(session: requests.Session) -> list[TrendItem]:
|
||
"""通过大都会艺术博物馆 (The Met) 和维多利亚与阿尔伯特博物馆 (V&A) 的公共 API 获取高清家纺设计大图。"""
|
||
items: list[TrendItem] = []
|
||
category_name = "灵感图库"
|
||
category_key = "textile_gallery"
|
||
|
||
# 1. 抓取大都会艺术博物馆 (The Met) — 四大垂直主题并发抓取 (textile, tapestry, embroidery, brocade)
|
||
met_queries = ["textile", "tapestry", "embroidery", "brocade"]
|
||
|
||
def fetch_single_met_query(q):
|
||
target_ids = []
|
||
try:
|
||
search_url = f"https://collectionapi.metmuseum.org/public/collection/v1/search?q={q}&hasImages=true"
|
||
res = session.get(search_url, timeout=20)
|
||
if res.status_code == 200:
|
||
object_ids = res.json().get("objectIDs", [])
|
||
if object_ids:
|
||
target_ids = object_ids[:10] # 每个查询取前 10 个
|
||
except Exception as e:
|
||
print(f" [博物馆图库] The Met 子查询 '{q}' 失败: {e}")
|
||
return target_ids
|
||
|
||
# 并发获取所有 object ID 并去重
|
||
met_object_ids = []
|
||
with ThreadPoolExecutor(max_workers=4) as met_search_executor:
|
||
met_search_futures = [met_search_executor.submit(fetch_single_met_query, q) for q in met_queries]
|
||
for fut in as_completed(met_search_futures):
|
||
o_ids = fut.result()
|
||
for oid in o_ids:
|
||
if oid not in met_object_ids:
|
||
met_object_ids.append(oid)
|
||
|
||
# 并发请求 object 详情
|
||
def fetch_met_detail(obj_id):
|
||
detail_url = f"https://collectionapi.metmuseum.org/public/collection/v1/objects/{obj_id}"
|
||
try:
|
||
res = session.get(detail_url, timeout=12)
|
||
if res.status_code == 200:
|
||
return res.json()
|
||
except Exception:
|
||
pass
|
||
return None
|
||
|
||
textile_terms = (
|
||
"textile", "tapestry", "embroidery", "brocade", "fabric", "woven", "weaving",
|
||
"cloth", "velvet", "silk", "cotton", "linen", "wool", "lace", "carpet", "rug",
|
||
"quilt", "blanket", "coverlet", "bedcover", "curtain", "sampler",
|
||
)
|
||
|
||
met_count = 0
|
||
# 取前 30 个 object 详情，控制并发数量，防止 Met 的限流
|
||
with ThreadPoolExecutor(max_workers=5) as met_detail_executor:
|
||
met_detail_futures = {met_detail_executor.submit(fetch_met_detail, oid): oid for oid in met_object_ids[:30]}
|
||
for fut in as_completed(met_detail_futures):
|
||
detail = fut.result()
|
||
if detail and (detail.get("primaryImageSmall") or detail.get("primaryImage")):
|
||
title = str(detail.get("title") or "精美印花织物").strip()
|
||
material = " ".join(str(detail.get(key) or "") for key in ("objectName", "medium", "classification", "title")).lower()
|
||
if not any(term in material for term in textile_terms):
|
||
continue
|
||
culture = str(detail.get("culture") or "").strip()
|
||
period = str(detail.get("period") or "").strip()
|
||
date = str(detail.get("objectDate") or "").strip()
|
||
|
||
parts = ["大都会博物馆藏"]
|
||
if culture:
|
||
parts.append(f"工艺文化: {culture}")
|
||
if detail.get("medium"):
|
||
parts.append(f"材质: {str(detail['medium']).strip()}")
|
||
if period or date:
|
||
parts.append(f"年代: {period or date}")
|
||
summary = " | ".join(parts)
|
||
|
||
items.append(TrendItem(
|
||
source="The Met",
|
||
category=category_name,
|
||
category_key=category_key,
|
||
title=title,
|
||
url=f"https://www.metmuseum.org/art/collection/search/{detail.get('objectID')}",
|
||
thumbnail=detail.get("primaryImageSmall") or detail.get("primaryImage"),
|
||
image_original=detail.get("primaryImage") or detail.get("primaryImageSmall"),
|
||
summary=summary,
|
||
))
|
||
met_count += 1
|
||
print(f" [博物馆图库] The Met 成功提取并合并: {met_count} 张高清图")
|
||
|
||
# 2. 抓取维多利亚与阿尔伯特博物馆 (V&A) — 细分多渠道并发抓取 (技术、壁挂、纹样、刺绣)
|
||
va_queries = [
|
||
{"name": "V&A 工艺技术", "url": "https://api.vam.ac.uk/v2/objects/search?id_technique=x40351&q=textile&images_exist=1&page_size=15"},
|
||
{"name": "V&A 壁挂艺术", "url": "https://api.vam.ac.uk/v2/objects/search?q=Hanging&images_exist=1&page_size=15"},
|
||
{"name": "V&A 经典纹样", "url": "https://api.vam.ac.uk/v2/objects/search?q=Pattern&images_exist=1&page_size=15"},
|
||
{"name": "V&A 刺绣艺术", "url": "https://api.vam.ac.uk/v2/objects/search?q=embroidery&images_exist=1&page_size=15"},
|
||
]
|
||
|
||
def fetch_single_va_query(q_info):
|
||
va_items = []
|
||
try:
|
||
res = session.get(q_info["url"], timeout=20)
|
||
if res.status_code == 200:
|
||
records = res.json().get("records", [])
|
||
for rec in records:
|
||
img_info = rec.get("_images", {})
|
||
image_url = img_info.get("_primary_thumbnail")
|
||
if image_url:
|
||
thumb = image_url.replace("!100,100", "!600,")
|
||
original = image_url.replace("!100,100", "full")
|
||
else:
|
||
continue
|
||
|
||
primary_title = str(rec.get("_primaryTitle") or "").strip()
|
||
object_type = str(rec.get("objectType") or "").strip()
|
||
system_number = str(rec.get("systemNumber") or "").strip()
|
||
title = primary_title or object_type or "经典纺织藏品"
|
||
if not primary_title and system_number and object_type.casefold() in VAM_GENERIC_TITLE_ZH:
|
||
title = f"{object_type} [{system_number}]"
|
||
maker = rec.get("_primaryMaker", {}).get("name") if rec.get("_primaryMaker") else None
|
||
place = rec.get("_primaryPlace")
|
||
date = rec.get("_primaryDate")
|
||
|
||
parts = ["V&A 博物馆藏"]
|
||
if maker:
|
||
parts.append(f"制造/设计: {maker}")
|
||
if place or date:
|
||
parts.append(f"产地年代: {place or '未知'} ({date or '未知'})")
|
||
summary = " | ".join(parts)
|
||
|
||
va_items.append(TrendItem(
|
||
source="V&A Museum",
|
||
category=category_name,
|
||
category_key=category_key,
|
||
title=title,
|
||
url=f"https://collections.vam.ac.uk/item/{rec.get('systemNumber')}",
|
||
thumbnail=thumb,
|
||
image_original=original,
|
||
summary=summary,
|
||
))
|
||
else:
|
||
print(f" [博物馆图库] {q_info['name']} 返回状态受限 (HTTP {res.status_code})")
|
||
except Exception as e:
|
||
print(f" [博物馆图库] {q_info['name']} 请求异常: {e}")
|
||
return va_items
|
||
|
||
va_count = 0
|
||
with ThreadPoolExecutor(max_workers=4) as va_executor:
|
||
va_futures = [va_executor.submit(fetch_single_va_query, q) for q in va_queries]
|
||
for fut in as_completed(va_futures):
|
||
res_items = fut.result()
|
||
# 合并并去重图片/系统编号链接
|
||
for item in res_items:
|
||
if not any(x.url == item.url for x in items):
|
||
items.append(item)
|
||
va_count += 1
|
||
print(f" [博物馆图库] V&A Museum 成功提取并合并: {va_count} 张高清图")
|
||
|
||
return items
|
||
|
||
|
||
|
||
# ────────────── 数据汇总 & 输出 ──────────────
|
||
|
||
|
||
def collect_all_trends(session: requests.Session, now: datetime, cache: dict[str, str]) -> dict[str, Any]:
|
||
"""并发采集所有数据源，汇总为输出 payload。"""
|
||
|
||
print("\n[开始] 家纺流行趋势数据采集")
|
||
print("=" * 60)
|
||
|
||
all_items: list[TrendItem] = []
|
||
errors: list[str] = []
|
||
|
||
# POP 趋势网需要付费账号，本项目不再采集；图片搜索改为零抓取成本的前端快捷入口。
|
||
tasks = {
|
||
"免费趋势RSS": lambda: fetch_free_trend_rss(session, now),
|
||
"精选趋势页": lambda: fetch_curated_trend_pages(session, now),
|
||
"Dezeen展厅": lambda: fetch_dezeen_showroom(session, now),
|
||
"博物馆图库": lambda: fetch_museum_images(session),
|
||
"TrendHunter": lambda: fetch_trendhunter(session, now),
|
||
}
|
||
|
||
with ThreadPoolExecutor(max_workers=4) as executor:
|
||
futures = {executor.submit(fn): name for name, fn in tasks.items()}
|
||
for future in as_completed(futures):
|
||
name = futures[future]
|
||
try:
|
||
result = future.result(timeout=90)
|
||
all_items.extend(result)
|
||
except Exception as exc:
|
||
error_msg = f"{name}: {exc}"
|
||
errors.append(error_msg)
|
||
print(f" [ERROR] {error_msg}")
|
||
|
||
print(f"\n[汇总] 共采集 {len(all_items)} 条趋势数据")
|
||
|
||
# 按分类分组
|
||
categories: dict[str, dict[str, Any]] = {}
|
||
seen_urls: set[str] = set()
|
||
|
||
# 限制单次运行新翻译上限，防止高频调用被谷歌限制IP
|
||
max_new_translations = 250
|
||
translated_count = 0
|
||
|
||
# 优先将最新发布的文章排在前面，确保首屏最新文章必定获得翻译
|
||
all_items.sort(
|
||
key=lambda x: x.published_at or "1970-01-01",
|
||
reverse=True
|
||
)
|
||
|
||
for item in all_items:
|
||
blocked_text = f"{item.category_key} {item.source} {item.url}".lower()
|
||
if any(marker in blocked_text for marker in BLOCKED_PAID_TREND_MARKERS):
|
||
continue
|
||
# 针对灵感图库，如果没有缩略图，则直接跳过不录入，保障不出现空白卡片
|
||
if item.category_key == "textile_gallery" and not item.thumbnail:
|
||
continue
|
||
|
||
# 去重
|
||
if item.url in seen_urls:
|
||
continue
|
||
seen_urls.add(item.url)
|
||
|
||
title = item.title
|
||
summary = item.summary
|
||
title_original = ""
|
||
summary_original = ""
|
||
|
||
# 针对外部英文源以及灵感图库进行免费汉化翻译
|
||
if item.category_key in ("trend_blogs", "fashion_trends", "textile_gallery"):
|
||
if title:
|
||
title_original = title
|
||
curated_title = curate_vam_title(title) if item.source == "V&A Museum" else ""
|
||
if curated_title:
|
||
title = curated_title
|
||
elif title in cache:
|
||
title = cache[title]
|
||
elif translated_count < max_new_translations:
|
||
translated = translate_to_zh_cn(session, title)
|
||
if translated:
|
||
cache[title] = translated
|
||
title = translated
|
||
translated_count += 1
|
||
|
||
if summary:
|
||
summary_original = summary
|
||
if summary in cache:
|
||
summary = cache[summary]
|
||
elif translated_count < max_new_translations:
|
||
translated = translate_to_zh_cn(session, summary)
|
||
if translated:
|
||
cache[summary] = translated
|
||
summary = translated
|
||
translated_count += 1
|
||
|
||
if item.category_key not in categories:
|
||
categories[item.category_key] = {
|
||
"name": item.category,
|
||
"source": item.source,
|
||
"items": [],
|
||
}
|
||
|
||
id_title = re.sub(r"\s+\[O\d+\]\s*$", "", item.title, flags=re.IGNORECASE) if item.source == "V&A Museum" else item.title
|
||
item_dict = {
|
||
"id": make_trend_id(item.source, id_title, item.url),
|
||
"title": title,
|
||
"title_original": title_original,
|
||
"url": item.url,
|
||
"source": item.source,
|
||
"category": item.category,
|
||
"category_key": item.category_key,
|
||
"thumbnail": item.thumbnail,
|
||
"image_original": item.image_original,
|
||
"summary": summary,
|
||
"summary_original": summary_original,
|
||
"published_at": item.published_at,
|
||
}
|
||
item_dict.update(translation_fields(
|
||
item.url,
|
||
source_text=f"{title_original or item.title} {summary_original or item.summary}",
|
||
))
|
||
categories[item.category_key]["items"].append(item_dict)
|
||
|
||
# 馆藏没有新闻发布时间，按来源和永久链接稳定排序；其他分类仍按发布时间倒序。
|
||
for category_key, cat_data in categories.items():
|
||
if category_key == "textile_gallery":
|
||
cat_data["items"].sort(key=lambda x: (x.get("source", ""), x.get("url", "")))
|
||
else:
|
||
cat_data["items"].sort(
|
||
key=lambda x: (
|
||
bool(x.get("thumbnail")) if category_key == "fashion_trends" else False,
|
||
x.get("published_at") or "1970-01-01",
|
||
),
|
||
reverse=True,
|
||
)
|
||
cat_data["count"] = len(cat_data["items"])
|
||
|
||
total_items = sum(len(c["items"]) for c in categories.values())
|
||
|
||
payload = {
|
||
"generated_at": iso(now),
|
||
"total_items": total_items,
|
||
"category_count": len(categories),
|
||
"source_count": len({i.source for i in all_items}),
|
||
"categories": categories,
|
||
"errors": errors if errors else None,
|
||
}
|
||
|
||
return payload
|
||
|
||
|
||
def validate_payload(payload: dict[str, Any]) -> None:
|
||
def walk(value: Any, path: str = "$") -> None:
|
||
if isinstance(value, dict):
|
||
for key, item in value.items():
|
||
walk(item, f"{path}.{key}")
|
||
elif isinstance(value, list):
|
||
for index, item in enumerate(value):
|
||
walk(item, f"{path}[{index}]")
|
||
elif isinstance(value, str) and "????" in value:
|
||
raise ValueError(f"前台数据文本损坏: {path}")
|
||
|
||
walk(payload)
|
||
|
||
|
||
# ────────────── 主函数 ──────────────
|
||
|
||
def main() -> int:
|
||
parser = argparse.ArgumentParser(description="家纺流行趋势数据采集")
|
||
parser.add_argument("--output-dir", default="data", help="输出目录")
|
||
args = parser.parse_args()
|
||
|
||
now = utc_now()
|
||
output_dir = Path(args.output_dir)
|
||
output_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
output_path = output_dir / "textile-trends.json"
|
||
|
||
session = create_session()
|
||
cache = load_zh_cache(output_dir)
|
||
payload = collect_all_trends(session, now, cache)
|
||
validate_payload(payload)
|
||
|
||
output_path.write_text(
|
||
json.dumps(payload, ensure_ascii=False, indent=2),
|
||
encoding="utf-8",
|
||
)
|
||
save_zh_cache(output_dir, cache)
|
||
|
||
print(f"\n[完成] 数据已写入: {output_path}")
|
||
print(f" 总条目: {payload['total_items']}")
|
||
print(f" 分类数: {payload['category_count']}")
|
||
print(f" 数据源: {payload['source_count']}")
|
||
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|