feat: 发布三笙AI情报局独立版公司研发基线
同步家纺情报 v2 数据、59 个严格权威源、趋势图库、移动端交互、Google/Yandex 翻译门控及静态 SEO/GEO 构建。 补充面向 C# 研发的 README，明确当前无 LLM、多模态与生图模型，记录翻译请求参数、缓存与失败策略、数据处理边界、前端职责和无 GitHub Pages 部署流程。 新增按日期倒序维护的 CHANGELOG，记录影响、验证与代码/数据/dist 回滚方法。 验证：11 个 Python 契约测试、13 个 Node 前端测试通过；静态构建生成 1096 个 Sitemap URL。
This commit is contained in:
Jack
@@ -0,0 +1,243 @@
|
||||
"""权威源 HTTP 获取、证据词校验与稳定内容哈希。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import ssl
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util.retry import Retry
|
||||
|
||||
if __package__:
|
||||
from .intelligence_sources import SOURCES, SourceSpec
|
||||
else:
|
||||
from intelligence_sources import SOURCES, SourceSpec
|
||||
|
||||
|
||||
UTC = timezone.utc
|
||||
BROWSER_UA = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0 Safari/537.36"
|
||||
)
|
||||
|
||||
|
||||
class LegacyCertificateAdapter(HTTPAdapter):
|
||||
"""仍校验 CA 与域名，仅兼容少数政府站点缺少 SKI 的旧证书。"""
|
||||
|
||||
@staticmethod
|
||||
def _context():
|
||||
context = ssl.create_default_context()
|
||||
if hasattr(ssl, "VERIFY_X509_STRICT"):
|
||||
context.verify_flags &= ~ssl.VERIFY_X509_STRICT
|
||||
return context
|
||||
|
||||
def init_poolmanager(self, *args, **kwargs):
|
||||
kwargs["ssl_context"] = self._context()
|
||||
return super().init_poolmanager(*args, **kwargs)
|
||||
|
||||
def proxy_manager_for(self, proxy, **proxy_kwargs):
|
||||
proxy_kwargs["proxy_ssl_context"] = self._context()
|
||||
proxy_kwargs["ssl_context"] = self._context()
|
||||
return super().proxy_manager_for(proxy, **proxy_kwargs)
|
||||
|
||||
|
||||
@dataclass
|
||||
class FetchResult:
|
||||
source: SourceSpec
|
||||
ok: bool
|
||||
status_code: int | None
|
||||
final_url: str
|
||||
content_type: str
|
||||
elapsed_ms: int
|
||||
size_bytes: int
|
||||
content_hash: str
|
||||
checked_at: str
|
||||
title: str = ""
|
||||
description: str = ""
|
||||
image: str = ""
|
||||
published_at: str = ""
|
||||
page_language: str = ""
|
||||
evidence: list[str] | None = None
|
||||
error: str = ""
|
||||
raw: bytes = b""
|
||||
changed: bool = False
|
||||
|
||||
|
||||
def iso_now() -> str:
|
||||
return datetime.now(tz=UTC).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def make_session() -> requests.Session:
|
||||
session = requests.Session()
|
||||
retry = Retry(
|
||||
total=3,
|
||||
connect=3,
|
||||
read=3,
|
||||
backoff_factor=1.2,
|
||||
status_forcelist=(429, 500, 502, 503, 504),
|
||||
allowed_methods=frozenset(("GET",)),
|
||||
)
|
||||
session.mount("http://", HTTPAdapter(max_retries=retry))
|
||||
session.mount("https://", HTTPAdapter(max_retries=retry))
|
||||
session.mount("https://www.bsmi.gov.tw", LegacyCertificateAdapter(max_retries=retry))
|
||||
session.headers.update({
|
||||
"User-Agent": BROWSER_UA,
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
"Accept": "text/html,application/json;q=0.9,*/*;q=0.8",
|
||||
})
|
||||
return session
|
||||
|
||||
|
||||
def normalize_text(value: str) -> str:
|
||||
return " ".join((value or "").replace("\u00a0", " ").split())
|
||||
|
||||
|
||||
def fetch_json(url: str):
|
||||
response = make_session().get(url, timeout=(10, 35))
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
|
||||
def _meta_content(soup: BeautifulSoup, *selectors: str) -> str:
|
||||
for selector in selectors:
|
||||
node = soup.select_one(selector)
|
||||
if node and node.get("content"):
|
||||
return normalize_text(str(node.get("content")))
|
||||
return ""
|
||||
|
||||
|
||||
def _parse_html_metadata(raw: bytes, base_url: str) -> tuple[str, str, str, str, str, str]:
|
||||
soup = BeautifulSoup(raw, "html.parser")
|
||||
page_language = normalize_text(str(soup.html.get("lang") or "")) if soup.html else ""
|
||||
title = _meta_content(soup, 'meta[property="og:title"]', 'meta[name="twitter:title"]')
|
||||
if not title and soup.title:
|
||||
title = normalize_text(soup.title.get_text(" ", strip=True))
|
||||
description = _meta_content(
|
||||
soup,
|
||||
'meta[property="og:description"]',
|
||||
'meta[name="description"]',
|
||||
'meta[name="twitter:description"]',
|
||||
)
|
||||
image = _meta_content(soup, 'meta[property="og:image"]', 'meta[name="twitter:image"]')
|
||||
if image:
|
||||
image = urljoin(base_url, image)
|
||||
published_at = _meta_content(
|
||||
soup,
|
||||
'meta[property="article:published_time"]',
|
||||
'meta[name="date"]',
|
||||
'meta[name="DC.date"]',
|
||||
)
|
||||
for node in soup(("script", "style", "noscript", "svg")):
|
||||
node.decompose()
|
||||
text = normalize_text(soup.get_text(" ", strip=True))
|
||||
return title[:240], description[:600], image[:1000], published_at[:80], text, page_language[:40]
|
||||
|
||||
|
||||
def _stable_content_hash(raw: bytes, content_type: str, semantic_text: str = "") -> str:
|
||||
if "json" in content_type:
|
||||
try:
|
||||
payload = json.loads(raw.decode("utf-8"))
|
||||
canonical = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||||
except (UnicodeDecodeError, json.JSONDecodeError):
|
||||
pass
|
||||
if semantic_text:
|
||||
stable_text = normalize_text(semantic_text)
|
||||
stable_text = re.sub(r"\b\d{1,2}:\d{2}(?::\d{2})?\b", "", stable_text)
|
||||
stable_text = re.sub(r"\b\d{8,}\b", "", stable_text)
|
||||
return hashlib.sha256(stable_text[:2_000_000].encode("utf-8")).hexdigest()
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def fetch_source(source: SourceSpec) -> FetchResult:
|
||||
started = time.perf_counter()
|
||||
checked_at = iso_now()
|
||||
try:
|
||||
response = make_session().get(source.url, timeout=(12, 45), allow_redirects=True)
|
||||
elapsed_ms = round((time.perf_counter() - started) * 1000)
|
||||
raw = response.content
|
||||
content_type = response.headers.get("content-type", "").lower()
|
||||
title = description = image = published_at = page_language = ""
|
||||
evidence_text = ""
|
||||
if "html" in content_type or raw.lstrip().startswith(b"<"):
|
||||
title, description, image, published_at, evidence_text, page_language = _parse_html_metadata(raw, response.url)
|
||||
evidence_text = f"{title} {description} {evidence_text}"
|
||||
elif "json" in content_type:
|
||||
payload = response.json()
|
||||
if isinstance(payload, str) and "<html" in payload[:2_000].lower():
|
||||
title, description, image, published_at, evidence_text, page_language = _parse_html_metadata(payload.encode("utf-8"), response.url)
|
||||
evidence_text = f"{title} {description} {evidence_text}"
|
||||
else:
|
||||
evidence_text = raw[:2_000_000].decode("utf-8", errors="ignore")
|
||||
elif source.extractor == "pdf":
|
||||
evidence_text = response.url
|
||||
else:
|
||||
evidence_text = raw[:2_000_000].decode("utf-8", errors="ignore")
|
||||
|
||||
matched = [token for token in source.expected if token.lower() in evidence_text.lower()]
|
||||
ok = response.ok and len(raw) >= 200 and len(matched) == len(source.expected)
|
||||
if not response.ok:
|
||||
error = f"HTTP {response.status_code}"
|
||||
elif len(raw) < 200:
|
||||
error = "响应正文过短"
|
||||
elif len(matched) != len(source.expected):
|
||||
missing = [token for token in source.expected if token.lower() not in evidence_text.lower()]
|
||||
error = f"缺少证据词：{', '.join(missing)}"
|
||||
else:
|
||||
error = ""
|
||||
|
||||
hash_text = f"{title} {description}" if source.change_detection == "metadata" else evidence_text
|
||||
return FetchResult(
|
||||
source=source,
|
||||
ok=ok,
|
||||
status_code=response.status_code,
|
||||
final_url=response.url,
|
||||
content_type=content_type.split(";")[0],
|
||||
elapsed_ms=elapsed_ms,
|
||||
size_bytes=len(raw),
|
||||
content_hash=_stable_content_hash(raw, content_type, hash_text),
|
||||
checked_at=checked_at,
|
||||
title=title,
|
||||
description=description,
|
||||
image=image,
|
||||
published_at=published_at,
|
||||
page_language=page_language,
|
||||
evidence=matched,
|
||||
error=error,
|
||||
raw=raw,
|
||||
)
|
||||
except Exception as exc:
|
||||
return FetchResult(
|
||||
source=source,
|
||||
ok=False,
|
||||
status_code=None,
|
||||
final_url=source.url,
|
||||
content_type="",
|
||||
elapsed_ms=round((time.perf_counter() - started) * 1000),
|
||||
size_bytes=0,
|
||||
content_hash="",
|
||||
checked_at=checked_at,
|
||||
evidence=[],
|
||||
error=str(exc)[:300],
|
||||
)
|
||||
|
||||
|
||||
def fetch_all_sources(max_workers: int) -> dict[str, FetchResult]:
|
||||
results: dict[str, FetchResult] = {}
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
futures = {executor.submit(fetch_source, source): source for source in SOURCES}
|
||||
for future in as_completed(futures):
|
||||
result = future.result()
|
||||
results[result.source.id] = result
|
||||
marker = "通过" if result.ok else "失败"
|
||||
print(f"[{marker}] {result.source.name}: {result.status_code or '-'} / {result.elapsed_ms}ms {result.error}")
|
||||
return results
|
||||
Reference in New Issue
Block a user