244 lines
8.7 KiB
Python
244 lines
8.7 KiB
Python
|
"""权威源 HTTP 获取、证据词校验与稳定内容哈希。"""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import hashlib
|
||
|
|
import json
|
||
|
|
import re
|
||
|
|
import ssl
|
||
|
|
import time
|
||
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
|
|
from dataclasses import dataclass
|
||
|
|
from datetime import datetime, timezone
|
||
|
|
from urllib.parse import urljoin
|
||
|
|
|
||
|
|
import requests
|
||
|
|
from bs4 import BeautifulSoup
|
||
|
|
from requests.adapters import HTTPAdapter
|
||
|
|
from urllib3.util.retry import Retry
|
||
|
|
|
||
|
|
if __package__:
|
||
|
|
from .intelligence_sources import SOURCES, SourceSpec
|
||
|
|
else:
|
||
|
|
from intelligence_sources import SOURCES, SourceSpec
|
||
|
|
|
||
|
|
|
||
|
|
UTC = timezone.utc
|
||
|
|
BROWSER_UA = (
|
||
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||
|
|
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0 Safari/537.36"
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
class LegacyCertificateAdapter(HTTPAdapter):
|
||
|
|
"""仍校验 CA 与域名，仅兼容少数政府站点缺少 SKI 的旧证书。"""
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _context():
|
||
|
|
context = ssl.create_default_context()
|
||
|
|
if hasattr(ssl, "VERIFY_X509_STRICT"):
|
||
|
|
context.verify_flags &= ~ssl.VERIFY_X509_STRICT
|
||
|
|
return context
|
||
|
|
|
||
|
|
def init_poolmanager(self, *args, **kwargs):
|
||
|
|
kwargs["ssl_context"] = self._context()
|
||
|
|
return super().init_poolmanager(*args, **kwargs)
|
||
|
|
|
||
|
|
def proxy_manager_for(self, proxy, **proxy_kwargs):
|
||
|
|
proxy_kwargs["proxy_ssl_context"] = self._context()
|
||
|
|
proxy_kwargs["ssl_context"] = self._context()
|
||
|
|
return super().proxy_manager_for(proxy, **proxy_kwargs)
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass
|
||
|
|
class FetchResult:
|
||
|
|
source: SourceSpec
|
||
|
|
ok: bool
|
||
|
|
status_code: int | None
|
||
|
|
final_url: str
|
||
|
|
content_type: str
|
||
|
|
elapsed_ms: int
|
||
|
|
size_bytes: int
|
||
|
|
content_hash: str
|
||
|
|
checked_at: str
|
||
|
|
title: str = ""
|
||
|
|
description: str = ""
|
||
|
|
image: str = ""
|
||
|
|
published_at: str = ""
|
||
|
|
page_language: str = ""
|
||
|
|
evidence: list[str] | None = None
|
||
|
|
error: str = ""
|
||
|
|
raw: bytes = b""
|
||
|
|
changed: bool = False
|
||
|
|
|
||
|
|
|
||
|
|
def iso_now() -> str:
|
||
|
|
return datetime.now(tz=UTC).isoformat().replace("+00:00", "Z")
|
||
|
|
|
||
|
|
|
||
|
|
def make_session() -> requests.Session:
|
||
|
|
session = requests.Session()
|
||
|
|
retry = Retry(
|
||
|
|
total=3,
|
||
|
|
connect=3,
|
||
|
|
read=3,
|
||
|
|
backoff_factor=1.2,
|
||
|
|
status_forcelist=(429, 500, 502, 503, 504),
|
||
|
|
allowed_methods=frozenset(("GET",)),
|
||
|
|
)
|
||
|
|
session.mount("http://", HTTPAdapter(max_retries=retry))
|
||
|
|
session.mount("https://", HTTPAdapter(max_retries=retry))
|
||
|
|
session.mount("https://www.bsmi.gov.tw", LegacyCertificateAdapter(max_retries=retry))
|
||
|
|
session.headers.update({
|
||
|
|
"User-Agent": BROWSER_UA,
|
||
|
|
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||
|
|
"Accept": "text/html,application/json;q=0.9,*/*;q=0.8",
|
||
|
|
})
|
||
|
|
return session
|
||
|
|
|
||
|
|
|
||
|
|
def normalize_text(value: str) -> str:
|
||
|
|
return " ".join((value or "").replace("\u00a0", " ").split())
|
||
|
|
|
||
|
|
|
||
|
|
def fetch_json(url: str):
|
||
|
|
response = make_session().get(url, timeout=(10, 35))
|
||
|
|
response.raise_for_status()
|
||
|
|
return response.json()
|
||
|
|
|
||
|
|
|
||
|
|
def _meta_content(soup: BeautifulSoup, *selectors: str) -> str:
|
||
|
|
for selector in selectors:
|
||
|
|
node = soup.select_one(selector)
|
||
|
|
if node and node.get("content"):
|
||
|
|
return normalize_text(str(node.get("content")))
|
||
|
|
return ""
|
||
|
|
|
||
|
|
|
||
|
|
def _parse_html_metadata(raw: bytes, base_url: str) -> tuple[str, str, str, str, str, str]:
|
||
|
|
soup = BeautifulSoup(raw, "html.parser")
|
||
|
|
page_language = normalize_text(str(soup.html.get("lang") or "")) if soup.html else ""
|
||
|
|
title = _meta_content(soup, 'meta[property="og:title"]', 'meta[name="twitter:title"]')
|
||
|
|
if not title and soup.title:
|
||
|
|
title = normalize_text(soup.title.get_text(" ", strip=True))
|
||
|
|
description = _meta_content(
|
||
|
|
soup,
|
||
|
|
'meta[property="og:description"]',
|
||
|
|
'meta[name="description"]',
|
||
|
|
'meta[name="twitter:description"]',
|
||
|
|
)
|
||
|
|
image = _meta_content(soup, 'meta[property="og:image"]', 'meta[name="twitter:image"]')
|
||
|
|
if image:
|
||
|
|
image = urljoin(base_url, image)
|
||
|
|
published_at = _meta_content(
|
||
|
|
soup,
|
||
|
|
'meta[property="article:published_time"]',
|
||
|
|
'meta[name="date"]',
|
||
|
|
'meta[name="DC.date"]',
|
||
|
|
)
|
||
|
|
for node in soup(("script", "style", "noscript", "svg")):
|
||
|
|
node.decompose()
|
||
|
|
text = normalize_text(soup.get_text(" ", strip=True))
|
||
|
|
return title[:240], description[:600], image[:1000], published_at[:80], text, page_language[:40]
|
||
|
|
|
||
|
|
|
||
|
|
def _stable_content_hash(raw: bytes, content_type: str, semantic_text: str = "") -> str:
|
||
|
|
if "json" in content_type:
|
||
|
|
try:
|
||
|
|
payload = json.loads(raw.decode("utf-8"))
|
||
|
|
canonical = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||
|
|
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||
|
|
except (UnicodeDecodeError, json.JSONDecodeError):
|
||
|
|
pass
|
||
|
|
if semantic_text:
|
||
|
|
stable_text = normalize_text(semantic_text)
|
||
|
|
stable_text = re.sub(r"\b\d{1,2}:\d{2}(?::\d{2})?\b", "", stable_text)
|
||
|
|
stable_text = re.sub(r"\b\d{8,}\b", "", stable_text)
|
||
|
|
return hashlib.sha256(stable_text[:2_000_000].encode("utf-8")).hexdigest()
|
||
|
|
return hashlib.sha256(raw).hexdigest()
|
||
|
|
|
||
|
|
|
||
|
|
def fetch_source(source: SourceSpec) -> FetchResult:
|
||
|
|
started = time.perf_counter()
|
||
|
|
checked_at = iso_now()
|
||
|
|
try:
|
||
|
|
response = make_session().get(source.url, timeout=(12, 45), allow_redirects=True)
|
||
|
|
elapsed_ms = round((time.perf_counter() - started) * 1000)
|
||
|
|
raw = response.content
|
||
|
|
content_type = response.headers.get("content-type", "").lower()
|
||
|
|
title = description = image = published_at = page_language = ""
|
||
|
|
evidence_text = ""
|
||
|
|
if "html" in content_type or raw.lstrip().startswith(b"<"):
|
||
|
|
title, description, image, published_at, evidence_text, page_language = _parse_html_metadata(raw, response.url)
|
||
|
|
evidence_text = f"{title} {description} {evidence_text}"
|
||
|
|
elif "json" in content_type:
|
||
|
|
payload = response.json()
|
||
|
|
if isinstance(payload, str) and "<html" in payload[:2_000].lower():
|
||
|
|
title, description, image, published_at, evidence_text, page_language = _parse_html_metadata(payload.encode("utf-8"), response.url)
|
||
|
|
evidence_text = f"{title} {description} {evidence_text}"
|
||
|
|
else:
|
||
|
|
evidence_text = raw[:2_000_000].decode("utf-8", errors="ignore")
|
||
|
|
elif source.extractor == "pdf":
|
||
|
|
evidence_text = response.url
|
||
|
|
else:
|
||
|
|
evidence_text = raw[:2_000_000].decode("utf-8", errors="ignore")
|
||
|
|
|
||
|
|
matched = [token for token in source.expected if token.lower() in evidence_text.lower()]
|
||
|
|
ok = response.ok and len(raw) >= 200 and len(matched) == len(source.expected)
|
||
|
|
if not response.ok:
|
||
|
|
error = f"HTTP {response.status_code}"
|
||
|
|
elif len(raw) < 200:
|
||
|
|
error = "响应正文过短"
|
||
|
|
elif len(matched) != len(source.expected):
|
||
|
|
missing = [token for token in source.expected if token.lower() not in evidence_text.lower()]
|
||
|
|
error = f"缺少证据词：{', '.join(missing)}"
|
||
|
|
else:
|
||
|
|
error = ""
|
||
|
|
|
||
|
|
hash_text = f"{title} {description}" if source.change_detection == "metadata" else evidence_text
|
||
|
|
return FetchResult(
|
||
|
|
source=source,
|
||
|
|
ok=ok,
|
||
|
|
status_code=response.status_code,
|
||
|
|
final_url=response.url,
|
||
|
|
content_type=content_type.split(";")[0],
|
||
|
|
elapsed_ms=elapsed_ms,
|
||
|
|
size_bytes=len(raw),
|
||
|
|
content_hash=_stable_content_hash(raw, content_type, hash_text),
|
||
|
|
checked_at=checked_at,
|
||
|
|
title=title,
|
||
|
|
description=description,
|
||
|
|
image=image,
|
||
|
|
published_at=published_at,
|
||
|
|
page_language=page_language,
|
||
|
|
evidence=matched,
|
||
|
|
error=error,
|
||
|
|
raw=raw,
|
||
|
|
)
|
||
|
|
except Exception as exc:
|
||
|
|
return FetchResult(
|
||
|
|
source=source,
|
||
|
|
ok=False,
|
||
|
|
status_code=None,
|
||
|
|
final_url=source.url,
|
||
|
|
content_type="",
|
||
|
|
elapsed_ms=round((time.perf_counter() - started) * 1000),
|
||
|
|
size_bytes=0,
|
||
|
|
content_hash="",
|
||
|
|
checked_at=checked_at,
|
||
|
|
evidence=[],
|
||
|
|
error=str(exc)[:300],
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def fetch_all_sources(max_workers: int) -> dict[str, FetchResult]:
|
||
|
|
results: dict[str, FetchResult] = {}
|
||
|
|
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||
|
|
futures = {executor.submit(fetch_source, source): source for source in SOURCES}
|
||
|
|
for future in as_completed(futures):
|
||
|
|
result = future.result()
|
||
|
|
results[result.source.id] = result
|
||
|
|
marker = "通过" if result.ok else "失败"
|
||
|
|
print(f"[{marker}] {result.source.name}: {result.status_code or '-'} / {result.elapsed_ms}ms {result.error}")
|
||
|
|
return results
|