import { db } from './db'; import type { MessageRow } from './messages-store'; export type MessageLinkSource = 'wechat_raw' | 'plain_url' | 'public_search' | 'manual'; export interface ParsedMessageLink { url: string; canonical_url: string; title: string | null; description: string | null; domain: string; source: MessageLinkSource; raw_kind: string; confidence: number; } type LinkInput = Pick< MessageRow, 'chatroom_id' | 'local_id' | 'date' | 'sender' | 'content' | 'time' | 'timestamp' >; export function decodeHtmlEntities(s: string): string { return s .replace(/&/g, '&') .replace(/</g, '<') .replace(/>/g, '>') .replace(/"/g, '"') .replace(/'/g, "'") .replace(/&#x([0-9a-f]+);/gi, (_, hex: string) => String.fromCodePoint(Number.parseInt(hex, 16))) .replace(/&#(\d+);/g, (_, num: string) => String.fromCodePoint(Number.parseInt(num, 10))); } export function cleanUrl(raw: string): string { return decodeHtmlEntities(raw) .replace(/[),,。;;!?!?、\]}>]+$/g, '') .replace(/\.{3,}$/g, '') .trim(); } export function normalizeUrl(raw: string): string | null { if (!raw || raw.includes('...') || raw.includes('…')) return null; try { const u = new URL(cleanUrl(raw)); u.hash = ''; for (const key of ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content']) { u.searchParams.delete(key); } return u.toString(); } catch { return null; } } export function domainOf(url: string): string { try { return new URL(url).hostname.replace(/^www\./, ''); } catch { return ''; } } function isWechatArticleUrl(url: string): boolean { try { const u = new URL(cleanUrl(url)); return u.hostname === 'mp.weixin.qq.com' && (/^\/s\/?/.test(u.pathname) || u.searchParams.has('__biz')); } catch { return false; } } function tagText(content: string, tag: string): string { const text = content.match(new RegExp(`<${tag}[^>]*>([\\s\\S]*?)<\\/${tag}>`, 'i'))?.[1] ?? ''; return decodeHtmlEntities(text) .replace(//g, '$1') .replace(/\s+/g, ' ') .trim(); } function attrValues(content: string, attr: string): string[] { return Array.from(content.matchAll(new RegExp(`${attr}=["']([^"']+)["']`, 'gi'))) .map((m) => decodeHtmlEntities(m[1]).trim()) .filter(Boolean); } function titleFromContext(content: string, url: string): string | null { const xmlTitle = tagText(content, 'title'); if (xmlTitle) return xmlTitle.slice(0, 160); const decoded = decodeHtmlEntities(content).replace(/<\?xml[\s\S]+?<\/msg>/g, ' '); const lines = decoded .split(/\n+/) .map((line) => line .replace(url, '') .replace(/https?:\/\/\S+/g, '') .replace(/\[引用\]/g, '') .replace(/^\s*↳\s*/, '') .replace(/^\s*\[链接\]\s*/, '') .trim(), ) .filter((line) => line.length >= 4 && line.length <= 120); return lines.find((line) => !/^[@#\d\s::-]+$/.test(line))?.slice(0, 160) ?? null; } export function extractMessageLinks(content: string): ParsedMessageLink[] { const decoded = decodeHtmlEntities(content); const hasXml = /]|]/i.test(decoded); const xmlTitle = tagText(decoded, 'title') || null; const xmlDescription = tagText(decoded, 'des') || tagText(decoded, 'digest') || null; const candidates: Array<{ url: string; source: MessageLinkSource; raw_kind: string; confidence: number }> = []; for (const tag of ['url', 'lowurl']) { const url = tagText(decoded, tag); if (url && isWechatArticleUrl(url)) { candidates.push({ url, source: 'wechat_raw', raw_kind: `appmsg_${tag}`, confidence: 1 }); } } for (const value of attrValues(decoded, 'url')) { if (!isWechatArticleUrl(value)) continue; candidates.push({ url: value, source: hasXml ? 'wechat_raw' : 'plain_url', raw_kind: hasXml ? 'appmsg_attr_url' : 'plain_attr_url', confidence: hasXml ? 0.98 : 0.9, }); } for (const m of decoded.matchAll(/https?:\/\/[^\s<>"']+/g)) { const rawUrl = cleanUrl(m[0]); const article = isWechatArticleUrl(rawUrl); const source: MessageLinkSource = hasXml && article ? 'wechat_raw' : 'plain_url'; candidates.push({ url: rawUrl, source, raw_kind: hasXml && article ? 'appmsg_url_text' : 'plain_url', confidence: article ? 0.96 : 0.9, }); } const out = new Map(); for (const c of candidates) { const canonical = normalizeUrl(c.url); if (!canonical) continue; const domain = domainOf(canonical); if (!domain) continue; const existing = out.get(canonical); if (existing && existing.confidence >= c.confidence) continue; out.set(canonical, { url: cleanUrl(c.url), canonical_url: canonical, title: xmlTitle ?? titleFromContext(decoded, c.url), description: xmlDescription, domain, source: c.source, raw_kind: c.raw_kind, confidence: c.confidence, }); } return Array.from(out.values()); } const upsertMessageLink = () => db().prepare(` INSERT INTO message_links ( chatroom_id, local_id, date, sender, time, timestamp, url, canonical_url, title, description, domain, source, raw_kind, confidence, created_at ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(chatroom_id, local_id, canonical_url) DO UPDATE SET url = excluded.url, title = COALESCE(excluded.title, message_links.title), description = COALESCE(excluded.description, message_links.description), domain = excluded.domain, source = excluded.source, raw_kind = excluded.raw_kind, confidence = excluded.confidence `); export function upsertLinksForMessage(message: LinkInput): number { const links = extractMessageLinks(message.content); if (links.length === 0) return 0; const stmt = upsertMessageLink(); let changed = 0; for (const link of links) { const r = stmt.run( message.chatroom_id, message.local_id, message.date, message.sender ?? '', message.time ?? '', message.timestamp ?? 0, link.url, link.canonical_url, link.title, link.description, link.domain, link.source, link.raw_kind, link.confidence, Date.now(), ); changed += r.changes; } return changed; } export function upsertResolvedLinkForMessage(input: { chatroom_id: string; local_id: number; url: string; title?: string | null; description?: string | null; source: Extract; confidence?: number; }): { ok: boolean; error?: string } { const message = db() .prepare( `SELECT chatroom_id, local_id, sender, content, time, timestamp, type, date FROM messages WHERE chatroom_id = ? AND local_id = ?`, ) .get(input.chatroom_id, input.local_id) as MessageRow | undefined; if (!message) return { ok: false, error: 'message not found' }; const canonical = normalizeUrl(input.url); if (!canonical) return { ok: false, error: 'invalid url' }; const domain = domainOf(canonical); if (!domain) return { ok: false, error: 'invalid domain' }; upsertMessageLink().run( message.chatroom_id, message.local_id, message.date, message.sender ?? '', message.time ?? '', message.timestamp ?? 0, cleanUrl(input.url), canonical, input.title?.trim() || titleFromContext(message.content, input.url), input.description?.trim() || null, domain, input.source, input.source, input.confidence ?? (input.source === 'manual' ? 0.95 : 0.72), Date.now(), ); return { ok: true }; } export function backfillMessageLinks(since?: string, until?: string): { scanned: number; links: number } { const clauses = ["(content LIKE '%http%' OR content LIKE '%%' OR content LIKE '%imgsourceurl=%')"]; const params: string[] = []; if (since) { clauses.push('date >= ?'); params.push(since); } if (until) { clauses.push('date <= ?'); params.push(until); } const rows = db() .prepare( `SELECT chatroom_id, local_id, sender, content, time, timestamp, type, date FROM messages WHERE ${clauses.join(' AND ')} ORDER BY timestamp DESC`, ) .all(...params) as MessageRow[]; let links = 0; const tx = db().transaction(() => { db() .prepare( `DELETE FROM message_links WHERE source = 'wechat_raw' AND canonical_url NOT LIKE '%://mp.weixin.qq.com/%'`, ) .run(); for (const row of rows) links += upsertLinksForMessage(row); }); tx(); return { scanned: rows.length, links }; }