mirror of
https://github.com/joeseesun/wechat-radar.git
synced 2026-09-08 03:38:30 +09:00
Initial open source release
This commit is contained in:
joeseesun
+496
@@ -0,0 +1,496 @@
|
||||
import { spawn } from 'node:child_process';
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { db } from './db';
|
||||
import { wxSessions } from './wx';
|
||||
|
||||
const MIN_MESSAGES_PER_TOPIC = 4;
|
||||
const MIN_MESSAGE_LENGTH = 20;
|
||||
const MAX_MESSAGE_LENGTH = 400;
|
||||
const MAX_MESSAGES_TO_PROCESS = 3000;
|
||||
const MAX_TOPICS_TO_SAVE = 30;
|
||||
const CODEX_CHUNK_SIZE = Number(process.env.WECHAT_RADAR_TOPIC_CHUNK_SIZE ?? 250);
|
||||
const CODEX_TIMEOUT_MS = Number(process.env.WECHAT_RADAR_CODEX_TIMEOUT_MS ?? 300_000);
|
||||
const CODEX_MODEL = process.env.WECHAT_RADAR_CODEX_MODEL;
|
||||
const TOPICS_PER_CHUNK = 12;
|
||||
|
||||
interface SourceMsg {
|
||||
chatroom_id: string;
|
||||
local_id: number;
|
||||
sender: string;
|
||||
content: string;
|
||||
time: string;
|
||||
timestamp: number;
|
||||
}
|
||||
|
||||
interface LlmTopic {
|
||||
title: string;
|
||||
summary: string;
|
||||
message_ids: string[];
|
||||
}
|
||||
|
||||
interface LlmTopicResponse {
|
||||
topics: LlmTopic[];
|
||||
}
|
||||
|
||||
type TopicWithMembers = {
|
||||
title: string;
|
||||
summary: string;
|
||||
members: SourceMsg[];
|
||||
groupSet: Set<string>;
|
||||
};
|
||||
|
||||
function cleanContent(s: string): string {
|
||||
return s
|
||||
.replace(/\[图片\]\s*local_id=\d+/g, '')
|
||||
.replace(/\[引用\][^\n]*\n?/g, '')
|
||||
.replace(/\[小程序\][^\n]*/g, '')
|
||||
.replace(/↳\s*[^\n]*/g, '')
|
||||
.replace(/<\?xml[\s\S]+?\?>[\s\S]*?<\/msg>/g, '')
|
||||
.replace(/https?:\/\/\S+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
// 这些消息整体就是占位符 / wrapper，无实质内容
|
||||
const PLACEHOLDER_PATTERNS = [
|
||||
/^\[链接\]\s*当前版本不支持/,
|
||||
/^\[文件\]\s*[^\s]+\.\w+\s*$/,
|
||||
/^\[视频\]\s*$/,
|
||||
/^\[音频\]\s*$/,
|
||||
/^\[语音\]\s*$/,
|
||||
/^\[表情\]\s*$/,
|
||||
/^\[图片\]\s*$/,
|
||||
/^\[位置\]/,
|
||||
/^\[名片\]/,
|
||||
/^\[小程序\]\s*[^\s]*\s*$/,
|
||||
/^\[转账\]/,
|
||||
/^\[红包\]/,
|
||||
];
|
||||
|
||||
function isPlaceholderOnly(content: string): boolean {
|
||||
if (!content) return true;
|
||||
return PLACEHOLDER_PATTERNS.some((p) => p.test(content));
|
||||
}
|
||||
|
||||
function loadCandidateMessages(date: string): SourceMsg[] {
|
||||
const rows = db()
|
||||
.prepare(
|
||||
`SELECT chatroom_id, local_id, sender, content, time, timestamp
|
||||
FROM messages
|
||||
WHERE date = ?
|
||||
AND type IN ('文本', '链接/文件')
|
||||
AND length(content) >= ?
|
||||
ORDER BY timestamp ASC
|
||||
LIMIT ?`,
|
||||
)
|
||||
.all(date, MIN_MESSAGE_LENGTH, MAX_MESSAGES_TO_PROCESS) as SourceMsg[];
|
||||
|
||||
// 1. 过滤占位符 + 清洗 + 长度筛选
|
||||
const cleaned = rows
|
||||
.map((r) => ({ ...r, content: cleanContent(r.content).slice(0, MAX_MESSAGE_LENGTH) }))
|
||||
.filter((r) => !isPlaceholderOnly(r.content) && r.content.length >= MIN_MESSAGE_LENGTH);
|
||||
|
||||
// 2. 去重：相同内容（同一条转发消息）只保留第一次出现
|
||||
// 这是真信号（同一篇文章被多群转发）但不应该堆成「话题」— 简化为信源（前 3 条群即可）
|
||||
const seen = new Map<string, SourceMsg>();
|
||||
for (const r of cleaned) {
|
||||
const key = r.content.slice(0, 80); // 前 80 字相同 ≈ 同一条转发
|
||||
if (!seen.has(key)) seen.set(key, r);
|
||||
}
|
||||
return Array.from(seen.values());
|
||||
}
|
||||
|
||||
const TOPIC_RESPONSE_SCHEMA = {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
topics: {
|
||||
type: 'array',
|
||||
items: {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
title: { type: 'string' },
|
||||
summary: { type: 'string' },
|
||||
message_ids: {
|
||||
type: 'array',
|
||||
items: { type: 'string' },
|
||||
},
|
||||
},
|
||||
required: ['title', 'summary', 'message_ids'],
|
||||
},
|
||||
},
|
||||
},
|
||||
required: ['topics'],
|
||||
};
|
||||
|
||||
function sourceId(m: SourceMsg): string {
|
||||
return `${m.chatroom_id}#${m.local_id}`;
|
||||
}
|
||||
|
||||
function chunk<T>(items: T[], size: number): T[][] {
|
||||
const out: T[][] = [];
|
||||
for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size));
|
||||
return out;
|
||||
}
|
||||
|
||||
function parseJsonOutput<T>(raw: string): T {
|
||||
const trimmed = raw.trim();
|
||||
try {
|
||||
return JSON.parse(trimmed) as T;
|
||||
} catch {
|
||||
const fenced = trimmed.match(/```(?:json)?\s*([\s\S]*?)```/i);
|
||||
if (fenced) return JSON.parse(fenced[1]) as T;
|
||||
const obj = trimmed.match(/\{[\s\S]*\}/);
|
||||
if (obj) return JSON.parse(obj[0]) as T;
|
||||
throw new Error('codex returned non-JSON');
|
||||
}
|
||||
}
|
||||
|
||||
function runCodexJson<T>(prompt: string, timeoutMs = CODEX_TIMEOUT_MS): Promise<T> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'wechat-topics-'));
|
||||
const schemaPath = join(dir, 'schema.json');
|
||||
const outPath = join(dir, 'response.json');
|
||||
writeFileSync(schemaPath, JSON.stringify(TOPIC_RESPONSE_SCHEMA), 'utf8');
|
||||
|
||||
const args = [
|
||||
'-a',
|
||||
'never',
|
||||
'exec',
|
||||
'--sandbox',
|
||||
'read-only',
|
||||
'--ephemeral',
|
||||
'--ignore-rules',
|
||||
'--output-schema',
|
||||
schemaPath,
|
||||
'--output-last-message',
|
||||
outPath,
|
||||
];
|
||||
if (CODEX_MODEL) args.push('--model', CODEX_MODEL);
|
||||
args.push('-');
|
||||
|
||||
const proc = spawn(
|
||||
'codex',
|
||||
args,
|
||||
{ env: { ...process.env, NO_COLOR: '1' }, stdio: ['pipe', 'pipe', 'pipe'] },
|
||||
);
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
const t = setTimeout(() => {
|
||||
proc.kill('SIGTERM');
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
reject(new Error('codex CLI timeout'));
|
||||
}, timeoutMs);
|
||||
proc.stdout.on('data', (d) => (stdout += d.toString()));
|
||||
proc.stderr.on('data', (d) => (stderr += d.toString()));
|
||||
proc.on('error', (e) => {
|
||||
clearTimeout(t);
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
reject(e);
|
||||
});
|
||||
proc.on('close', (code) => {
|
||||
clearTimeout(t);
|
||||
try {
|
||||
if (code !== 0) {
|
||||
reject(new Error(`codex exit ${code}: ${stderr.slice(0, 800)}`));
|
||||
return;
|
||||
}
|
||||
const raw = readFileSync(outPath, 'utf8') || stdout;
|
||||
resolve(parseJsonOutput<T>(raw));
|
||||
} catch (e) {
|
||||
reject(e);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
proc.stdin.write(prompt);
|
||||
proc.stdin.end();
|
||||
});
|
||||
}
|
||||
|
||||
function formatMessagesForPrompt(messages: SourceMsg[], groupNameMap: Map<string, string>): string {
|
||||
return messages
|
||||
.map((m) =>
|
||||
JSON.stringify({
|
||||
id: sourceId(m),
|
||||
group: groupNameMap.get(m.chatroom_id) ?? m.chatroom_id,
|
||||
sender: m.sender,
|
||||
time: m.time,
|
||||
content: m.content,
|
||||
}),
|
||||
)
|
||||
.join('\n');
|
||||
}
|
||||
|
||||
function buildExtractionPrompt(
|
||||
date: string,
|
||||
messages: SourceMsg[],
|
||||
groupNameMap: Map<string, string>,
|
||||
maxTopics: number,
|
||||
): string {
|
||||
return `你是微信群「话题雷达」的聚合引擎。请直接用 LLM 判断语义相关性，找出 ${date} 的主要讨论话题。
|
||||
|
||||
任务要求：
|
||||
- 只做话题聚合，不要逐条摘要。
|
||||
- 合并同一事件、产品、工具、论文、观点、问题及其追问/回应/转述。
|
||||
- 优先保留跨群出现的话题；同一群内高密度连续讨论也可以保留。
|
||||
- 忽略问候、纯闲聊、广告、无上下文碎片、纯占位内容和过泛的「AI 很火」类讨论。
|
||||
- 每个话题至少包含 ${MIN_MESSAGES_PER_TOPIC} 条消息。
|
||||
- 最多输出 ${maxTopics} 个话题，按重要性排序。
|
||||
- title 用 8-15 个汉字，优先写产品名/事件名/讨论焦点。
|
||||
- summary 用 1-2 句中文说明大家在讨论什么。
|
||||
- message_ids 必须只使用输入消息的 id；不要编造 id；同一个 id 不要重复。
|
||||
|
||||
只输出严格 JSON，格式：
|
||||
{"topics":[{"title":"...","summary":"...","message_ids":["群id#local_id"]}]}
|
||||
|
||||
输入消息为 JSONL：
|
||||
${formatMessagesForPrompt(messages, groupNameMap)}`;
|
||||
}
|
||||
|
||||
function buildMergePrompt(date: string, drafts: LlmTopic[], maxTopics: number): string {
|
||||
const lines = drafts
|
||||
.map((t, i) =>
|
||||
JSON.stringify({
|
||||
id: `draft-${i + 1}`,
|
||||
title: t.title,
|
||||
summary: t.summary,
|
||||
message_ids: t.message_ids,
|
||||
message_count: t.message_ids.length,
|
||||
}),
|
||||
)
|
||||
.join('\n');
|
||||
|
||||
return `下面是 ${date} 分批得到的话题草稿。请继续用 LLM 完成最终跨批合并。
|
||||
|
||||
任务要求：
|
||||
- 合并语义相同或强相关的话题草稿，message_ids 取并集。
|
||||
- 删除过泛、重复、证据不足的话题。
|
||||
- 每个最终话题至少包含 ${MIN_MESSAGES_PER_TOPIC} 条消息。
|
||||
- 最多输出 ${maxTopics} 个最终话题，按重要性排序。
|
||||
- title 用 8-15 个汉字，summary 用 1-2 句中文。
|
||||
- message_ids 必须来自输入草稿，不要编造。
|
||||
|
||||
只输出严格 JSON：
|
||||
{"topics":[{"title":"...","summary":"...","message_ids":["群id#local_id"]}]}
|
||||
|
||||
话题草稿 JSONL：
|
||||
${lines}`;
|
||||
}
|
||||
|
||||
function normalizeTopics(rawTopics: LlmTopic[], messageMap: Map<string, SourceMsg>): TopicWithMembers[] {
|
||||
const out: TopicWithMembers[] = [];
|
||||
const seenSignatures = new Set<string>();
|
||||
|
||||
for (const raw of rawTopics) {
|
||||
const ids = Array.from(new Set((raw.message_ids ?? []).filter((id) => messageMap.has(id))));
|
||||
if (ids.length < MIN_MESSAGES_PER_TOPIC) continue;
|
||||
|
||||
const members = ids.map((id) => messageMap.get(id)!).sort((a, b) => a.timestamp - b.timestamp);
|
||||
const signature = ids.slice().sort().join('|');
|
||||
if (seenSignatures.has(signature)) continue;
|
||||
seenSignatures.add(signature);
|
||||
|
||||
out.push({
|
||||
title: (raw.title || members[0].content.slice(0, 16) || '未命名话题').slice(0, 80),
|
||||
summary: (raw.summary || '').slice(0, 400),
|
||||
members,
|
||||
groupSet: new Set(members.map((m) => m.chatroom_id)),
|
||||
});
|
||||
}
|
||||
|
||||
return out.sort((a, b) => b.members.length - a.members.length).slice(0, MAX_TOPICS_TO_SAVE);
|
||||
}
|
||||
|
||||
async function aggregateWithCodex(
|
||||
date: string,
|
||||
messages: SourceMsg[],
|
||||
groupNameMap: Map<string, string>,
|
||||
onProgress?: (p: TopicProgress) => void,
|
||||
): Promise<TopicWithMembers[]> {
|
||||
const messageMap = new Map(messages.map((m) => [sourceId(m), m]));
|
||||
const chunks = chunk(messages, Math.max(50, CODEX_CHUNK_SIZE));
|
||||
const drafts: LlmTopic[] = [];
|
||||
|
||||
onProgress?.({
|
||||
type: 'llm',
|
||||
done: 0,
|
||||
total: chunks.length,
|
||||
message: `Codex CLI 聚合 ${messages.length} 条消息…`,
|
||||
});
|
||||
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
const response = await runCodexJson<LlmTopicResponse>(
|
||||
buildExtractionPrompt(date, chunks[i], groupNameMap, TOPICS_PER_CHUNK),
|
||||
);
|
||||
drafts.push(...(response.topics ?? []));
|
||||
onProgress?.({
|
||||
type: 'llm',
|
||||
done: i + 1,
|
||||
total: chunks.length,
|
||||
message: `Codex CLI 分批聚合 ${i + 1}/${chunks.length}`,
|
||||
});
|
||||
}
|
||||
|
||||
if (drafts.length === 0) return [];
|
||||
|
||||
if (chunks.length > 1) {
|
||||
onProgress?.({
|
||||
type: 'llm',
|
||||
done: chunks.length,
|
||||
total: chunks.length,
|
||||
message: `Codex CLI 合并 ${drafts.length} 个话题草稿…`,
|
||||
});
|
||||
}
|
||||
|
||||
const final =
|
||||
chunks.length === 1
|
||||
? { topics: drafts }
|
||||
: await runCodexJson<LlmTopicResponse>(buildMergePrompt(date, drafts, MAX_TOPICS_TO_SAVE));
|
||||
|
||||
return normalizeTopics(final.topics ?? [], messageMap);
|
||||
}
|
||||
|
||||
export interface TopicProgress {
|
||||
type: 'load' | 'llm' | 'save' | 'done' | 'error';
|
||||
done?: number;
|
||||
total?: number;
|
||||
count?: number;
|
||||
message?: string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export async function buildTopicsForDate(
|
||||
date: string,
|
||||
onProgress?: (p: TopicProgress) => void,
|
||||
): Promise<{ topics: number; messages: number }> {
|
||||
onProgress?.({ type: 'load', message: '加载当日消息…' });
|
||||
const msgs = loadCandidateMessages(date);
|
||||
if (msgs.length === 0) {
|
||||
onProgress?.({ type: 'done', count: 0 });
|
||||
return { topics: 0, messages: 0 };
|
||||
}
|
||||
|
||||
const sessions = await wxSessions(500).catch(() => []);
|
||||
const groupNameMap = new Map<string, string>();
|
||||
for (const s of sessions) groupNameMap.set(s.username, s.chat);
|
||||
|
||||
const valid = await aggregateWithCodex(date, msgs, groupNameMap, onProgress);
|
||||
|
||||
// 清空当日旧话题
|
||||
db().prepare('DELETE FROM topics WHERE date = ?').run(date);
|
||||
|
||||
let savedTopics = 0;
|
||||
let savedMessages = 0;
|
||||
for (let i = 0; i < valid.length; i++) {
|
||||
const c = valid[i];
|
||||
onProgress?.({
|
||||
type: 'save',
|
||||
done: i + 1,
|
||||
total: valid.length,
|
||||
message: c.title,
|
||||
});
|
||||
|
||||
const insertTopic = db().prepare(
|
||||
'INSERT INTO topics (date, title, summary, message_count, group_count, created_at) VALUES (?, ?, ?, ?, ?, ?)',
|
||||
);
|
||||
const insertMsg = db().prepare(
|
||||
'INSERT OR IGNORE INTO topic_messages (topic_id, chatroom_id, local_id, score) VALUES (?, ?, ?, ?)',
|
||||
);
|
||||
|
||||
const tx = db().transaction(() => {
|
||||
const info = insertTopic.run(
|
||||
date,
|
||||
c.title,
|
||||
c.summary,
|
||||
c.members.length,
|
||||
c.groupSet.size,
|
||||
Date.now(),
|
||||
);
|
||||
const tid = Number(info.lastInsertRowid);
|
||||
for (let index = 0; index < c.members.length; index++) {
|
||||
const member = c.members[index];
|
||||
insertMsg.run(tid, member.chatroom_id, member.local_id, 1 - index / 1000);
|
||||
savedMessages++;
|
||||
}
|
||||
});
|
||||
tx();
|
||||
savedTopics++;
|
||||
}
|
||||
|
||||
onProgress?.({ type: 'done', count: savedTopics });
|
||||
return { topics: savedTopics, messages: savedMessages };
|
||||
}
|
||||
|
||||
export interface TopicListItem {
|
||||
id: number;
|
||||
date: string;
|
||||
title: string;
|
||||
summary: string;
|
||||
message_count: number;
|
||||
group_count: number;
|
||||
}
|
||||
|
||||
export function listTopics(date: string): TopicListItem[] {
|
||||
return db()
|
||||
.prepare(
|
||||
'SELECT id, date, title, summary, message_count, group_count FROM topics WHERE date = ? ORDER BY message_count DESC',
|
||||
)
|
||||
.all(date) as TopicListItem[];
|
||||
}
|
||||
|
||||
export interface TopicDetail extends TopicListItem {
|
||||
messages: Array<{
|
||||
chatroom_id: string;
|
||||
chat_name: string;
|
||||
local_id: number;
|
||||
sender: string;
|
||||
content: string;
|
||||
time: string;
|
||||
timestamp: number;
|
||||
type: string;
|
||||
score: number;
|
||||
}>;
|
||||
}
|
||||
|
||||
export async function getTopicDetail(id: number): Promise<TopicDetail | null> {
|
||||
const topic = db()
|
||||
.prepare(
|
||||
'SELECT id, date, title, summary, message_count, group_count FROM topics WHERE id = ?',
|
||||
)
|
||||
.get(id) as TopicListItem | undefined;
|
||||
if (!topic) return null;
|
||||
|
||||
const rows = db()
|
||||
.prepare(
|
||||
`SELECT m.chatroom_id, m.local_id, m.sender, m.content, m.time, m.timestamp, m.type, tm.score
|
||||
FROM topic_messages tm
|
||||
JOIN messages m ON m.chatroom_id = tm.chatroom_id AND m.local_id = tm.local_id
|
||||
WHERE tm.topic_id = ?
|
||||
ORDER BY tm.score DESC, m.timestamp ASC`,
|
||||
)
|
||||
.all(id) as Array<{
|
||||
chatroom_id: string;
|
||||
local_id: number;
|
||||
sender: string;
|
||||
content: string;
|
||||
time: string;
|
||||
timestamp: number;
|
||||
type: string;
|
||||
score: number;
|
||||
}>;
|
||||
|
||||
const sessions = await wxSessions(500).catch(() => []);
|
||||
const nameMap = new Map<string, string>();
|
||||
for (const s of sessions) nameMap.set(s.username, s.chat);
|
||||
|
||||
return {
|
||||
...topic,
|
||||
messages: rows.map((r) => ({
|
||||
...r,
|
||||
chat_name: nameMap.get(r.chatroom_id) ?? r.chatroom_id,
|
||||
})),
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user