+
+
+
+
-
-
diff --git a/results.json b/results.json
index 0637a08..7381f54 100644
--- a/results.json
+++ b/results.json
@@ -1 +1,266 @@
-[]
\ No newline at end of file
+[
+ {
+ "taskId": "task-1710000000001",
+ "city": "南京市",
+ "scrapedAt": "2026-03-10T08:12:18.355Z",
+ "data": {
+ "results": [
+ {
+ "type": "房建市政",
+ "project_name": "星甸污水处理厂提标改造工程（一期）- 施工",
+ "amount_yuan": 5205328.1,
+ "date": "2026-03-10",
+ "target_link": "https://njggzy.nanjing.gov.cn/njweb/fjsz/068001/068001002/20260310/84ce20ef-3ccf-414e-ad95-969296ae19a6.html"
+ },
+ {
+ "type": "交通水务",
+ "project_name": "南京港华燃气 sueper 大客户管理系统项目",
+ "amount_yuan": 8820000,
+ "date": "2026-03-10",
+ "target_link": "https://njggzy.nanjing.gov.cn/njweb/jtsw/069002/20260310/4c99f020-5b79-dffb-e063-3345c40a2f88.html"
+ },
+ {
+ "type": "政府采购",
+ "project_name": "南京市公安局交通管理局机动车驾驶人考试中心考场服务项目",
+ "amount_yuan": 11130000,
+ "date": "2026-03-10",
+ "target_link": "https://njggzy.nanjing.gov.cn/njweb/zfcg/055001/055001001/20260310/4c99f020-5c2c-dffb-e063-3345c40a2f88.html"
+ }
+ ],
+ "total": 3
+ },
+ "id": "result-1773130338356-w41hf"
+ },
+ {
+ "taskId": "task-1710000000001",
+ "city": "南京市",
+ "scrapedAt": "2026-03-10T07:53:50.177Z",
+ "data": {
+ "results": [
+ {
+ "type": "房建市政",
+ "project_name": "星甸污水处理厂提标改造工程（一期）",
+ "amount_yuan": 2776800,
+ "date": "2026-03-10",
+ "target_link": "https://njggzy.nanjing.gov.cn/njweb/fjsz/068001/068001002/20260310/84ce20ef-3ccf-414e-ad95-969296ae19a6.html"
+ },
+ {
+ "type": "房建市政",
+ "project_name": "【澄清公告】麒麟科创园具身智能训练场装修项目",
+ "amount_yuan": 833000,
+ "date": "2026-03-10",
+ "target_link": "https://njggzy.nanjing.gov.cn/njweb/fjsz/068001/068001002/20260310/945533de-a0ed-4118-a30e-ec831a4e337b.html"
+ },
+ {
+ "type": "房建市政",
+ "project_name": "【澄清公告】轻质耐热合金制造基地项目",
+ "amount_yuan": 100000000,
+ "date": "2026-03-09",
+ "target_link": "https://njggzy.nanjing.gov.cn/njweb/fjsz/068001/068001002/20260309/fb9e7e41-d809-458c-9cd4-59b90e84fbc5.html"
+ },
+ {
+ "type": "工程货物",
+ "project_name": "JNFJ2500777-04HW-GHa01",
+ "amount_yuan": 66000000,
+ "date": "2026-03-07",
+ "target_link": "https://njggzy.nanjing.gov.cn/njweb/gchw/070001/20260307/041138ff-870c-4240-b78c-f6861f242594.html"
+ },
+ {
+ "type": "工程货物",
+ "project_name": "NJFJ2500863-03HWGH",
+ "amount_yuan": 484057073,
+ "date": "2026-03-06",
+ "target_link": "https://njggzy.nanjing.gov.cn/njweb/gchw/070001/20260306/a547fb75-c984-4479-948b-b5800829742a.html"
+ },
+ {
+ "type": "工程货物",
+ "project_name": "JNFJ2500777-06QTGH",
+ "amount_yuan": 535784084,
+ "date": "2026-03-06",
+ "target_link": "https://njggzy.nanjing.gov.cn/njweb/gchw/070001/20260306/cab7606b-df8f-419e-be5a-4ce5213ef5af.html"
+ },
+ {
+ "type": "交通水务",
+ "project_name": "(市交易中心) 八卦洲街道CBA3上坝子堤路等农村公路改善提升工程施工招标公告",
+ "amount_yuan": 338600000,
+ "date": "2026-03-04",
+ "target_link": "https://njggzy.nanjing.gov.cn/njweb/jtsw/069001/069001001/20260304/3d25610f-239d-4e1c-9923-0d22820c711a.html"
+ },
+ {
+ "type": "交通水务",
+ "project_name": "(市交易中心) 南京地铁1号线通信信号系统设备更新改造项目通信系统安装施工招标公告",
+ "amount_yuan": 1693000000,
+ "date": "2026-02-14",
+ "target_link": "https://njggzy.nanjing.gov.cn/njweb/jtsw/069001/069001001/20260214/4ddc9129-c508-4ae9-b103-f52851ff2ee2.html"
+ },
+ {
+ "type": "交通水务",
+ "project_name": "(市交易中心) 356省道浦口西江路至苏皖省界段建设工程起点至龙港路段供电管线迁改工程招标公告",
+ "amount_yuan": 2000000,
+ "date": "2026-02-12",
+ "target_link": "https://njggzy.nanjing.gov.cn/njweb/jtsw/069001/069001001/20260212/5ae97a66-ca0d-40e4-991c-f91d6164661f.html"
+ },
+ {
+ "type": "政府采购",
+ "project_name": "JSZC-320111-JZCG-G2026-0013",
+ "amount_yuan": 204000,
+ "date": "2026-03-10",
+ "target_link": "https://njggzy.nanjing.gov.cn/njweb/zfcg/067001/067001001/20260310/adb322241a19459cbaaf158e4ad87500.html"
+ },
+ {
+ "type": "政府采购",
+ "project_name": "JSZC-320100-JZCG-C2026-0026",
+ "amount_yuan": 1020000,
+ "date": "2026-03-10",
+ "target_link": "https://njggzy.nanjing.gov.cn/njweb/zfcg/067001/067001001/20260310/a801f9ba484741f9b373207edcbc0320.html"
+ },
+ {
+ "type": "政府采购",
+ "project_name": "JSZC-320100-JZCG-C2026-0030",
+ "amount_yuan": 50000,
+ "date": "2026-03-10",
+ "target_link": "https://njggzy.nanjing.gov.cn/njweb/zfcg/067001/067001001/20260310/19c6d526a57c451fbbbe2c1b7a779c02.html"
+ },
+ {
+ "type": "产权交易",
+ "project_name": "北京北汽福斯特股份有限公司1.3884%股权（138.8373万股）股权转让",
+ "amount_yuan": 66000000,
+ "date": "2026-04-20",
+ "target_link": "https://njggzy.nanjing.gov.cn/njweb/gycq/071001/071001002/20260309/64b16240-07de-417c-8248-c44967c508f9.html"
+ },
+ {
+ "type": "产权交易",
+ "project_name": "北京北汽九龙出租汽车股份有限公司1.3884%（91.6327万股）股权转让",
+ "amount_yuan": 484057073,
+ "date": "2026-04-20",
+ "target_link": "https://njggzy.nanjing.gov.cn/njweb/gycq/071001/071001002/20260309/57c7f45a-b28e-4ba9-b11b-62cca6f6218e.html"
+ },
+ {
+ "type": "产权交易",
+ "project_name": "南京鑫一汇企业管理有限公司20%股权转让",
+ "amount_yuan": 2000000,
+ "date": "2026-02-25",
+ "target_link": "https://njggzy.nanjing.gov.cn/njweb/gycq/071001/071001002/20260108/442d7bf6-f11d-49f5-9409-aef8acd4fbf6.html"
+ },
+ {
+ "type": "土地矿产",
+ "project_name": "南京市规划和自然资源局六合分局国有土地使用权挂牌出让公告(宁六工[2026]第5号)",
+ "amount_yuan": 664000,
+ "date": "2026-03-06",
+ "target_link": "https://njggzy.nanjing.gov.cn/njweb/tdkc/072001/20260306/10586876.html"
+ },
+ {
+ "type": "土地矿产",
+ "project_name": "南京市规划和自然资源局六合分局国有土地使用权挂牌出让公告(宁六工[2026]第4号)",
+ "amount_yuan": 3320000,
+ "date": "2026-03-04",
+ "target_link": "https://njggzy.nanjing.gov.cn/njweb/tdkc/072001/20260304/10561567.html"
+ },
+ {
+ "type": "土地矿产",
+ "project_name": "南京市规划和自然资源局国有土地使用权挂牌出让公告(宁出[2026]01号)",
+ "amount_yuan": 50000,
+ "date": "2026-02-28",
+ "target_link": "https://njggzy.nanjing.gov.cn/njweb/tdkc/072001/20260228/10528773.html"
+ },
+ {
+ "type": "铁路航运",
+ "project_name": "【施工类】暂无最新项目信息",
+ "amount_yuan": 0,
+ "date": "2026-03-02",
+ "target_link": "https://njggzy.nanjing.gov.cn/njweb/tlhy/073001/073001001/20240702/3a9b3413-3931-4d51-af72-78e11162ed74.html"
+ },
+ {
+ "type": "农村产权",
+ "project_name": "栖霞区龙潭街道马渡村新桥组12.01亩土地出租通知公告主要信息",
+ "amount_yuan": 28824,
+ "date": "2026-03-09",
+ "target_link": "https://njggzy.nanjing.gov.cn/njweb/nccq/102001/20260309/4c99efee-79f1-e35e-e063-3345c40a7178.html"
+ },
+ {
+ "type": "农村产权",
+ "project_name": "栖霞区龙潭街道马渡村新春组18.93亩土地出租通知公告主要信息",
+ "amount_yuan": 18015,
+ "date": "2026-03-09",
+ "target_link": "https://njggzy.nanjing.gov.cn/njweb/nccq/102001/20260309/4c99efee-79ed-e35e-e063-3345c40a7178.html"
+ },
+ {
+ "type": "农村产权",
+ "project_name": "栖霞区龙潭街道马渡村厢林组47.37亩土地出租通知公告主要信息",
+ "amount_yuan": 113688,
+ "date": "2026-03-09",
+ "target_link": "https://njggzy.nanjing.gov.cn/njweb/nccq/102001/20260309/4c99efee-79e9-e35e-e063-3345c40a7178.html"
+ }
+ ],
+ "total": 22
+ },
+ "id": "result-1773129230177-5tjno"
+ },
+ {
+ "taskId": "task-1710000000001",
+ "city": "南京市",
+ "scrapedAt": "2026-03-10T07:34:51.928Z",
+ "data": {
+ "results": [
+ {
+ "type": "房建市政",
+ "project_name": "星甸污水处理厂提标改造工程（一期）",
+ "amount_yuan": 5205328.1,
+ "date": "2026-03-10",
+ "target_link": "https://njggzy.nanjing.gov.cn/njweb/fjsz/068001/068001002/20260310/84ce20ef-3ccf-414e-ad95-969296ae19a6.html"
+ },
+ {
+ "type": "工程货物",
+ "project_name": "【澄清公告】NO.2024G39地块建设项目 - 电梯设备采购及相关服务（暂估价）",
+ "amount_yuan": 4069576.31,
+ "date": "2026-03-07",
+ "target_link": "https://njggzy.nanjing.gov.cn/njweb/gchw/070001/20260307/041138ff-870c-4240-b78c-f6861f242594.html"
+ },
+ {
+ "type": "交通水务",
+ "project_name": "【施工】八卦洲街道CBA3上坝子堤路等农村公路改善提升工程施工招标公告",
+ "amount_yuan": 4020000,
+ "date": "2026-03-04",
+ "target_link": "https://njggzy.nanjing.gov.cn/njweb/jtsw/069001/069001001/20260304/3d25610f-239d-4e1c-9923-0d22820c711a.html"
+ },
+ {
+ "type": "政府采购",
+ "project_name": "JSZC-320111-JZCG-G2026-0013 政府采购公告",
+ "amount_yuan": 11130000,
+ "date": "2026-03-10",
+ "target_link": "https://njggzy.nanjing.gov.cn/njweb/zfcg/067001/067001001/20260310/adb322241a19459cbaaf158e4ad87500.html"
+ },
+ {
+ "type": "产权交易",
+ "project_name": "2026320100CA00003 产权交易公告",
+ "amount_yuan": 2776800,
+ "date": "2026-04-20",
+ "target_link": "https://njggzy.nanjing.gov.cn/njweb/gycq/071001/071001002/20260309/64b16240-07de-417c-8248-c44967c508f9.html"
+ },
+ {
+ "type": "土地矿产",
+ "project_name": "南京市规划和自然资源局六合分局国有土地使用权挂牌出让公告(宁六工[2026]第5号)",
+ "amount_yuan": 664000,
+ "date": "2026-03-06",
+ "target_link": "https://njggzy.nanjing.gov.cn/njweb/tdkc/072001/20260306/10586876.html"
+ },
+ {
+ "type": "铁路航运",
+ "project_name": "【施工类】暂无最新项目信息",
+ "amount_yuan": 0,
+ "date": "2026-03-02",
+ "target_link": "https://njggzy.nanjing.gov.cn/njweb/tlhy/073001/073001001/20240702/3a9b3413-3931-4d51-af72-78e11162ed74.html"
+ },
+ {
+ "type": "农村产权",
+ "project_name": "栖霞区龙潭街道马渡村新桥组12.01亩土地出租通知公告",
+ "amount_yuan": 9608,
+ "date": "2026-03-09",
+ "target_link": "https://njggzy.nanjing.gov.cn/njweb/nccq/102001/20260309/4c99efee-79f1-e35e-e063-3345c40a7178.html"
+ }
+ ],
+ "total": 8
+ },
+ "id": "result-1773128091930-edg4p"
+ }
+]
\ No newline at end of file
diff --git a/src/agentService.js b/src/agentService.js
new file mode 100644
index 0000000..b3a9693
--- /dev/null
+++ b/src/agentService.js
@@ -0,0 +1,137 @@
+/**
+ * Agent API 服务封装
+ * 调用本地部署的 agent 进行公告抓取
+ */
+
+const DEFAULT_BASE_URL = 'http://192.168.3.65:18625';
+const DEFAULT_POLL_INTERVAL = 3000; // 3秒轮询
+const DEFAULT_TIMEOUT = 3600000; // 1小时超时
+const FETCH_TIMEOUT = 30000; // 单次 fetch 30秒超时
+const MAX_FETCH_RETRIES = 5; // 网络错误最多重试5次
+
+function generateTaskId() {
+ return `task-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`;
+}
+
+function sleep(ms) {
+ return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+/**
+ * 带超时和重试的 fetch
+ */
+async function fetchWithRetry(url, fetchOptions, retries = MAX_FETCH_RETRIES, logPrefix = '[Agent]') {
+ for (let attempt = 1; attempt <= retries; attempt++) {
+ try {
+ const controller = new AbortController();
+ const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT);
+ const res = await fetch(url, { ...fetchOptions, signal: controller.signal });
+ clearTimeout(timer);
+ return res;
+ } catch (err) {
+ const isLast = attempt === retries;
+ console.warn(`${logPrefix} fetch 失败 (${attempt}/${retries}): ${err.message}`);
+ if (isLast) throw err;
+ await sleep(3000 * attempt); // 递增等待
+ }
+ }
+}
+
+/**
+ * 创建 agent 任务
+ */
+async function createTask(prompt, options = {}) {
+ const baseUrl = options.baseUrl || DEFAULT_BASE_URL;
+ const useBrowser = options.useBrowser ?? false;
+ const taskId = generateTaskId();
+ const logPrefix = options.logPrefix || '[Agent]';
+
+ const res = await fetchWithRetry(`${baseUrl}/agent/createTask`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ taskId, prompt, useBrowser }),
+ }, MAX_FETCH_RETRIES, logPrefix);
+
+ if (!res.ok) {
+ throw new Error(`创建任务失败: HTTP ${res.status}`);
+ }
+
+ return { taskId };
+}
+
+/**
+ * 检查任务状态
+ * 返回空/null 表示任务还在运行，返回 { success, message, data } 表示完成
+ */
+async function checkTask(taskId, options = {}) {
+ const baseUrl = options.baseUrl || DEFAULT_BASE_URL;
+ const logPrefix = options.logPrefix || '[Agent]';
+
+ const res = await fetchWithRetry(`${baseUrl}/agent/checkTask`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ taskId }),
+ }, MAX_FETCH_RETRIES, logPrefix);
+
+ if (!res.ok) {
+ throw new Error(`检查任务失败: HTTP ${res.status}`);
+ }
+
+ const text = await res.text();
+ console.log(`${logPrefix} checkTask(${taskId}) 返回:`, text ? text.substring(0, 500) : '(空)');
+ if (!text || text.trim() === '' || text.trim() === 'null') {
+ return null; // 任务还在运行
+ }
+
+ try {
+ return JSON.parse(text);
+ } catch {
+ return null;
+ }
+}
+
+/**
+ * 运行 agent 任务：创建 + 轮询直到完成
+ * 返回 { results: [{ type, project_name, amount_yuan, date, target_link }] }
+ */
+export async function runAgentTask(prompt, options = {}) {
+ const baseUrl = options.baseUrl || DEFAULT_BASE_URL;
+ const useBrowser = options.useBrowser ?? false;
+ const pollInterval = options.pollInterval || DEFAULT_POLL_INTERVAL;
+ const timeout = options.timeout || DEFAULT_TIMEOUT;
+ const logPrefix = options.logPrefix || '[Agent]';
+
+ console.log(`${logPrefix} 创建任务...`);
+ const { taskId } = await createTask(prompt, { baseUrl, useBrowser, logPrefix });
+ console.log(`${logPrefix} 任务已创建: ${taskId}`);
+
+ const startTime = Date.now();
+
+ while (true) {
+ if (Date.now() - startTime > timeout) {
+ throw new Error(`任务超时 (${timeout / 1000}秒): ${taskId}`);
+ }
+
+ await sleep(pollInterval);
+
+ const result = await checkTask(taskId, { baseUrl, logPrefix });
+
+ if (result === null) {
+ const elapsed = Math.round((Date.now() - startTime) / 1000);
+ console.log(`${logPrefix} 任务进行中... (${elapsed}秒)`);
+ continue;
+ }
+
+ if (result.success) {
+ console.log(`${logPrefix} 任务完成: ${result.message}`);
+ const data = result.data || {};
+ const results = Array.isArray(data.results) ? data.results : [];
+ console.log(`${logPrefix} 获取到 ${results.length} 条结果`);
+ return { results };
+ } else {
+ throw new Error(`任务失败: ${result.message || '未知错误'}`);
+ }
+ }
+}
+
+export { generateTaskId, createTask, checkTask };
diff --git a/src/scheduler.js b/src/scheduler.js
index d780ada..00437d5 100644
--- a/src/scheduler.js
+++ b/src/scheduler.js
@@ -3,19 +3,14 @@ import cron from 'node-cron';
import { readFileSync, writeFileSync, existsSync } from 'fs';
import { fileURLToPath } from 'url';
import { dirname, join } from 'path';
-import Firecrawl from '@mendable/firecrawl-js';
import { sendScraperResultsEmail } from './emailService.js';
-import { runScraperWithBrowser } from './firecrawlBrowserScraper.js';
+import { runAgentTask } from './agentService.js';
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
-// 初始化 Firecrawl 客户端
-const firecrawl = new Firecrawl({ apiKey: process.env.FIRECRAWL_API_KEY });
-
const RESULTS_PATH = join(__dirname, '..', 'results.json');
-// 加载配置文件
function loadConfig() {
try {
const configPath = join(__dirname, '..', 'config.json');
@@ -26,7 +21,7 @@ function loadConfig() {
}
}
-// ========== 结果存取（与 server.js 保持一致） ==========
+// ========== 结果存取 ==========
function readResults() {
if (!existsSync(RESULTS_PATH)) return [];
@@ -48,22 +43,24 @@ function appendResult(result) {
saveResults(results);
}
-// ========== 抓取执行（复用 server.js 中 runScraper 的逻辑） ==========
+// ========== 任务执行 ==========
-async function runScraper(scraper) {
- console.log(`[定时任务][Browser] ${scraper.city} - ${scraper.section} ${scraper.subsection} - ${scraper.type}：${scraper.url}`);
- const { items } = await runScraperWithBrowser(firecrawl, scraper, { logPrefix: '[Browser][Scheduler]' });
- console.log(`[定时任务][Browser] 提取到 ${items.length} 条公告`);
+async function runTask(task, agentCfg) {
+ console.log(`[定时任务][Agent] ${task.city}：开始执行`);
+ const { results } = await runAgentTask(task.prompt, {
+ baseUrl: agentCfg.baseUrl,
+ useBrowser: agentCfg.useBrowser,
+ pollInterval: agentCfg.pollInterval,
+ timeout: agentCfg.timeout,
+ logPrefix: `[定时任务][Agent][${task.city}]`,
+ });
+ console.log(`[定时任务][Agent] ${task.city}：获取到 ${results.length} 条结果`);
const record = {
- scraperId: scraper.id,
- city: scraper.city,
- section: scraper.section,
- subsection: scraper.subsection,
- type: scraper.type,
- url: scraper.url,
+ taskId: task.id,
+ city: task.city,
scrapedAt: new Date().toISOString(),
- data: { result: items, total: items.length },
+ data: { results, total: results.length },
};
appendResult(record);
return record;
@@ -78,33 +75,28 @@ async function executeScheduledTask(config) {
console.log('执行时间:', new Date().toLocaleString('zh-CN'));
console.log('========================================');
- // 获取所有已启用的抓取来源
- const scrapers = (config.scrapers || []).filter(s => s.enabled);
+ const tasks = (config.tasks || []).filter(t => t.enabled);
+ const agentCfg = config.agent || {};
- if (scrapers.length === 0) {
- console.log('没有已启用的抓取来源，跳过');
+ if (tasks.length === 0) {
+ console.log('没有已启用的任务，跳过');
return;
}
- console.log(`共 ${scrapers.length} 个已启用的抓取来源`);
+ console.log(`共 ${tasks.length} 个已启用的任务`);
- // 逐个运行抓取任务
const results = [];
- for (const scraper of scrapers) {
+ for (const task of tasks) {
try {
- console.log(`\n---------- 抓取: ${scraper.city} - ${scraper.section} ${scraper.type} ----------`);
- const r = await runScraper(scraper);
+ console.log(`\n---------- 任务: ${task.city} ----------`);
+ const r = await runTask(task, agentCfg);
results.push(r);
- console.log(`✓ 抓取成功`);
+ console.log(`✓ 执行成功`);
} catch (err) {
- console.error(`✗ 抓取失败: ${err.message}`);
+ console.error(`✗ 执行失败: ${err.message}`);
const errRecord = {
- scraperId: scraper.id,
- city: scraper.city,
- section: scraper.section,
- subsection: scraper.subsection,
- type: scraper.type,
- url: scraper.url,
+ taskId: task.id,
+ city: task.city,
scrapedAt: new Date().toISOString(),
error: err.message,
data: null,
@@ -116,18 +108,16 @@ async function executeScheduledTask(config) {
const successCount = results.filter(r => !r.error).length;
const failCount = results.filter(r => r.error).length;
- console.log(`\n========== 抓取完成 ==========`);
- console.log(`成功: ${successCount} 条，失败: ${failCount} 条`);
+ console.log(`\n========== 执行完成 ==========`);
+ console.log(`成功: ${successCount}，失败: ${failCount}`);
- // 检查是否需要发送邮件
if (successCount === 0) {
- console.log('没有成功的抓取结果，不发送邮件');
+ console.log('没有成功的结果，不发送邮件');
return;
}
- // 发送邮件报告
if (config.email?.smtpHost && config.email?.smtpUser) {
- console.log('\n正在发送抓取结果邮件...');
+ console.log('\n正在发送结果邮件...');
try {
const emailResult = await sendScraperResultsEmail(config.email, results);
console.log('邮件发送成功! MessageId:', emailResult.messageId);
@@ -139,7 +129,6 @@ async function executeScheduledTask(config) {
}
console.log('========================================');
-
} catch (error) {
console.error('========================================');
console.error('定时任务执行失败:', error.message);
@@ -157,17 +146,16 @@ export function initScheduler() {
if (!config.scheduler?.enabled) { console.log('定时任务已禁用'); return; }
const cronTime = config.scheduler.cronTime || '0 9 * * *';
- const enabledCount = (config.scrapers || []).filter(s => s.enabled).length;
+ const enabledCount = (config.tasks || []).filter(t => t.enabled).length;
console.log('========================================');
console.log('定时任务已启动，执行计划:', cronTime);
- console.log(`已启用的抓取来源: ${enabledCount} 个`);
+ console.log(`已启用的任务: ${enabledCount} 个`);
if (config.email?.recipients) console.log('收件人:', config.email.recipients);
console.log('========================================');
if (currentScheduledTask) { currentScheduledTask.stop(); }
currentScheduledTask = cron.schedule(cronTime, () => {
- // 每次执行时重新加载配置，确保使用最新的 scrapers
const latestConfig = loadConfig();
if (latestConfig) {
executeScheduledTask(latestConfig);
@@ -191,10 +179,10 @@ export function stopScheduler() {
export function getSchedulerStatus() {
const config = loadConfig();
- const enabledScrapers = (config?.scrapers || []).filter(s => s.enabled).length;
+ const enabledTasks = (config?.tasks || []).filter(t => t.enabled).length;
return {
isRunning: currentScheduledTask !== null,
- enabledScrapers,
+ enabledTasks,
config: config ? {
enabled: config.scheduler?.enabled || false,
cronTime: config.scheduler?.cronTime || '0 9 * * *',
diff --git a/src/server.js b/src/server.js
index c072413..ce02b05 100644
--- a/src/server.js
+++ b/src/server.js
@@ -1,13 +1,11 @@
import 'dotenv/config';
import express from 'express';
import cors from 'cors';
-import Firecrawl from '@mendable/firecrawl-js';
import { readFileSync, writeFileSync, existsSync } from 'fs';
import { fileURLToPath } from 'url';
import { dirname, join } from 'path';
-import { sendCombinedReportEmail } from './emailService.js';
import { initScheduler, runTaskNow, reloadScheduler, getSchedulerStatus } from './scheduler.js';
-import { runScraperWithBrowser } from './firecrawlBrowserScraper.js';
+import { runAgentTask } from './agentService.js';
const app = express();
const PORT = process.env.PORT || 5000;
@@ -16,8 +14,6 @@ app.use(cors());
app.use(express.json());
app.use(express.static('public'));
-const firecrawl = new Firecrawl({ apiKey: process.env.FIRECRAWL_API_KEY });
-
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const CONFIG_PATH = join(__dirname, '..', 'config.json');
@@ -49,7 +45,6 @@ function saveResults(results) {
function appendResult(result) {
const results = readResults();
results.unshift({ ...result, id: `result-${Date.now()}-${Math.random().toString(36).slice(2, 7)}` });
- // 最多保留 500 条
if (results.length > 500) results.splice(500);
saveResults(results);
}
@@ -57,12 +52,14 @@ function appendResult(result) {
// 查询结果（支持分页与筛选）
app.get('/api/results', (req, res) => {
try {
- const { city, type, section, page = 1, pageSize = 20, scraperId } = req.query;
+ const { city, type, page = 1, pageSize = 20, taskId } = req.query;
let results = readResults();
if (city) results = results.filter(r => r.city === city);
- if (type) results = results.filter(r => r.type === type);
- if (section) results = results.filter(r => r.section === section);
- if (scraperId) results = results.filter(r => r.scraperId === scraperId);
+ if (type) results = results.filter(r => {
+ const items = r.data?.results || [];
+ return items.some(item => item.type === type);
+ });
+ if (taskId) results = results.filter(r => r.taskId === taskId);
const total = results.length;
const start = (parseInt(page) - 1) * parseInt(pageSize);
const data = results.slice(start, start + parseInt(pageSize));
@@ -87,7 +84,7 @@ app.delete('/api/results/:id', (req, res) => {
});
// 清空所有结果
-app.delete('/api/results', (req, res) => {
+app.delete('/api/results', (_req, res) => {
try {
saveResults([]);
res.json({ success: true, message: '已清空所有结果' });
@@ -96,46 +93,45 @@ app.delete('/api/results', (req, res) => {
}
});
-// 获取结果的筛选选项（城市/板块/类型下拉枚举）
+// 获取结果的筛选选项
app.get('/api/results/filters', (req, res) => {
try {
const results = readResults();
const cities = [...new Set(results.map(r => r.city).filter(Boolean))];
- const sections = [...new Set(results.map(r => r.section).filter(Boolean))];
- const types = [...new Set(results.map(r => r.type).filter(Boolean))];
- res.json({ success: true, data: { cities, sections, types } });
+ const types = new Set();
+ for (const r of results) {
+ for (const item of (r.data?.results || [])) {
+ if (item.type) types.add(item.type);
+ }
+ }
+ res.json({ success: true, data: { cities, types: [...types] } });
} catch (e) {
res.status(500).json({ success: false, error: e.message });
}
});
-// ========== 抓取来源 CRUD ==========
+// ========== 任务配置 CRUD ==========
-app.get('/api/scrapers', (req, res) => {
+app.get('/api/tasks', (req, res) => {
try {
const cfg = readConfig();
- res.json({ success: true, data: cfg.scrapers || [] });
+ res.json({ success: true, data: cfg.tasks || [] });
} catch (e) {
res.status(500).json({ success: false, error: e.message });
}
});
-app.post('/api/scrapers', (req, res) => {
+app.post('/api/tasks', (req, res) => {
try {
const cfg = readConfig();
- if (!cfg.scrapers) cfg.scrapers = [];
+ if (!cfg.tasks) cfg.tasks = [];
const item = {
- id: `scraper-${Date.now()}`,
+ id: `task-${Date.now()}`,
city: req.body.city || '',
- url: req.body.url || '',
- section: req.body.section || '',
- subsection: req.body.subsection || '',
- type: req.body.type || '招标公告',
prompt: req.body.prompt || '',
enabled: req.body.enabled !== false,
- model: req.body.model || 'spark-1-mini',
};
- cfg.scrapers.push(item);
+ cfg.tasks.push(item);
saveConfig(cfg);
res.json({ success: true, data: item });
} catch (e) {
@@ -143,25 +139,25 @@ app.post('/api/scrapers', (req, res) => {
}
});
-app.put('/api/scrapers/:id', (req, res) => {
+app.put('/api/tasks/:id', (req, res) => {
try {
const cfg = readConfig();
- const idx = (cfg.scrapers || []).findIndex(s => s.id === req.params.id);
+ const idx = (cfg.tasks || []).findIndex(t => t.id === req.params.id);
if (idx === -1) return res.status(404).json({ success: false, error: '未找到该配置' });
- cfg.scrapers[idx] = { ...cfg.scrapers[idx], ...req.body, id: req.params.id };
+ cfg.tasks[idx] = { ...cfg.tasks[idx], ...req.body, id: req.params.id };
saveConfig(cfg);
- res.json({ success: true, data: cfg.scrapers[idx] });
+ res.json({ success: true, data: cfg.tasks[idx] });
} catch (e) {
res.status(500).json({ success: false, error: e.message });
}
});
-app.delete('/api/scrapers/:id', (req, res) => {
+app.delete('/api/tasks/:id', (req, res) => {
try {
const cfg = readConfig();
- const before = (cfg.scrapers || []).length;
- cfg.scrapers = (cfg.scrapers || []).filter(s => s.id !== req.params.id);
- if (cfg.scrapers.length === before) return res.status(404).json({ success: false, error: '未找到' });
+ const before = (cfg.tasks || []).length;
+ cfg.tasks = (cfg.tasks || []).filter(t => t.id !== req.params.id);
+ if (cfg.tasks.length === before) return res.status(404).json({ success: false, error: '未找到' });
saveConfig(cfg);
res.json({ success: true });
} catch (e) {
@@ -169,85 +165,120 @@ app.delete('/api/scrapers/:id', (req, res) => {
}
});
-// ========== 统一抓取执行 ==========
+// ========== 任务执行（异步 + 串行锁） ==========
-// 执行单个抓取来源并保存结果
-async function runScraper(scraper) {
- console.log(`[Browser] ${scraper.city} - ${scraper.section} ${scraper.subsection} - ${scraper.type}：${scraper.url}`);
- const { items } = await runScraperWithBrowser(firecrawl, scraper, { logPrefix: '[Browser][API]' });
- console.log(`[Browser] 提取到 ${items.length} 条公告`);
+let isRunning = false;
+let runningStatus = null; // { taskId, city, startTime, current, total, finished, error }
+
+async function runTask(task) {
+ const cfg = readConfig();
+ const agentCfg = cfg.agent || {};
+
+ console.log(`[Agent] ${task.city}：开始执行`);
+ const { results } = await runAgentTask(task.prompt, {
+ baseUrl: agentCfg.baseUrl,
+ useBrowser: agentCfg.useBrowser,
+ pollInterval: agentCfg.pollInterval,
+ timeout: agentCfg.timeout,
+ logPrefix: `[Agent][${task.city}]`,
+ });
const record = {
- scraperId: scraper.id,
- city: scraper.city,
- section: scraper.section,
- subsection: scraper.subsection,
- type: scraper.type,
- url: scraper.url,
+ taskId: task.id,
+ city: task.city,
scrapedAt: new Date().toISOString(),
- data: { result: items, total: items.length }, // 统一为 result 字段
+ data: { results, total: results.length },
};
appendResult(record);
return record;
}
-// 运行指定 ID 的抓取来源（单条测试）
-app.post('/api/scrapers/:id/run', async (req, res) => {
- try {
- const cfg = readConfig();
- const scraper = (cfg.scrapers || []).find(s => s.id === req.params.id);
- if (!scraper) return res.status(404).json({ success: false, error: '未找到该配置' });
- const result = await runScraper(scraper);
- res.json({ success: true, data: result });
- } catch (e) {
- console.error('测试抓取失败:', e.message);
- res.status(500).json({ success: false, error: e.message });
- }
+// 后台执行单个任务
+function runTaskInBackground(task) {
+ runningStatus = { taskId: task.id, city: task.city, startTime: Date.now(), current: 0, total: 1, finished: false, error: null };
+ runTask(task).then(record => {
+ runningStatus = { ...runningStatus, finished: true, result: record, current: 1 };
+ }).catch(err => {
+ console.error('任务执行失败:', err.message);
+ runningStatus = { ...runningStatus, finished: true, error: err.message, current: 1 };
+ }).finally(() => {
+ isRunning = false;
+ });
+}
+
+// 后台执行批量任务
+function runTasksInBackground(tasks) {
+ runningStatus = { taskId: null, city: null, startTime: Date.now(), current: 0, total: tasks.length, finished: false, error: null, results: [] };
+ (async () => {
+ for (const task of tasks) {
+ runningStatus = { ...runningStatus, taskId: task.id, city: task.city };
+ try {
+ const r = await runTask(task);
+ runningStatus.results.push(r);
+ } catch (err) {
+ const errRecord = { taskId: task.id, city: task.city, scrapedAt: new Date().toISOString(), error: err.message, data: null };
+ appendResult(errRecord);
+ runningStatus.results.push(errRecord);
+ }
+ runningStatus.current++;
+ }
+ runningStatus.finished = true;
+ })().catch(err => {
+ runningStatus = { ...runningStatus, finished: true, error: err.message };
+ }).finally(() => {
+ isRunning = false;
+ });
+}
+
+// 查询运行状态
+app.get('/api/tasks/status', (_req, res) => {
+ if (!runningStatus) return res.json({ success: true, data: { isRunning: false } });
+ const elapsed = Math.round((Date.now() - runningStatus.startTime) / 1000);
+ res.json({
+ success: true,
+ data: {
+ isRunning,
+ elapsed,
+ city: runningStatus.city,
+ current: runningStatus.current,
+ total: runningStatus.total,
+ finished: runningStatus.finished,
+ error: runningStatus.error,
+ results: runningStatus.finished ? (runningStatus.results || (runningStatus.result ? [runningStatus.result] : [])) : undefined,
+ }
+ });
});
-// 批量运行多个抓取来源
-// body: { ids: ['id1','id2',...] } 不传则运行所有已启用的
-app.post('/api/scrape/run', async (req, res) => {
- try {
- const cfg = readConfig();
- let scrapers = cfg.scrapers || [];
+// 运行单个任务（立即返回）
+app.post('/api/tasks/:id/run', (req, res) => {
+ if (isRunning) return res.status(409).json({ success: false, error: '有任务正在运行中，请等待完成后再试' });
+ const cfg = readConfig();
+ const task = (cfg.tasks || []).find(t => t.id === req.params.id);
+ if (!task) return res.status(404).json({ success: false, error: '未找到该配置' });
+ isRunning = true;
+ runTaskInBackground(task);
+ res.json({ success: true, message: `任务「${task.city}」已开始执行` });
+});
- if (req.body.ids && req.body.ids.length > 0) {
- scrapers = scrapers.filter(s => req.body.ids.includes(s.id));
- } else {
- scrapers = scrapers.filter(s => s.enabled);
- }
+// 批量运行任务（立即返回）
+app.post('/api/tasks/run', (req, res) => {
+ if (isRunning) return res.status(409).json({ success: false, error: '有任务正在运行中，请等待完成后再试' });
+ const cfg = readConfig();
+ let tasks = cfg.tasks || [];
- if (scrapers.length === 0) {
- return res.json({ success: true, data: [], message: '没有可运行的抓取来源' });
- }
-
- const results = [];
- for (const scraper of scrapers) {
- try {
- const r = await runScraper(scraper);
- results.push(r);
- } catch (err) {
- const errRecord = {
- scraperId: scraper.id,
- city: scraper.city,
- section: scraper.section,
- subsection: scraper.subsection,
- type: scraper.type,
- url: scraper.url,
- scrapedAt: new Date().toISOString(),
- error: err.message,
- data: null,
- };
- appendResult(errRecord);
- results.push(errRecord);
- }
- }
-
- res.json({ success: true, data: results });
- } catch (e) {
- res.status(500).json({ success: false, error: e.message });
+ if (req.body.ids && req.body.ids.length > 0) {
+ tasks = tasks.filter(t => req.body.ids.includes(t.id));
+ } else {
+ tasks = tasks.filter(t => t.enabled);
}
+
+ if (tasks.length === 0) {
+ return res.json({ success: true, data: [], message: '没有可运行的任务' });
+ }
+
+ isRunning = true;
+ runTasksInBackground(tasks);
+ res.json({ success: true, message: `${tasks.length} 个任务已开始执行` });
});
// ========== 配置管理 ==========