chore(config): 更新.gitignore文件以忽略数据库相关文件

添加了data/目录、SQLite数据库文件及相关临时文件到.gitignore中,
避免敏感数据和临时文件被提交到版本控制系统。
```
This commit is contained in:
2026-03-19 10:18:25 +08:00
parent bd46d8f907
commit d78dc655ee
10 changed files with 1367 additions and 1538 deletions

View File

@@ -49,7 +49,6 @@ async function fetchWithRetry(url, fetchOptions, retries = MAX_FETCH_RETRIES, lo
*/
async function createTask(prompt, options = {}) {
const baseUrl = options.baseUrl || DEFAULT_BASE_URL;
const useBrowser = options.useBrowser ?? false;
const mode = normalizeMode(options.mode);
const taskId = generateTaskId();
const logPrefix = options.logPrefix || '[Agent]';
@@ -57,7 +56,7 @@ async function createTask(prompt, options = {}) {
const res = await fetchWithRetry(`${baseUrl}/agent/createTask`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ taskId, prompt, useBrowser, mode }),
body: JSON.stringify({ taskId, prompt, mode }),
}, MAX_FETCH_RETRIES, logPrefix);
if (!res.ok) {
@@ -104,7 +103,6 @@ async function checkTask(taskId, options = {}) {
*/
export async function runAgentTask(prompt, options = {}) {
const baseUrl = options.baseUrl || DEFAULT_BASE_URL;
const useBrowser = options.useBrowser ?? false;
const mode = normalizeMode(options.mode);
const pollInterval = options.pollInterval || DEFAULT_POLL_INTERVAL;
const timeout = options.timeout || DEFAULT_TIMEOUT;
@@ -112,7 +110,7 @@ export async function runAgentTask(prompt, options = {}) {
console.log(`${logPrefix} 创建任务...`);
console.log(`${logPrefix} 使用 mode: ${mode}`);
const { taskId } = await createTask(prompt, { baseUrl, useBrowser, mode, logPrefix });
const { taskId } = await createTask(prompt, { baseUrl, mode, logPrefix });
console.log(`${logPrefix} 任务已创建: ${taskId}`);
const startTime = Date.now();

471
src/resultStore.js Normal file
View File

@@ -0,0 +1,471 @@
import Database from 'better-sqlite3';
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
import { fileURLToPath } from 'url';
import { dirname, join } from 'path';
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const DB_PATH =
process.env.APP_DB_PATH ||
process.env.RESULTS_DB_PATH ||
join(__dirname, '..', 'data', 'results.sqlite');
const CONFIG_PATH = join(__dirname, '..', 'config.json');
const MAX_RESULT_RECORDS = 500;
const DEFAULT_TASK_MODE = 'qwen3.5-plus';
const TASK_COLUMNS = ['id', 'city', 'plate_name', 'prompt', 'enabled', 'mode', 'created_at', 'updated_at'];
let db = null;
let initialized = false;
function clone(value) {
return JSON.parse(JSON.stringify(value));
}
function generateResultId() {
return `result-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`;
}
function generateTaskId() {
return `task-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`;
}
function getDefaultJsonConfig() {
return {
agent: {
baseUrl: '',
pollInterval: 3000,
timeout: 300000,
},
scheduler: {
enabled: false,
cronTime: '0 9 * * *',
threshold: 100000,
description: '',
timeRange: 'thisMonth',
},
email: {
smtpHost: '',
smtpPort: 587,
smtpUser: '',
smtpPass: '',
recipients: '',
},
};
}
function normalizeJsonConfig(input = {}) {
const defaults = getDefaultJsonConfig();
const agent = input.agent || {};
const scheduler = input.scheduler || {};
const email = input.email || {};
return {
agent: {
baseUrl: typeof agent.baseUrl === 'string' ? agent.baseUrl : defaults.agent.baseUrl,
pollInterval: Number.isFinite(agent.pollInterval) ? agent.pollInterval : defaults.agent.pollInterval,
timeout: Number.isFinite(agent.timeout) ? agent.timeout : defaults.agent.timeout,
},
scheduler: {
enabled: scheduler.enabled === true,
cronTime: typeof scheduler.cronTime === 'string' && scheduler.cronTime.trim()
? scheduler.cronTime
: defaults.scheduler.cronTime,
threshold: Number.isFinite(scheduler.threshold) ? scheduler.threshold : defaults.scheduler.threshold,
description: typeof scheduler.description === 'string' ? scheduler.description : defaults.scheduler.description,
timeRange: typeof scheduler.timeRange === 'string' && scheduler.timeRange.trim()
? scheduler.timeRange
: defaults.scheduler.timeRange,
},
email: {
smtpHost: typeof email.smtpHost === 'string' ? email.smtpHost : defaults.email.smtpHost,
smtpPort: Number.isFinite(email.smtpPort) ? email.smtpPort : defaults.email.smtpPort,
smtpUser: typeof email.smtpUser === 'string' ? email.smtpUser : defaults.email.smtpUser,
smtpPass: typeof email.smtpPass === 'string' ? email.smtpPass : defaults.email.smtpPass,
recipients: typeof email.recipients === 'string' ? email.recipients : defaults.email.recipients,
},
};
}
function normalizeTaskMode(value) {
if (typeof value === 'string' && value.trim()) return value.trim();
return DEFAULT_TASK_MODE;
}
function buildTaskRecord(task = {}) {
return {
id: task.id || generateTaskId(),
city: task.city || '',
plateName: task.plateName || '',
prompt: task.prompt || '',
enabled: task.enabled !== false,
mode: normalizeTaskMode(task.mode),
};
}
function buildResultRecord(result = {}) {
return {
id: result.id || generateResultId(),
taskId: result.taskId || null,
city: result.city || null,
scrapedAt: result.scrapedAt || new Date().toISOString(),
error: result.error || null,
data: result.data ?? null,
};
}
function parseTaskRow(row) {
return {
id: row.id,
city: row.city,
plateName: row.plate_name,
prompt: row.prompt,
enabled: row.enabled === 1,
mode: normalizeTaskMode(row.mode),
};
}
function parseResultRow(row) {
return {
id: row.id,
taskId: row.task_id,
city: row.city,
scrapedAt: row.scraped_at,
error: row.error,
data: row.data_json ? JSON.parse(row.data_json) : null,
};
}
function getDb() {
if (db) return db;
mkdirSync(dirname(DB_PATH), { recursive: true });
db = new Database(DB_PATH);
db.pragma('journal_mode = WAL');
return db;
}
function ensureSchema() {
getDb().exec(`
CREATE TABLE IF NOT EXISTS results (
id TEXT PRIMARY KEY,
task_id TEXT,
city TEXT,
scraped_at TEXT NOT NULL,
error TEXT,
data_json TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now'))
);
CREATE INDEX IF NOT EXISTS idx_results_scraped_at ON results (scraped_at DESC);
CREATE INDEX IF NOT EXISTS idx_results_city ON results (city);
CREATE INDEX IF NOT EXISTS idx_results_task_id ON results (task_id);
CREATE TABLE IF NOT EXISTS tasks (
id TEXT PRIMARY KEY,
city TEXT,
plate_name TEXT,
prompt TEXT,
enabled INTEGER NOT NULL DEFAULT 1,
mode TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
);
`);
}
function ensureTasksTableShape() {
const columns = getDb().prepare(`PRAGMA table_info(tasks)`).all();
const columnNames = columns.map((column) => column.name);
const hasLegacyBrowserColumn = columnNames.includes('use_browser');
const matchesExpectedShape =
columnNames.length === TASK_COLUMNS.length &&
TASK_COLUMNS.every((column, index) => columnNames[index] === column);
if (!hasLegacyBrowserColumn && matchesExpectedShape) return;
getDb().exec(`
BEGIN;
ALTER TABLE tasks RENAME TO tasks_legacy;
CREATE TABLE tasks (
id TEXT PRIMARY KEY,
city TEXT,
plate_name TEXT,
prompt TEXT,
enabled INTEGER NOT NULL DEFAULT 1,
mode TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
);
INSERT INTO tasks (id, city, plate_name, prompt, enabled, mode, created_at, updated_at)
SELECT
id,
city,
plate_name,
prompt,
COALESCE(enabled, 1),
COALESCE(mode, '${DEFAULT_TASK_MODE}'),
COALESCE(created_at, datetime('now')),
COALESCE(updated_at, datetime('now'))
FROM tasks_legacy;
DROP TABLE tasks_legacy;
COMMIT;
`);
}
function trimResults(limit = MAX_RESULT_RECORDS) {
getDb().prepare(`
DELETE FROM results
WHERE id NOT IN (
SELECT id
FROM results
ORDER BY scraped_at DESC, rowid DESC
LIMIT ?
)
`).run(limit);
}
function readJsonIfExists(filePath) {
if (!existsSync(filePath)) return null;
return JSON.parse(readFileSync(filePath, 'utf-8'));
}
function stripTasksFromConfig(config) {
if (!config || typeof config !== 'object') return getDefaultJsonConfig();
const { agent, scheduler, email } = config;
return normalizeJsonConfig({ agent, scheduler, email });
}
function ensureJsonConfigExists() {
if (existsSync(CONFIG_PATH)) return;
writeFileSync(CONFIG_PATH, JSON.stringify(getDefaultJsonConfig(), null, 2), 'utf-8');
}
function queryBaseRows({ city, taskId }) {
const clauses = [];
const params = [];
if (city) {
clauses.push('city = ?');
params.push(city);
}
if (taskId) {
clauses.push('task_id = ?');
params.push(taskId);
}
const whereSql = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
const sql = `
SELECT id, task_id, city, scraped_at, error, data_json
FROM results
${whereSql}
ORDER BY scraped_at DESC, rowid DESC
`;
return getDb().prepare(sql).all(...params).map(parseResultRow);
}
function matchSection(record, section) {
if (!section) return true;
if (record.section === section || record.subsection === section) return true;
const items = record.data?.results || [];
return items.some((item) => item.section === section || item.subsection === section);
}
function matchType(record, type) {
if (!type) return true;
if (record.type === type) return true;
const items = record.data?.results || [];
return items.some((item) => item.type === type);
}
export function initResultsStore() {
if (initialized) return;
ensureSchema();
ensureTasksTableShape();
ensureJsonConfigExists();
initialized = true;
}
export function loadConfig() {
initResultsStore();
const jsonConfig = normalizeJsonConfig(readJsonIfExists(CONFIG_PATH) || getDefaultJsonConfig());
return {
...clone(jsonConfig),
tasks: listTasks(),
};
}
export function saveConfig(config) {
initResultsStore();
const jsonConfig = stripTasksFromConfig(config);
writeFileSync(CONFIG_PATH, JSON.stringify(jsonConfig, null, 2), 'utf-8');
return {
...clone(jsonConfig),
tasks: listTasks(),
};
}
export function listTasks() {
initResultsStore();
return getDb()
.prepare(`
SELECT id, city, plate_name, prompt, enabled, mode
FROM tasks
ORDER BY rowid DESC
`)
.all()
.map(parseTaskRow);
}
export function getTaskById(id) {
initResultsStore();
const row = getDb()
.prepare(`
SELECT id, city, plate_name, prompt, enabled, mode
FROM tasks
WHERE id = ?
`)
.get(id);
return row ? parseTaskRow(row) : null;
}
export function createTask(task) {
initResultsStore();
const record = buildTaskRecord(task);
getDb().prepare(`
INSERT INTO tasks (id, city, plate_name, prompt, enabled, mode, updated_at)
VALUES (?, ?, ?, ?, ?, ?, datetime('now'))
`).run(
record.id,
record.city,
record.plateName,
record.prompt,
record.enabled ? 1 : 0,
record.mode,
);
return record;
}
export function updateTask(id, patch) {
initResultsStore();
const current = getTaskById(id);
if (!current) return null;
const next = buildTaskRecord({ ...current, ...patch, id });
getDb().prepare(`
UPDATE tasks
SET city = ?,
plate_name = ?,
prompt = ?,
enabled = ?,
mode = ?,
updated_at = datetime('now')
WHERE id = ?
`).run(
next.city,
next.plateName,
next.prompt,
next.enabled ? 1 : 0,
next.mode,
id,
);
return next;
}
export function deleteTaskById(id) {
initResultsStore();
const result = getDb().prepare('DELETE FROM tasks WHERE id = ?').run(id);
return result.changes > 0;
}
export function appendResult(result) {
initResultsStore();
const record = buildResultRecord(result);
getDb().prepare(`
INSERT INTO results (id, task_id, city, scraped_at, error, data_json)
VALUES (?, ?, ?, ?, ?, ?)
`).run(
record.id,
record.taskId,
record.city,
record.scrapedAt,
record.error,
record.data === null ? null : JSON.stringify(record.data),
);
trimResults();
return record;
}
export function listResults({ city, section, type, taskId, page = 1, pageSize = 20 } = {}) {
initResultsStore();
let results = queryBaseRows({ city, taskId });
if (section) results = results.filter((record) => matchSection(record, section));
if (type) results = results.filter((record) => matchType(record, type));
const normalizedPage = Math.max(1, parseInt(page, 10) || 1);
const normalizedPageSize = Math.max(1, parseInt(pageSize, 10) || 20);
const start = (normalizedPage - 1) * normalizedPageSize;
return {
total: results.length,
page: normalizedPage,
pageSize: normalizedPageSize,
data: results.slice(start, start + normalizedPageSize),
};
}
export function deleteResultById(id) {
initResultsStore();
const result = getDb().prepare('DELETE FROM results WHERE id = ?').run(id);
return result.changes > 0;
}
export function clearResults() {
initResultsStore();
getDb().prepare('DELETE FROM results').run();
}
export function getResultFilters() {
initResultsStore();
const rows = queryBaseRows({});
const cities = [...new Set(rows.map((row) => row.city).filter(Boolean))];
const sections = new Set();
const types = new Set();
for (const row of rows) {
if (row.section) sections.add(row.section);
if (row.subsection) sections.add(row.subsection);
if (row.type) types.add(row.type);
for (const item of row.data?.results || []) {
if (item.section) sections.add(item.section);
if (item.subsection) sections.add(item.subsection);
if (item.type) types.add(item.type);
}
}
return {
cities,
sections: [...sections],
types: [...types],
};
}
export function getResultsDbPath() {
return DB_PATH;
}

View File

@@ -1,208 +1,177 @@
import 'dotenv/config';
import cron from 'node-cron';
import { readFileSync, writeFileSync, existsSync } from 'fs';
import { fileURLToPath } from 'url';
import { dirname, join } from 'path';
import { sendScraperResultsEmail } from './emailService.js';
import { runAgentTask } from './agentService.js';
import { initResultsStore, loadConfig, appendResult } from './resultStore.js';
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const RESULTS_PATH = join(__dirname, '..', 'results.json');
const DEFAULT_TASK_MODE = 'qwen3.5-plus';
let currentScheduledTask = null;
function normalizeTaskMode(value) {
if (typeof value === 'string' && value.trim()) return value.trim();
return DEFAULT_TASK_MODE;
}
function loadConfig() {
try {
const configPath = join(__dirname, '..', 'config.json');
return JSON.parse(readFileSync(configPath, 'utf-8'));
} catch (error) {
console.error('加载配置文件失败:', error.message);
return null;
}
}
// ========== 结果存取 ==========
function readResults() {
if (!existsSync(RESULTS_PATH)) return [];
try {
return JSON.parse(readFileSync(RESULTS_PATH, 'utf-8'));
} catch (e) {
return [];
}
}
function saveResults(results) {
writeFileSync(RESULTS_PATH, JSON.stringify(results, null, 2), 'utf-8');
}
function appendResult(result) {
const results = readResults();
results.unshift({ ...result, id: `result-${Date.now()}-${Math.random().toString(36).slice(2, 7)}` });
if (results.length > 500) results.splice(500);
saveResults(results);
}
// ========== 任务执行 ==========
async function runTask(task, agentCfg) {
const useBrowser = typeof task.useBrowser === 'boolean' ? task.useBrowser : agentCfg.useBrowser;
const mode = normalizeTaskMode(task.mode);
console.log(`[定时任务][Agent] ${task.city}:开始执行`);
console.log(`[定时任务][Agent] ${task.city}mode=${mode}`);
console.log(`[Scheduler][Agent] ${task.city}: start`);
console.log(`[Scheduler][Agent] ${task.city}: mode=${mode}`);
const { results } = await runAgentTask(task.prompt, {
baseUrl: agentCfg.baseUrl,
useBrowser,
mode,
pollInterval: agentCfg.pollInterval,
timeout: agentCfg.timeout,
logPrefix: `[定时任务][Agent][${task.city}]`,
logPrefix: `[Scheduler][Agent][${task.city}]`,
});
console.log(`[定时任务][Agent] ${task.city}:获取到 ${results.length} 条结果`);
const record = {
console.log(`[Scheduler][Agent] ${task.city}: ${results.length} results`);
return appendResult({
taskId: task.id,
city: task.city,
scrapedAt: new Date().toISOString(),
data: { results, total: results.length },
};
appendResult(record);
return record;
});
}
// ========== 定时任务执行函数 ==========
async function executeScheduledTask(config) {
try {
console.log('========================================');
console.log('定时任务开始执行');
console.log('执行时间:', new Date().toLocaleString('zh-CN'));
console.log('Scheduler started');
console.log('Time:', new Date().toLocaleString('zh-CN'));
console.log('========================================');
const tasks = (config.tasks || []).filter(t => t.enabled);
const tasks = (config.tasks || []).filter((task) => task.enabled);
const agentCfg = config.agent || {};
if (tasks.length === 0) {
console.log('没有已启用的任务,跳过');
console.log('No enabled tasks, skip');
return;
}
console.log(` ${tasks.length} 个已启用的任务`);
console.log(`Enabled tasks: ${tasks.length}`);
const results = [];
for (const task of tasks) {
try {
console.log(`\n---------- 任务: ${task.city} ----------`);
const r = await runTask(task, agentCfg);
results.push(r);
console.log(`✓ 执行成功`);
} catch (err) {
console.error(`✗ 执行失败: ${err.message}`);
const errRecord = {
console.log(`\n---------- Task: ${task.city} ----------`);
const record = await runTask(task, agentCfg);
results.push(record);
console.log('Task completed');
} catch (error) {
console.error(`Task failed: ${error.message}`);
const errorRecord = appendResult({
taskId: task.id,
city: task.city,
scrapedAt: new Date().toISOString(),
error: err.message,
error: error.message,
data: null,
};
appendResult(errRecord);
results.push(errRecord);
});
results.push(errorRecord);
}
}
const successCount = results.filter(r => !r.error).length;
const failCount = results.filter(r => r.error).length;
console.log(`\n========== 执行完成 ==========`);
console.log(`成功: ${successCount},失败: ${failCount}`);
const successCount = results.filter((item) => !item.error).length;
const failCount = results.filter((item) => item.error).length;
console.log('\n========== Scheduler finished ==========');
console.log(`Success: ${successCount}, Failed: ${failCount}`);
if (successCount === 0) {
console.log('没有成功的结果,不发送邮件');
console.log('No successful results, skip email');
return;
}
if (config.email?.smtpHost && config.email?.smtpUser) {
console.log('\n正在发送结果邮件...');
console.log('\nSending email...');
try {
const emailResult = await sendScraperResultsEmail(config.email, results);
console.log('邮件发送成功! MessageId:', emailResult.messageId);
} catch (emailErr) {
console.error('邮件发送失败:', emailErr.message);
console.log('Email sent:', emailResult.messageId);
} catch (error) {
console.error('Email failed:', error.message);
}
} else {
console.log('邮件配置不完整,跳过邮件发送');
console.log('Email config incomplete, skip email');
}
console.log('========================================');
} catch (error) {
console.error('========================================');
console.error('定时任务执行失败:', error.message);
console.error('Scheduler failed:', error.message);
console.error(error.stack);
console.error('========================================');
}
}
// 存储当前的定时任务
let currentScheduledTask = null;
export function initScheduler() {
initResultsStore();
const config = loadConfig();
if (!config) { console.error('无法启动定时任务: 配置文件加载失败'); return; }
if (!config.scheduler?.enabled) { console.log('定时任务已禁用'); return; }
if (!config.scheduler?.enabled) {
console.log('Scheduler disabled');
return;
}
const cronTime = config.scheduler.cronTime || '0 9 * * *';
const enabledCount = (config.tasks || []).filter(t => t.enabled).length;
const enabledCount = (config.tasks || []).filter((task) => task.enabled).length;
console.log('========================================');
console.log('定时任务已启动,执行计划:', cronTime);
console.log(`已启用的任务: ${enabledCount}`);
if (config.email?.recipients) console.log('收件人:', config.email.recipients);
console.log('Scheduler enabled:', cronTime);
console.log(`Enabled tasks: ${enabledCount}`);
if (config.email?.recipients) {
console.log('Recipients:', config.email.recipients);
}
console.log('========================================');
if (currentScheduledTask) { currentScheduledTask.stop(); }
if (currentScheduledTask) {
currentScheduledTask.stop();
}
currentScheduledTask = cron.schedule(cronTime, () => {
const latestConfig = loadConfig();
if (latestConfig) {
executeScheduledTask(latestConfig);
}
}, { timezone: 'Asia/Shanghai' });
currentScheduledTask = cron.schedule(
cronTime,
() => {
executeScheduledTask(loadConfig());
},
{ timezone: 'Asia/Shanghai' },
);
}
export function reloadScheduler() {
console.log('重新加载定时任务配置...');
if (currentScheduledTask) { currentScheduledTask.stop(); currentScheduledTask = null; }
console.log('Reloading scheduler...');
if (currentScheduledTask) {
currentScheduledTask.stop();
currentScheduledTask = null;
}
initScheduler();
}
export function stopScheduler() {
if (currentScheduledTask) {
currentScheduledTask.stop(); currentScheduledTask = null;
console.log('定时任务已停止'); return true;
}
return false;
if (!currentScheduledTask) return false;
currentScheduledTask.stop();
currentScheduledTask = null;
console.log('Scheduler stopped');
return true;
}
export function getSchedulerStatus() {
const config = loadConfig();
const enabledTasks = (config?.tasks || []).filter(t => t.enabled).length;
const enabledTasks = (config.tasks || []).filter((task) => task.enabled).length;
return {
isRunning: currentScheduledTask !== null,
enabledTasks,
config: config ? {
config: {
enabled: config.scheduler?.enabled || false,
cronTime: config.scheduler?.cronTime || '0 9 * * *',
description: config.scheduler?.description || '',
} : null,
},
};
}
export async function runTaskNow() {
const config = loadConfig();
if (!config) throw new Error('配置文件加载失败');
await executeScheduledTask(config);
initResultsStore();
await executeScheduledTask(loadConfig());
}

View File

@@ -1,263 +1,272 @@
import 'dotenv/config';
import express from 'express';
import cors from 'cors';
import { readFileSync, writeFileSync, existsSync } from 'fs';
import { fileURLToPath } from 'url';
import { dirname, join } from 'path';
import {
initResultsStore,
loadConfig,
saveConfig,
listTasks,
getTaskById,
createTask,
updateTask,
deleteTaskById,
listResults,
deleteResultById,
clearResults,
getResultFilters,
appendResult,
} from './resultStore.js';
import { initScheduler, runTaskNow, reloadScheduler, getSchedulerStatus } from './scheduler.js';
import { runAgentTask } from './agentService.js';
const app = express();
const PORT = process.env.PORT || 5000;
const DEFAULT_TASK_MODE = 'qwen3.5-plus';
const MASKED_PASSWORD = '***已配置***';
app.use(cors());
app.use(express.json({ limit: '50mb' }));
app.use(express.urlencoded({ limit: '50mb', extended: true }));
app.use(express.static('public'));
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const CONFIG_PATH = join(__dirname, '..', 'config.json');
const RESULTS_PATH = join(__dirname, '..', 'results.json');
const DEFAULT_TASK_MODE = 'qwen3.5-plus';
function readConfig() {
return JSON.parse(readFileSync(CONFIG_PATH, 'utf-8'));
}
function saveConfig(cfg) {
writeFileSync(CONFIG_PATH, JSON.stringify(cfg, null, 2), 'utf-8');
}
function normalizeUseBrowser(value) {
return value === true || value === 'true';
}
function normalizeTaskMode(value) {
if (typeof value === 'string' && value.trim()) return value.trim();
return DEFAULT_TASK_MODE;
}
// ========== 抓取结果存取 ==========
function buildTaskPayload(body = {}, { partial = false } = {}) {
const payload = {};
function readResults() {
if (!existsSync(RESULTS_PATH)) return [];
try {
return JSON.parse(readFileSync(RESULTS_PATH, 'utf-8'));
} catch (e) {
return [];
if (!partial || Object.prototype.hasOwnProperty.call(body, 'city')) {
payload.city = body.city || '';
}
if (!partial || Object.prototype.hasOwnProperty.call(body, 'plateName')) {
payload.plateName = body.plateName || '';
}
if (!partial || Object.prototype.hasOwnProperty.call(body, 'prompt')) {
payload.prompt = body.prompt || '';
}
if (!partial || Object.prototype.hasOwnProperty.call(body, 'enabled')) {
payload.enabled = body.enabled !== false;
}
if (!partial || Object.prototype.hasOwnProperty.call(body, 'mode')) {
payload.mode = normalizeTaskMode(body.mode);
}
return payload;
}
function saveResults(results) {
writeFileSync(RESULTS_PATH, JSON.stringify(results, null, 2), 'utf-8');
}
function appendResult(result) {
const results = readResults();
results.unshift({ ...result, id: `result-${Date.now()}-${Math.random().toString(36).slice(2, 7)}` });
if (results.length > 500) results.splice(500);
saveResults(results);
}
// 查询结果(支持分页与筛选)
app.get('/api/results', (req, res) => {
try {
const { city, type, page = 1, pageSize = 20, taskId } = req.query;
let results = readResults();
if (city) results = results.filter(r => r.city === city);
if (type) results = results.filter(r => {
const items = r.data?.results || [];
return items.some(item => item.type === type);
});
if (taskId) results = results.filter(r => r.taskId === taskId);
const total = results.length;
const start = (parseInt(page) - 1) * parseInt(pageSize);
const data = results.slice(start, start + parseInt(pageSize));
res.json({ success: true, total, page: parseInt(page), pageSize: parseInt(pageSize), data });
} catch (e) {
res.status(500).json({ success: false, error: e.message });
}
});
// 删除单条结果
app.delete('/api/results/:id', (req, res) => {
try {
const results = readResults();
const before = results.length;
const updated = results.filter(r => r.id !== req.params.id);
if (updated.length === before) return res.status(404).json({ success: false, error: '未找到' });
saveResults(updated);
res.json({ success: true });
} catch (e) {
res.status(500).json({ success: false, error: e.message });
}
});
// 清空所有结果
app.delete('/api/results', (_req, res) => {
try {
saveResults([]);
res.json({ success: true, message: '已清空所有结果' });
} catch (e) {
res.status(500).json({ success: false, error: e.message });
}
});
// 获取结果的筛选选项
app.get('/api/results/filters', (req, res) => {
try {
const results = readResults();
const cities = [...new Set(results.map(r => r.city).filter(Boolean))];
const types = new Set();
for (const r of results) {
for (const item of (r.data?.results || [])) {
if (item.type) types.add(item.type);
}
}
res.json({ success: true, data: { cities, types: [...types] } });
} catch (e) {
res.status(500).json({ success: false, error: e.message });
}
});
// ========== 任务配置 CRUD ==========
app.get('/api/tasks', (req, res) => {
try {
const cfg = readConfig();
const tasks = (cfg.tasks || []).map(t => ({ ...t, mode: normalizeTaskMode(t.mode) }));
res.json({ success: true, data: tasks });
} catch (e) {
res.status(500).json({ success: false, error: e.message });
}
});
app.post('/api/tasks', (req, res) => {
try {
const cfg = readConfig();
if (!cfg.tasks) cfg.tasks = [];
const item = {
id: `task-${Date.now()}`,
city: req.body.city || '',
prompt: req.body.prompt || '',
enabled: req.body.enabled !== false,
useBrowser: normalizeUseBrowser(req.body.useBrowser),
mode: normalizeTaskMode(req.body.mode),
function maskConfigSecrets(config) {
const next = { ...config };
if (config.email) {
next.email = {
...config.email,
smtpPass: config.email.smtpPass ? MASKED_PASSWORD : '',
};
cfg.tasks.push(item);
saveConfig(cfg);
res.json({ success: true, data: item });
} catch (e) {
res.status(500).json({ success: false, error: e.message });
}
});
return next;
}
app.put('/api/tasks/:id', (req, res) => {
try {
const cfg = readConfig();
const idx = (cfg.tasks || []).findIndex(t => t.id === req.params.id);
if (idx === -1) return res.status(404).json({ success: false, error: '未找到该配置' });
const patch = { ...req.body };
if (Object.prototype.hasOwnProperty.call(patch, 'useBrowser')) {
patch.useBrowser = normalizeUseBrowser(patch.useBrowser);
}
if (Object.prototype.hasOwnProperty.call(patch, 'mode')) {
patch.mode = normalizeTaskMode(patch.mode);
}
cfg.tasks[idx] = { ...cfg.tasks[idx], ...patch, id: req.params.id };
saveConfig(cfg);
res.json({ success: true, data: cfg.tasks[idx] });
} catch (e) {
res.status(500).json({ success: false, error: e.message });
function mergeConfigWithExistingSecrets(incoming = {}) {
const current = loadConfig();
const next = {
...current,
...incoming,
agent: { ...(current.agent || {}), ...(incoming.agent || {}) },
scheduler: { ...(current.scheduler || {}), ...(incoming.scheduler || {}) },
email: { ...(current.email || {}), ...(incoming.email || {}) },
};
if (next.email?.smtpPass === MASKED_PASSWORD) {
next.email.smtpPass = current.email?.smtpPass || '';
}
});
app.delete('/api/tasks/:id', (req, res) => {
try {
const cfg = readConfig();
const before = (cfg.tasks || []).length;
cfg.tasks = (cfg.tasks || []).filter(t => t.id !== req.params.id);
if (cfg.tasks.length === before) return res.status(404).json({ success: false, error: '未找到' });
saveConfig(cfg);
res.json({ success: true });
} catch (e) {
res.status(500).json({ success: false, error: e.message });
}
});
// ========== 任务执行(异步 + 串行锁) ==========
return next;
}
let isRunning = false;
let runningStatus = null; // { taskId, city, startTime, current, total, finished, error }
let runningStatus = null;
async function runTask(task) {
const cfg = readConfig();
const agentCfg = cfg.agent || {};
const useBrowser = typeof task.useBrowser === 'boolean' ? task.useBrowser : agentCfg.useBrowser;
const config = loadConfig();
const agentCfg = config.agent || {};
const mode = normalizeTaskMode(task.mode);
console.log(`[Agent] ${task.city}:开始执行`);
console.log(`[Agent] ${task.city}mode=${mode}`);
console.log(`[Agent] ${task.city}: start`);
console.log(`[Agent] ${task.city}: mode=${mode}`);
const { results } = await runAgentTask(task.prompt, {
baseUrl: agentCfg.baseUrl,
useBrowser,
mode,
pollInterval: agentCfg.pollInterval,
timeout: agentCfg.timeout,
logPrefix: `[Agent][${task.city}]`,
});
const record = {
return appendResult({
taskId: task.id,
city: task.city,
scrapedAt: new Date().toISOString(),
data: { results, total: results.length },
};
appendResult(record);
return record;
}
// 后台执行单个任务
function runTaskInBackground(task) {
runningStatus = { taskId: task.id, city: task.city, startTime: Date.now(), current: 0, total: 1, finished: false, error: null };
runTask(task).then(record => {
runningStatus = { ...runningStatus, finished: true, result: record, current: 1 };
}).catch(err => {
console.error('任务执行失败:', err.message);
runningStatus = { ...runningStatus, finished: true, error: err.message, current: 1 };
}).finally(() => {
isRunning = false;
});
}
// 后台执行批量任务
function runTaskInBackground(task) {
runningStatus = {
taskId: task.id,
city: task.city,
startTime: Date.now(),
current: 0,
total: 1,
finished: false,
error: null,
};
runTask(task)
.then((record) => {
runningStatus = { ...runningStatus, finished: true, result: record, current: 1 };
})
.catch((error) => {
console.error('任务执行失败:', error.message);
runningStatus = { ...runningStatus, finished: true, error: error.message, current: 1 };
})
.finally(() => {
isRunning = false;
});
}
function runTasksInBackground(tasks) {
runningStatus = { taskId: null, city: null, startTime: Date.now(), current: 0, total: tasks.length, finished: false, error: null, results: [] };
runningStatus = {
taskId: null,
city: null,
startTime: Date.now(),
current: 0,
total: tasks.length,
finished: false,
error: null,
results: [],
};
(async () => {
for (const task of tasks) {
runningStatus = { ...runningStatus, taskId: task.id, city: task.city };
try {
const r = await runTask(task);
runningStatus.results.push(r);
} catch (err) {
const errRecord = { taskId: task.id, city: task.city, scrapedAt: new Date().toISOString(), error: err.message, data: null };
appendResult(errRecord);
runningStatus.results.push(errRecord);
const record = await runTask(task);
runningStatus.results.push(record);
} catch (error) {
const errorRecord = appendResult({
taskId: task.id,
city: task.city,
scrapedAt: new Date().toISOString(),
error: error.message,
data: null,
});
runningStatus.results.push(errorRecord);
}
runningStatus.current++;
runningStatus.current += 1;
}
runningStatus.finished = true;
})().catch(err => {
runningStatus = { ...runningStatus, finished: true, error: err.message };
}).finally(() => {
isRunning = false;
});
})()
.catch((error) => {
runningStatus = { ...runningStatus, finished: true, error: error.message };
})
.finally(() => {
isRunning = false;
});
}
// 查询运行状态
app.get('/api/results', (req, res) => {
try {
const { city, section, type, page = 1, pageSize = 20, taskId } = req.query;
const result = listResults({ city, section, type, page, pageSize, taskId });
res.json({ success: true, ...result });
} catch (error) {
res.status(500).json({ success: false, error: error.message });
}
});
app.delete('/api/results/:id', (req, res) => {
try {
const deleted = deleteResultById(req.params.id);
if (!deleted) {
return res.status(404).json({ success: false, error: '未找到结果' });
}
res.json({ success: true });
} catch (error) {
res.status(500).json({ success: false, error: error.message });
}
});
app.delete('/api/results', (_req, res) => {
try {
clearResults();
res.json({ success: true, message: '已清空所有结果' });
} catch (error) {
res.status(500).json({ success: false, error: error.message });
}
});
app.get('/api/results/filters', (_req, res) => {
try {
res.json({ success: true, data: getResultFilters() });
} catch (error) {
res.status(500).json({ success: false, error: error.message });
}
});
app.get('/api/tasks', (_req, res) => {
try {
res.json({ success: true, data: listTasks() });
} catch (error) {
res.status(500).json({ success: false, error: error.message });
}
});
app.post('/api/tasks', (req, res) => {
try {
const task = createTask(buildTaskPayload(req.body));
res.json({ success: true, data: task });
} catch (error) {
res.status(500).json({ success: false, error: error.message });
}
});
app.put('/api/tasks/:id', (req, res) => {
try {
const task = updateTask(req.params.id, buildTaskPayload(req.body, { partial: true }));
if (!task) {
return res.status(404).json({ success: false, error: '未找到该任务' });
}
res.json({ success: true, data: task });
} catch (error) {
res.status(500).json({ success: false, error: error.message });
}
});
app.delete('/api/tasks/:id', (req, res) => {
try {
const deleted = deleteTaskById(req.params.id);
if (!deleted) {
return res.status(404).json({ success: false, error: '未找到该任务' });
}
res.json({ success: true });
} catch (error) {
res.status(500).json({ success: false, error: error.message });
}
});
app.get('/api/tasks/status', (_req, res) => {
if (!runningStatus) return res.json({ success: true, data: { isRunning: false } });
if (!runningStatus) {
return res.json({ success: true, data: { isRunning: false } });
}
const elapsed = Math.round((Date.now() - runningStatus.startTime) / 1000);
res.json({
success: true,
@@ -269,109 +278,119 @@ app.get('/api/tasks/status', (_req, res) => {
total: runningStatus.total,
finished: runningStatus.finished,
error: runningStatus.error,
results: runningStatus.finished ? (runningStatus.results || (runningStatus.result ? [runningStatus.result] : [])) : undefined,
}
results: runningStatus.finished
? (runningStatus.results || (runningStatus.result ? [runningStatus.result] : []))
: undefined,
},
});
});
// 运行单个任务(立即返回)
app.post('/api/tasks/:id/run', (req, res) => {
if (isRunning) return res.status(409).json({ success: false, error: '有任务正在运行中,请等待完成后再试' });
const cfg = readConfig();
const task = (cfg.tasks || []).find(t => t.id === req.params.id);
if (!task) return res.status(404).json({ success: false, error: '未找到该配置' });
isRunning = true;
runTaskInBackground(task);
res.json({ success: true, message: `任务「${task.city}」已开始执行` });
});
// 批量运行任务(立即返回)
app.post('/api/tasks/run', (req, res) => {
if (isRunning) return res.status(409).json({ success: false, error: '有任务正在运行中,请等待完成后再试' });
const cfg = readConfig();
let tasks = cfg.tasks || [];
if (req.body.ids && req.body.ids.length > 0) {
tasks = tasks.filter(t => req.body.ids.includes(t.id));
} else {
tasks = tasks.filter(t => t.enabled);
if (isRunning) {
return res.status(409).json({ success: false, error: '当前已有任务在运行,请稍后再试' });
}
if (tasks.length === 0) {
return res.json({ success: true, data: [], message: '没有可运行的任务' });
}
isRunning = true;
runTasksInBackground(tasks);
res.json({ success: true, message: `${tasks.length} 个任务已开始执行` });
});
// ========== 配置管理 ==========
app.get('/api/config', (req, res) => {
try {
const cfg = readConfig();
if (cfg.email?.smtpPass) cfg.email.smtpPass = '***已配置***';
res.json({ success: true, data: cfg });
} catch (e) {
res.status(500).json({ success: false, error: e.message });
const task = getTaskById(req.params.id);
if (!task) {
return res.status(404).json({ success: false, error: '未找到该任务' });
}
isRunning = true;
runTaskInBackground(task);
res.json({ success: true, message: `任务“${task.city}”已开始执行` });
} catch (error) {
isRunning = false;
res.status(500).json({ success: false, error: error.message });
}
});
app.post('/api/tasks/run', (req, res) => {
if (isRunning) {
return res.status(409).json({ success: false, error: '当前已有任务在运行,请稍后再试' });
}
try {
let tasks = listTasks();
if (Array.isArray(req.body?.ids) && req.body.ids.length > 0) {
const idSet = new Set(req.body.ids);
tasks = tasks.filter((task) => idSet.has(task.id));
} else {
tasks = tasks.filter((task) => task.enabled);
}
if (tasks.length === 0) {
return res.json({ success: true, data: [], message: '没有可运行的任务' });
}
isRunning = true;
runTasksInBackground(tasks);
res.json({ success: true, message: `${tasks.length} 个任务已开始执行` });
} catch (error) {
isRunning = false;
res.status(500).json({ success: false, error: error.message });
}
});
app.get('/api/config', (_req, res) => {
try {
res.json({ success: true, data: maskConfigSecrets(loadConfig()) });
} catch (error) {
res.status(500).json({ success: false, error: error.message });
}
});
app.post('/api/config', (req, res) => {
try {
const newCfg = req.body;
const oldCfg = readConfig();
if (newCfg.email?.smtpPass === '***已配置***') {
newCfg.email.smtpPass = oldCfg.email?.smtpPass || '';
}
saveConfig(newCfg);
saveConfig(mergeConfigWithExistingSecrets(req.body));
reloadScheduler();
res.json({ success: true, message: '配置已保存' });
} catch (e) {
res.status(500).json({ success: false, error: e.message });
} catch (error) {
res.status(500).json({ success: false, error: error.message });
}
});
// ========== 邮件 ==========
app.post('/api/send-email', async (req, res) => {
try {
const { emailConfig, report } = req.body;
if (!emailConfig?.smtpHost || !emailConfig?.smtpUser || !emailConfig?.smtpPass)
if (!emailConfig?.smtpHost || !emailConfig?.smtpUser || !emailConfig?.smtpPass) {
return res.status(400).json({ success: false, error: '邮件配置不完整' });
if (!emailConfig.recipients?.trim())
}
if (!emailConfig.recipients?.trim()) {
return res.status(400).json({ success: false, error: '请指定收件人' });
if (!report)
}
if (!report) {
return res.status(400).json({ success: false, error: '没有报告数据' });
}
const { sendReportEmail } = await import('./emailService.js');
const result = await sendReportEmail(emailConfig, report);
res.json({ success: true, message: '邮件发送成功', messageId: result.messageId });
} catch (e) {
res.status(500).json({ success: false, error: e.message });
} catch (error) {
res.status(500).json({ success: false, error: error.message });
}
});
// ========== 定时任务 ==========
app.get('/api/scheduler/status', (req, res) => {
app.get('/api/scheduler/status', (_req, res) => {
try {
res.json({ success: true, data: getSchedulerStatus() });
} catch (e) {
res.status(500).json({ success: false, error: e.message });
} catch (error) {
res.status(500).json({ success: false, error: error.message });
}
});
app.post('/api/run-scheduled-task', (req, res) => {
app.post('/api/run-scheduled-task', (_req, res) => {
try {
runTaskNow().catch(err => console.error('定时任务执行失败:', err));
runTaskNow().catch((error) => console.error('定时任务执行失败:', error));
res.json({ success: true, message: '定时任务已在后台触发' });
} catch (e) {
res.status(500).json({ success: false, error: e.message });
} catch (error) {
res.status(500).json({ success: false, error: error.message });
}
});
initResultsStore();
app.listen(PORT, () => {
console.log(`Server running at http://localhost:${PORT}`);
initScheduler();