Files
tool-node/src/server.js
zhaojunlong f2c856ab05 ```
feat(scheduler): 更新定时任务配置以支持中标与招标分别设置阈值

将原先单一的 threshold 配置项拆分为 winningThreshold 和 bidThreshold,
分别用于控制中标公示和招标公告的金额筛选条件。同时调整了默认值及描述信息,
使配置更清晰灵活。

此外,更新了定时任务状态展示逻辑,支持显示两个独立的阈值及其单位转换(万元/亿元)。
当阈值为 0 时显示“不筛选”,提高用户理解度。

配置文件 config.json 中相关字段已同步修改,并调整了时间范围字段 timeRange 的默认值。
```
2025-12-15 21:06:10 +08:00

838 lines
25 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import 'dotenv/config';
import express from 'express';
import cors from 'cors';
import axios from 'axios';
import * as cheerio from 'cheerio';
import iconv from 'iconv-lite';
import { sendReportEmail, sendBidAnnounceReportEmail, sendCombinedReportEmail } from './emailService.js';
import { initScheduler, runTaskNow, reloadScheduler, getSchedulerStatus } from './scheduler.js';
import { log } from 'console';
const app = express();
const PORT = process.env.PORT || 5000;
app.use(cors());
app.use(express.json());
app.use(express.static('public'));
// 南京市公共资源交易平台 - 交通水务中标结果公示
const BASE_URL = 'https://njggzy.nanjing.gov.cn/njweb/jtsw/069008/';
// 南京市公共资源交易平台 - 交通水务招标公告
const BID_ANNOUNCE_BASE_URL = 'https://njggzy.nanjing.gov.cn/njweb/jtsw/069001/';
// 获取分页URL (实际数据在moreinfosl3.html中分页为2.html, 3.html...)
function getPageUrl(pageIndex) {
if (pageIndex === 1) {
return `${BASE_URL}moreinfosl3.html`;
}
return `${BASE_URL}${pageIndex}.html`;
}
// 获取招标公告分页URL
// 数据通过AJAX加载第1页是 moreinfo5dc.html第2页起是 2.html, 3.html...
function getBidAnnouncePageUrl(pageIndex) {
if (pageIndex === 1) {
return `${BID_ANNOUNCE_BASE_URL}moreinfo5dc.html`;
}
return `${BID_ANNOUNCE_BASE_URL}${pageIndex}.html`;
}
// 检查日期是否在范围内
function isDateInRange(dateStr, startDate, endDate) {
if (!dateStr) return false;
const date = new Date(dateStr);
if (isNaN(date.getTime())) return false;
if (startDate && date < new Date(startDate)) return false;
if (endDate && date > new Date(endDate)) return false;
return true;
}
// 按时间范围采集多页列表
async function fetchListByDateRange(startDate, endDate, maxPages = 50) {
const allItems = [];
let shouldContinue = true;
let pageIndex = 1;
console.log(`开始按时间范围采集: ${startDate || '不限'}${endDate || '不限'}`);
while (shouldContinue && pageIndex <= maxPages) {
const pageUrl = getPageUrl(pageIndex);
console.log(`正在采集第 ${pageIndex} 页: ${pageUrl}`);
try {
const html = await fetchHtml(pageUrl);
const items = parseList(html);
if (items.length === 0) {
console.log(`${pageIndex} 页没有数据,停止采集`);
break;
}
let hasItemsInRange = false;
let allItemsBeforeRange = true;
for (const item of items) {
if (isDateInRange(item.date, startDate, endDate)) {
allItems.push(item);
hasItemsInRange = true;
allItemsBeforeRange = false;
} else if (startDate && new Date(item.date) < new Date(startDate)) {
allItemsBeforeRange = allItemsBeforeRange && true;
} else {
allItemsBeforeRange = false;
}
}
if (allItemsBeforeRange && startDate) {
console.log(`${pageIndex} 页所有项目都早于起始日期,停止采集`);
shouldContinue = false;
}
console.log(`${pageIndex} 页找到 ${items.length} 条,符合条件 ${hasItemsInRange ? '有' : '无'}`);
pageIndex++;
if (shouldContinue && pageIndex <= maxPages) {
await new Promise(resolve => setTimeout(resolve, 500));
}
} catch (err) {
console.error(`采集第 ${pageIndex} 页失败: ${err.message}`);
break;
}
}
console.log(`总共采集了 ${pageIndex - 1} 页,找到 ${allItems.length} 条符合条件的公告`);
return allItems;
}
const http = axios.create({
responseType: 'arraybuffer',
timeout: 15000,
headers: {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
},
});
function pickEncoding(contentType = '') {
const match = /charset=([^;]+)/i.exec(contentType);
if (!match) return 'utf-8';
const charset = match[1].trim().toLowerCase();
if (charset.includes('gb')) return 'gbk';
return charset;
}
async function fetchHtml(url) {
const res = await http.get(url);
const encoding = pickEncoding(res.headers['content-type']);
const html = iconv.decode(res.data, encoding || 'utf-8');
return html;
}
// 解析列表页HTML提取中标结果信息
function parseList(html) {
const $ = cheerio.load(html);
const items = [];
// 解析南京公共资源交易平台的列表结构(交通水务中标结果公示)
// <li class="ewb-info-item2 clearfix" onclick="window.open('详情URL');">
$('li.ewb-info-item2').each((_, row) => {
const $row = $(row);
const cells = $row.find('div.ewb-info-num2');
if (cells.length >= 5) {
// 获取各字段
const bidNo = $(cells[0]).find('p').attr('title') || $(cells[0]).find('p').text().trim();
const projectName = $(cells[1]).find('p').attr('title') || $(cells[1]).find('p').text().trim();
const bidName = $(cells[2]).find('p').attr('title') || $(cells[2]).find('p').text().trim();
const winningPrice = $(cells[3]).find('p').text().trim(); // 中标价格
const winningDate = $(cells[4]).find('p').text().trim(); // 中标日期
// 从onclick提取详情链接
const onclick = $row.attr('onclick') || '';
const hrefMatch = onclick.match(/window\.open\(['"]([^'"]+)['"]\)/);
let href = '';
if (hrefMatch) {
href = hrefMatch[1];
// 转换为绝对URL
if (href.startsWith('/')) {
href = `https://njggzy.nanjing.gov.cn${href}`;
}
}
// 验证日期格式 (YYYY-MM-DD)
if (!/^\d{4}-\d{2}-\d{2}$/.test(winningDate)) return;
// 解析中标价格
const price = parseFloat(winningPrice);
if (isNaN(price)) return;
items.push({
bidNo, // 标段编号
title: projectName, // 项目名称
bidName, // 标段名称
winningBid: { // 中标金额
amount: price,
unit: '万元'
},
date: winningDate, // 中标日期
href
});
}
});
return items;
}
// 解析招标公告列表页HTML
function parseBidAnnounceList(html) {
const $ = cheerio.load(html);
const items = [];
// 解析南京公共资源交易平台的招标公告列表结构
// <li class="ewb-info-item2 clearfix" onclick="window.open('...');">
$('li.ewb-info-item2').each((_, row) => {
const $row = $(row);
const onclick = $row.attr('onclick') || '';
// 提取详情链接
const hrefMatch = onclick.match(/window\.open\(['"]([^'"]+)['"]\)/);
if (!hrefMatch) return;
let href = hrefMatch[1];
if (href.startsWith('/')) {
href = `https://njggzy.nanjing.gov.cn${href}`;
}
// 获取标题从p标签的title属性或文本
const $titleP = $row.find('.ewb-info-num2').first().find('p');
const title = $titleP.attr('title') || $titleP.text().trim();
// 获取日期
const $dateP = $row.find('.ewb-info-num2').last().find('p');
const dateText = $dateP.text().trim();
const dateMatch = dateText.match(/\d{4}-\d{2}-\d{2}/);
const date = dateMatch ? dateMatch[0] : '';
if (title && date) {
items.push({
title,
date,
href,
estimatedAmount: null
});
}
});
return items;
}
// 解析招标公告详情页,获取合同估算价
async function fetchBidAnnounceDetail(url) {
try {
const html = await fetchHtml(url);
const $ = cheerio.load(html);
// 获取页面全部文本
const bodyText = $('body').text();
// 查找合同估算价 (格式如: 合同估算价4,300,000.00 元)
const amountMatch = bodyText.match(/合同估算价[:]\s*([\d,]+\.?\d*)\s*元/);
let estimatedAmount = null;
if (amountMatch) {
// 去掉逗号,转换为数字(单位:元)
const amountStr = amountMatch[1].replace(/,/g, '');
estimatedAmount = parseFloat(amountStr);
}
// 获取标段编码
const bidCodeMatch = bodyText.match(/标段编码[:]\s*([A-Za-z0-9\-]+)/);
const bidCode = bidCodeMatch ? bidCodeMatch[1] : null;
// 获取招标人
const tendereeMatch = bodyText.match(/招标人[为是][:]?\s*([^\s,,。]+)/);
const tenderee = tendereeMatch ? tendereeMatch[1] : null;
// 获取计划工期
const durationMatch = bodyText.match(/计划工期[:]\s*(\d+)\s*日历天/);
const duration = durationMatch ? parseInt(durationMatch[1]) : null;
return {
estimatedAmount,
bidCode,
tenderee,
duration,
url
};
} catch (error) {
console.error(`获取招标详情失败 ${url}: ${error.message}`);
return { estimatedAmount: null, url };
}
}
// 按时间范围采集招标公告
async function fetchBidAnnounceByDateRange(startDate, endDate, maxPages = 20, fetchDetails = true) {
const allItems = [];
let shouldContinue = true;
let pageIndex = 1;
console.log(`开始采集招标公告: ${startDate || '不限'}${endDate || '不限'}`);
while (shouldContinue && pageIndex <= maxPages) {
const pageUrl = getBidAnnouncePageUrl(pageIndex);
console.log(`正在采集招标公告第 ${pageIndex} 页: ${pageUrl}`);
try {
const html = await fetchHtml(pageUrl);
const items = parseBidAnnounceList(html);
if (items.length === 0) {
console.log(`${pageIndex} 页没有数据,停止采集`);
break;
}
let hasItemsInRange = false;
let allItemsBeforeRange = true;
for (const item of items) {
if (isDateInRange(item.date, startDate, endDate)) {
allItems.push(item);
hasItemsInRange = true;
allItemsBeforeRange = false;
} else if (startDate && new Date(item.date) < new Date(startDate)) {
allItemsBeforeRange = allItemsBeforeRange && true;
} else {
allItemsBeforeRange = false;
}
}
if (allItemsBeforeRange && startDate) {
console.log(`${pageIndex} 页所有项目都早于起始日期,停止采集`);
shouldContinue = false;
}
console.log(`${pageIndex} 页找到 ${items.length} 条,符合条件 ${hasItemsInRange ? '有' : '无'}`);
pageIndex++;
if (shouldContinue && pageIndex <= maxPages) {
await new Promise(resolve => setTimeout(resolve, 500));
}
} catch (err) {
console.error(`采集第 ${pageIndex} 页失败: ${err.message}`);
break;
}
}
console.log(`总共采集了 ${pageIndex - 1} 页,找到 ${allItems.length} 条符合条件的招标公告`);
// 如果需要获取详情(合同估算价)
if (fetchDetails && allItems.length > 0) {
console.log(`开始获取 ${allItems.length} 条招标公告的详情...`);
for (let i = 0; i < allItems.length; i++) {
const item = allItems[i];
console.log(`获取详情 ${i + 1}/${allItems.length}: ${item.title.substring(0, 30)}...`);
const detail = await fetchBidAnnounceDetail(item.href);
item.estimatedAmount = detail.estimatedAmount;
item.bidCode = detail.bidCode;
item.tenderee = detail.tenderee;
item.duration = detail.duration;
// 添加延迟避免请求过快
if (i < allItems.length - 1) {
await new Promise(resolve => setTimeout(resolve, 300));
}
}
console.log('详情获取完成');
}
return allItems;
}
// API 路由
// 获取列表
app.get('/api/list', async (req, res) => {
try {
const page = parseInt(req.query.page) || 1;
const pageUrl = getPageUrl(page);
const html = await fetchHtml(pageUrl);
const items = parseList(html);
res.json({ success: true, data: items, page });
} catch (error) {
res.status(500).json({ success: false, error: error.message });
}
});
// 按时间范围获取列表
app.post('/api/list-daterange', async (req, res) => {
try {
const { startDate, endDate, maxPages = 50 } = req.body;
const items = await fetchListByDateRange(startDate, endDate, maxPages);
res.json({ success: true, data: items });
} catch (error) {
res.status(500).json({ success: false, error: error.message });
}
});
// 生成报告
app.post('/api/report', async (req, res) => {
try {
const { limit = 50, threshold = 50 } = req.body;
// 采集列表
const items = [];
let pageIndex = 1;
const maxPagesToFetch = Math.ceil(limit / 10) + 1;
while (items.length < limit && pageIndex <= maxPagesToFetch) {
const pageUrl = getPageUrl(pageIndex);
console.log(`正在采集第 ${pageIndex} 页: ${pageUrl}`);
try {
const html = await fetchHtml(pageUrl);
const pageItems = parseList(html);
if (pageItems.length === 0) {
console.log(`${pageIndex} 页没有数据,停止采集`);
break;
}
items.push(...pageItems);
pageIndex++;
if (items.length < limit && pageIndex <= maxPagesToFetch) {
await new Promise(resolve => setTimeout(resolve, 500));
}
} catch (err) {
console.error(`采集第 ${pageIndex} 页失败: ${err.message}`);
break;
}
}
const results = items.slice(0, limit);
// 按阈值筛选
const filtered = results.filter((item) => {
return item.winningBid && item.winningBid.amount > threshold;
});
const total = filtered.reduce(
(sum, item) => sum + (item.winningBid?.amount || 0),
0
);
const report = {
summary: {
total_count: results.length,
filtered_count: filtered.length,
threshold: `${threshold}万元`,
total_amount: `${total.toFixed(2)}万元`,
generated_at: new Date().toISOString(),
},
projects: filtered.map((item) => ({
bidNo: item.bidNo,
title: item.title,
bidName: item.bidName,
date: item.date,
winningBid: item.winningBid,
url: item.href,
})),
};
res.json({ success: true, data: report });
} catch (error) {
res.status(500).json({ success: false, error: error.message });
}
});
// 按时间范围生成报告
app.post('/api/report-daterange', async (req, res) => {
try {
const { startDate, endDate, threshold = 50, maxPages = 50 } = req.body;
// 按时间范围采集列表
const items = await fetchListByDateRange(startDate, endDate, maxPages);
if (items.length === 0) {
return res.json({
success: true,
data: {
summary: {
total_count: 0,
filtered_count: 0,
threshold: `${threshold}万元`,
total_amount: '0.00万元',
generated_at: new Date().toISOString(),
date_range: { startDate, endDate },
},
projects: [],
},
});
}
// 按阈值筛选
const filtered = items.filter((item) => {
return item.winningBid && item.winningBid.amount > threshold;
});
const total = filtered.reduce(
(sum, item) => sum + (item.winningBid?.amount || 0),
0
);
const report = {
summary: {
total_count: items.length,
filtered_count: filtered.length,
threshold: `${threshold}万元`,
total_amount: `${total.toFixed(2)}万元`,
generated_at: new Date().toISOString(),
date_range: { startDate, endDate },
},
projects: filtered.map((item) => ({
bidNo: item.bidNo,
title: item.title,
bidName: item.bidName,
date: item.date,
winningBid: item.winningBid,
url: item.href,
})),
};
res.json({ success: true, data: report });
} catch (error) {
res.status(500).json({ success: false, error: error.message });
}
});
// ========== 招标公告相关API ==========
// 获取招标公告列表(简单列表,按页码)
app.get('/api/bid-announce/list', async (req, res) => {
try {
const page = parseInt(req.query.page) || 1;
const pageUrl = getBidAnnouncePageUrl(page);
const html = await fetchHtml(pageUrl);
const items = parseBidAnnounceList(html);
res.json({ success: true, data: items, page });
} catch (error) {
res.status(500).json({ success: false, error: error.message });
}
});
// 获取招标公告列表(按时间范围)
app.post('/api/bid-announce/list', async (req, res) => {
try {
const { startDate, endDate, maxPages = 20, fetchDetails = false } = req.body;
const items = await fetchBidAnnounceByDateRange(startDate, endDate, maxPages, fetchDetails);
res.json({ success: true, data: items });
} catch (error) {
res.status(500).json({ success: false, error: error.message });
}
});
// 生成招标公告报告(含金额统计)
app.post('/api/bid-announce/report', async (req, res) => {
try {
const { startDate, endDate, threshold = 0, maxPages = 20 } = req.body;
// 采集招标公告(包含详情)
const items = await fetchBidAnnounceByDateRange(startDate, endDate, maxPages, true);
if (items.length === 0) {
return res.json({
success: true,
data: {
summary: {
total_count: 0,
filtered_count: 0,
threshold: threshold > 0 ? `${threshold}` : '无',
total_amount: '0.00元',
generated_at: new Date().toISOString(),
date_range: { startDate, endDate },
report_type: '招标公告'
},
projects: [],
},
});
}
// 按阈值筛选(阈值单位为元)
const filtered = threshold > 0
? items.filter(item => item.estimatedAmount && item.estimatedAmount >= threshold)
: items.filter(item => item.estimatedAmount);
// 计算总金额
const total = filtered.reduce((sum, item) => sum + (item.estimatedAmount || 0), 0);
const report = {
summary: {
total_count: items.length,
filtered_count: filtered.length,
has_amount_count: items.filter(i => i.estimatedAmount).length,
threshold: threshold > 0 ? `${(threshold / 10000).toFixed(2)}万元` : '无',
total_amount: `${(total / 10000).toFixed(2)}万元`,
total_amount_yuan: total,
generated_at: new Date().toISOString(),
date_range: { startDate, endDate },
report_type: '招标公告'
},
projects: filtered.map((item) => ({
title: item.title,
bidCode: item.bidCode,
tenderee: item.tenderee,
date: item.date,
duration: item.duration,
estimatedAmount: item.estimatedAmount ? {
amount: item.estimatedAmount,
amountWan: (item.estimatedAmount / 10000).toFixed(2),
unit: '元'
} : null,
url: item.href,
})),
};
res.json({ success: true, data: report });
} catch (error) {
res.status(500).json({ success: false, error: error.message });
}
});
// 发送招标公告报告邮件
app.post('/api/bid-announce/send-email', async (req, res) => {
try {
const { emailConfig, report } = req.body;
if (!emailConfig || !emailConfig.smtpHost || !emailConfig.smtpUser || !emailConfig.smtpPass) {
return res.status(400).json({
success: false,
error: '邮件配置不完整,请填写SMTP服务器、用户名和密码',
});
}
if (!emailConfig.recipients || emailConfig.recipients.trim() === '') {
return res.status(400).json({
success: false,
error: '请至少指定一个收件人',
});
}
if (!report) {
return res.status(400).json({
success: false,
error: '没有可发送的报告数据',
});
}
// 使用招标公告专用的邮件发送
const result = await sendBidAnnounceReportEmail(emailConfig, report);
res.json({
success: true,
message: '招标公告报告邮件发送成功',
messageId: result.messageId,
});
} catch (error) {
console.error('发送招标公告邮件API错误:', error);
res.status(500).json({
success: false,
error: error.message,
});
}
});
// 发送综合报告邮件(中标+招标)
app.post('/api/send-combined-email', async (req, res) => {
try {
const { emailConfig, winningReport, bidReport } = req.body;
if (!emailConfig || !emailConfig.smtpHost || !emailConfig.smtpUser || !emailConfig.smtpPass) {
return res.status(400).json({
success: false,
error: '邮件配置不完整,请填写SMTP服务器、用户名和密码',
});
}
if (!emailConfig.recipients || emailConfig.recipients.trim() === '') {
return res.status(400).json({
success: false,
error: '请至少指定一个收件人',
});
}
if (!winningReport && !bidReport) {
return res.status(400).json({
success: false,
error: '没有可发送的报告数据',
});
}
// 发送综合邮件
const result = await sendCombinedReportEmail(emailConfig, winningReport, bidReport);
res.json({
success: true,
message: '综合报告邮件发送成功',
messageId: result.messageId,
});
} catch (error) {
console.error('发送综合邮件API错误:', error);
res.status(500).json({
success: false,
error: error.message,
});
}
});
// 发送报告邮件
app.post('/api/send-email', async (req, res) => {
try {
const { emailConfig, report } = req.body;
// 验证必需的配置参数
if (!emailConfig || !emailConfig.smtpHost || !emailConfig.smtpUser || !emailConfig.smtpPass) {
return res.status(400).json({
success: false,
error: '邮件配置不完整,请填写SMTP服务器、用户名和密码',
});
}
if (!emailConfig.recipients || emailConfig.recipients.trim() === '') {
return res.status(400).json({
success: false,
error: '请至少指定一个收件人',
});
}
if (!report) {
return res.status(400).json({
success: false,
error: '没有可发送的报告数据',
});
}
// 发送邮件
const result = await sendReportEmail(emailConfig, report);
res.json({
success: true,
message: '邮件发送成功',
messageId: result.messageId,
});
} catch (error) {
console.error('发送邮件API错误:', error);
res.status(500).json({
success: false,
error: error.message,
});
}
});
// 获取配置
app.get('/api/config', async (req, res) => {
try {
const { readFileSync } = await import('fs');
const { join } = await import('path');
const { fileURLToPath } = await import('url');
const { dirname } = await import('path');
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const configPath = join(__dirname, '..', 'config.json');
const configContent = readFileSync(configPath, 'utf-8');
const config = JSON.parse(configContent);
// 不返回敏感信息(密码)
if (config.email && config.email.smtpPass) {
config.email.smtpPass = '***已配置***';
}
res.json({ success: true, data: config });
} catch (error) {
res.status(500).json({ success: false, error: error.message });
}
});
// 更新配置
app.post('/api/config', async (req, res) => {
try {
const { writeFileSync, readFileSync } = await import('fs');
const { join } = await import('path');
const { fileURLToPath } = await import('url');
const { dirname } = await import('path');
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const configPath = join(__dirname, '..', 'config.json');
const newConfig = req.body;
// 读取旧配置以保留敏感信息
const oldConfigContent = readFileSync(configPath, 'utf-8');
const oldConfig = JSON.parse(oldConfigContent);
// 如果密码字段是占位符,保留原密码
if (newConfig.email && newConfig.email.smtpPass === '***已配置***') {
newConfig.email.smtpPass = oldConfig.email?.smtpPass || '';
}
// 保存配置
writeFileSync(configPath, JSON.stringify(newConfig, null, 2), 'utf-8');
// 重新加载定时任务(如果定时任务配置有变化)
reloadScheduler();
res.json({ success: true, message: '配置已保存并重新加载定时任务' });
} catch (error) {
res.status(500).json({ success: false, error: error.message });
}
});
// 获取定时任务状态
app.get('/api/scheduler/status', async (req, res) => {
try {
const status = getSchedulerStatus();
res.json({ success: true, data: status });
} catch (error) {
res.status(500).json({ success: false, error: error.message });
}
});
// 手动触发定时任务的API(用于测试)
app.post('/api/run-scheduled-task', async (req, res) => {
try {
console.log('手动触发定时任务...');
// 在后台执行任务,不阻塞响应
runTaskNow().catch(err => {
console.error('定时任务执行失败:', err);
});
res.json({
success: true,
message: '定时任务已触发,正在后台执行...'
});
} catch (error) {
res.status(500).json({
success: false,
error: error.message
});
}
});
app.listen(PORT, () => {
console.log(`Server running at http://localhost:${PORT}`);
// 启动定时任务
console.log('正在初始化定时任务...');
initScheduler();
});