import 'dotenv/config';
import express from 'express';
import cors from 'cors';
import axios from 'axios';
import * as cheerio from 'cheerio';
import iconv from 'iconv-lite';
import { sendReportEmail, sendBidAnnounceReportEmail, sendCombinedReportEmail } from './emailService.js';
import { initScheduler, runTaskNow, reloadScheduler, getSchedulerStatus } from './scheduler.js';
import { log } from 'console';
const app = express();
const PORT = process.env.PORT || 5000;
app.use(cors());
app.use(express.json());
app.use(express.static('public'));
// 南京市公共资源交易平台 - 交通水务中标结果公示
const BASE_URL = 'https://njggzy.nanjing.gov.cn/njweb/jtsw/069008/';
// 南京市公共资源交易平台 - 交通水务招标公告
const BID_ANNOUNCE_BASE_URL = 'https://njggzy.nanjing.gov.cn/njweb/jtsw/069001/';
// 获取分页URL (实际数据在moreinfosl3.html中,分页为2.html, 3.html...)
function getPageUrl(pageIndex) {
if (pageIndex === 1) {
return `${BASE_URL}moreinfosl3.html`;
}
return `${BASE_URL}${pageIndex}.html`;
}
// 获取招标公告分页URL
// 数据通过AJAX加载,第1页是 moreinfo5dc.html,第2页起是 2.html, 3.html...
function getBidAnnouncePageUrl(pageIndex) {
if (pageIndex === 1) {
return `${BID_ANNOUNCE_BASE_URL}moreinfo5dc.html`;
}
return `${BID_ANNOUNCE_BASE_URL}${pageIndex}.html`;
}
// 检查日期是否在范围内
function isDateInRange(dateStr, startDate, endDate) {
if (!dateStr) return false;
const date = new Date(dateStr);
if (isNaN(date.getTime())) return false;
if (startDate && date < new Date(startDate)) return false;
if (endDate && date > new Date(endDate)) return false;
return true;
}
// 按时间范围采集多页列表
async function fetchListByDateRange(startDate, endDate, maxPages = 50) {
const allItems = [];
let shouldContinue = true;
let pageIndex = 1;
console.log(`开始按时间范围采集: ${startDate || '不限'} 至 ${endDate || '不限'}`);
while (shouldContinue && pageIndex <= maxPages) {
const pageUrl = getPageUrl(pageIndex);
console.log(`正在采集第 ${pageIndex} 页: ${pageUrl}`);
try {
const html = await fetchHtml(pageUrl);
const items = parseList(html);
if (items.length === 0) {
console.log(`第 ${pageIndex} 页没有数据,停止采集`);
break;
}
let hasItemsInRange = false;
let allItemsBeforeRange = true;
for (const item of items) {
if (isDateInRange(item.date, startDate, endDate)) {
allItems.push(item);
hasItemsInRange = true;
allItemsBeforeRange = false;
} else if (startDate && new Date(item.date) < new Date(startDate)) {
allItemsBeforeRange = allItemsBeforeRange && true;
} else {
allItemsBeforeRange = false;
}
}
if (allItemsBeforeRange && startDate) {
console.log(`第 ${pageIndex} 页所有项目都早于起始日期,停止采集`);
shouldContinue = false;
}
console.log(`第 ${pageIndex} 页找到 ${items.length} 条,符合条件 ${hasItemsInRange ? '有' : '无'}`);
pageIndex++;
if (shouldContinue && pageIndex <= maxPages) {
await new Promise(resolve => setTimeout(resolve, 500));
}
} catch (err) {
console.error(`采集第 ${pageIndex} 页失败: ${err.message}`);
break;
}
}
console.log(`总共采集了 ${pageIndex - 1} 页,找到 ${allItems.length} 条符合条件的公告`);
return allItems;
}
const http = axios.create({
responseType: 'arraybuffer',
timeout: 15000,
headers: {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
},
});
function pickEncoding(contentType = '') {
const match = /charset=([^;]+)/i.exec(contentType);
if (!match) return 'utf-8';
const charset = match[1].trim().toLowerCase();
if (charset.includes('gb')) return 'gbk';
return charset;
}
async function fetchHtml(url) {
const res = await http.get(url);
const encoding = pickEncoding(res.headers['content-type']);
const html = iconv.decode(res.data, encoding || 'utf-8');
return html;
}
// 解析列表页HTML,提取中标结果信息
function parseList(html) {
const $ = cheerio.load(html);
const items = [];
// 解析南京公共资源交易平台的列表结构(交通水务中标结果公示)
//
$('li.ewb-info-item2').each((_, row) => {
const $row = $(row);
const cells = $row.find('div.ewb-info-num2');
if (cells.length >= 5) {
// 获取各字段
const bidNo = $(cells[0]).find('p').attr('title') || $(cells[0]).find('p').text().trim();
const projectName = $(cells[1]).find('p').attr('title') || $(cells[1]).find('p').text().trim();
const bidName = $(cells[2]).find('p').attr('title') || $(cells[2]).find('p').text().trim();
const winningPrice = $(cells[3]).find('p').text().trim(); // 中标价格
const winningDate = $(cells[4]).find('p').text().trim(); // 中标日期
// 从onclick提取详情链接
const onclick = $row.attr('onclick') || '';
const hrefMatch = onclick.match(/window\.open\(['"]([^'"]+)['"]\)/);
let href = '';
if (hrefMatch) {
href = hrefMatch[1];
// 转换为绝对URL
if (href.startsWith('/')) {
href = `https://njggzy.nanjing.gov.cn${href}`;
}
}
// 验证日期格式 (YYYY-MM-DD)
if (!/^\d{4}-\d{2}-\d{2}$/.test(winningDate)) return;
// 解析中标价格
const price = parseFloat(winningPrice);
if (isNaN(price)) return;
items.push({
bidNo, // 标段编号
title: projectName, // 项目名称
bidName, // 标段名称
winningBid: { // 中标金额
amount: price,
unit: '万元'
},
date: winningDate, // 中标日期
href
});
}
});
return items;
}
// 解析招标公告列表页HTML
function parseBidAnnounceList(html) {
const $ = cheerio.load(html);
const items = [];
// 解析南京公共资源交易平台的招标公告列表结构
//
$('li.ewb-info-item2').each((_, row) => {
const $row = $(row);
const onclick = $row.attr('onclick') || '';
// 提取详情链接
const hrefMatch = onclick.match(/window\.open\(['"]([^'"]+)['"]\)/);
if (!hrefMatch) return;
let href = hrefMatch[1];
if (href.startsWith('/')) {
href = `https://njggzy.nanjing.gov.cn${href}`;
}
// 获取标题(从p标签的title属性或文本)
const $titleP = $row.find('.ewb-info-num2').first().find('p');
const title = $titleP.attr('title') || $titleP.text().trim();
// 获取日期
const $dateP = $row.find('.ewb-info-num2').last().find('p');
const dateText = $dateP.text().trim();
const dateMatch = dateText.match(/\d{4}-\d{2}-\d{2}/);
const date = dateMatch ? dateMatch[0] : '';
if (title && date) {
items.push({
title,
date,
href,
estimatedAmount: null
});
}
});
return items;
}
// 解析招标公告详情页,获取合同估算价
async function fetchBidAnnounceDetail(url) {
try {
const html = await fetchHtml(url);
const $ = cheerio.load(html);
// 获取页面全部文本
const bodyText = $('body').text();
// 查找合同估算价 (格式如: 合同估算价:4,300,000.00 元)
const amountMatch = bodyText.match(/合同估算价[::]\s*([\d,]+\.?\d*)\s*元/);
let estimatedAmount = null;
if (amountMatch) {
// 去掉逗号,转换为数字(单位:元)
const amountStr = amountMatch[1].replace(/,/g, '');
estimatedAmount = parseFloat(amountStr);
}
// 获取标段编码
const bidCodeMatch = bodyText.match(/标段编码[::]\s*([A-Za-z0-9\-]+)/);
const bidCode = bidCodeMatch ? bidCodeMatch[1] : null;
// 获取招标人
const tendereeMatch = bodyText.match(/招标人[为是][::]?\s*([^\s,,。]+)/);
const tenderee = tendereeMatch ? tendereeMatch[1] : null;
// 获取计划工期
const durationMatch = bodyText.match(/计划工期[::]\s*(\d+)\s*日历天/);
const duration = durationMatch ? parseInt(durationMatch[1]) : null;
return {
estimatedAmount,
bidCode,
tenderee,
duration,
url
};
} catch (error) {
console.error(`获取招标详情失败 ${url}: ${error.message}`);
return { estimatedAmount: null, url };
}
}
// 按时间范围采集招标公告
async function fetchBidAnnounceByDateRange(startDate, endDate, maxPages = 20, fetchDetails = true) {
const allItems = [];
let shouldContinue = true;
let pageIndex = 1;
console.log(`开始采集招标公告: ${startDate || '不限'} 至 ${endDate || '不限'}`);
while (shouldContinue && pageIndex <= maxPages) {
const pageUrl = getBidAnnouncePageUrl(pageIndex);
console.log(`正在采集招标公告第 ${pageIndex} 页: ${pageUrl}`);
try {
const html = await fetchHtml(pageUrl);
const items = parseBidAnnounceList(html);
if (items.length === 0) {
console.log(`第 ${pageIndex} 页没有数据,停止采集`);
break;
}
let hasItemsInRange = false;
let allItemsBeforeRange = true;
for (const item of items) {
if (isDateInRange(item.date, startDate, endDate)) {
allItems.push(item);
hasItemsInRange = true;
allItemsBeforeRange = false;
} else if (startDate && new Date(item.date) < new Date(startDate)) {
allItemsBeforeRange = allItemsBeforeRange && true;
} else {
allItemsBeforeRange = false;
}
}
if (allItemsBeforeRange && startDate) {
console.log(`第 ${pageIndex} 页所有项目都早于起始日期,停止采集`);
shouldContinue = false;
}
console.log(`第 ${pageIndex} 页找到 ${items.length} 条,符合条件 ${hasItemsInRange ? '有' : '无'}`);
pageIndex++;
if (shouldContinue && pageIndex <= maxPages) {
await new Promise(resolve => setTimeout(resolve, 500));
}
} catch (err) {
console.error(`采集第 ${pageIndex} 页失败: ${err.message}`);
break;
}
}
console.log(`总共采集了 ${pageIndex - 1} 页,找到 ${allItems.length} 条符合条件的招标公告`);
// 如果需要获取详情(合同估算价)
if (fetchDetails && allItems.length > 0) {
console.log(`开始获取 ${allItems.length} 条招标公告的详情...`);
for (let i = 0; i < allItems.length; i++) {
const item = allItems[i];
console.log(`获取详情 ${i + 1}/${allItems.length}: ${item.title.substring(0, 30)}...`);
const detail = await fetchBidAnnounceDetail(item.href);
item.estimatedAmount = detail.estimatedAmount;
item.bidCode = detail.bidCode;
item.tenderee = detail.tenderee;
item.duration = detail.duration;
// 添加延迟避免请求过快
if (i < allItems.length - 1) {
await new Promise(resolve => setTimeout(resolve, 300));
}
}
console.log('详情获取完成');
}
return allItems;
}
// API 路由
// 获取列表
app.get('/api/list', async (req, res) => {
try {
const page = parseInt(req.query.page) || 1;
const pageUrl = getPageUrl(page);
const html = await fetchHtml(pageUrl);
const items = parseList(html);
res.json({ success: true, data: items, page });
} catch (error) {
res.status(500).json({ success: false, error: error.message });
}
});
// 按时间范围获取列表
app.post('/api/list-daterange', async (req, res) => {
try {
const { startDate, endDate, maxPages = 50 } = req.body;
const items = await fetchListByDateRange(startDate, endDate, maxPages);
res.json({ success: true, data: items });
} catch (error) {
res.status(500).json({ success: false, error: error.message });
}
});
// 生成报告
app.post('/api/report', async (req, res) => {
try {
const { limit = 50, threshold = 50 } = req.body;
// 采集列表
const items = [];
let pageIndex = 1;
const maxPagesToFetch = Math.ceil(limit / 10) + 1;
while (items.length < limit && pageIndex <= maxPagesToFetch) {
const pageUrl = getPageUrl(pageIndex);
console.log(`正在采集第 ${pageIndex} 页: ${pageUrl}`);
try {
const html = await fetchHtml(pageUrl);
const pageItems = parseList(html);
if (pageItems.length === 0) {
console.log(`第 ${pageIndex} 页没有数据,停止采集`);
break;
}
items.push(...pageItems);
pageIndex++;
if (items.length < limit && pageIndex <= maxPagesToFetch) {
await new Promise(resolve => setTimeout(resolve, 500));
}
} catch (err) {
console.error(`采集第 ${pageIndex} 页失败: ${err.message}`);
break;
}
}
const results = items.slice(0, limit);
// 按阈值筛选
const filtered = results.filter((item) => {
return item.winningBid && item.winningBid.amount > threshold;
});
const total = filtered.reduce(
(sum, item) => sum + (item.winningBid?.amount || 0),
0
);
const report = {
summary: {
total_count: results.length,
filtered_count: filtered.length,
threshold: `${threshold}万元`,
total_amount: `${total.toFixed(2)}万元`,
generated_at: new Date().toISOString(),
},
projects: filtered.map((item) => ({
bidNo: item.bidNo,
title: item.title,
bidName: item.bidName,
date: item.date,
winningBid: item.winningBid,
url: item.href,
})),
};
res.json({ success: true, data: report });
} catch (error) {
res.status(500).json({ success: false, error: error.message });
}
});
// 按时间范围生成报告
app.post('/api/report-daterange', async (req, res) => {
try {
const { startDate, endDate, threshold = 50, maxPages = 50 } = req.body;
// 按时间范围采集列表
const items = await fetchListByDateRange(startDate, endDate, maxPages);
if (items.length === 0) {
return res.json({
success: true,
data: {
summary: {
total_count: 0,
filtered_count: 0,
threshold: `${threshold}万元`,
total_amount: '0.00万元',
generated_at: new Date().toISOString(),
date_range: { startDate, endDate },
},
projects: [],
},
});
}
// 按阈值筛选
const filtered = items.filter((item) => {
return item.winningBid && item.winningBid.amount > threshold;
});
const total = filtered.reduce(
(sum, item) => sum + (item.winningBid?.amount || 0),
0
);
const report = {
summary: {
total_count: items.length,
filtered_count: filtered.length,
threshold: `${threshold}万元`,
total_amount: `${total.toFixed(2)}万元`,
generated_at: new Date().toISOString(),
date_range: { startDate, endDate },
},
projects: filtered.map((item) => ({
bidNo: item.bidNo,
title: item.title,
bidName: item.bidName,
date: item.date,
winningBid: item.winningBid,
url: item.href,
})),
};
res.json({ success: true, data: report });
} catch (error) {
res.status(500).json({ success: false, error: error.message });
}
});
// ========== 招标公告相关API ==========
// 获取招标公告列表(简单列表,按页码)
app.get('/api/bid-announce/list', async (req, res) => {
try {
const page = parseInt(req.query.page) || 1;
const pageUrl = getBidAnnouncePageUrl(page);
const html = await fetchHtml(pageUrl);
const items = parseBidAnnounceList(html);
res.json({ success: true, data: items, page });
} catch (error) {
res.status(500).json({ success: false, error: error.message });
}
});
// 获取招标公告列表(按时间范围)
app.post('/api/bid-announce/list', async (req, res) => {
try {
const { startDate, endDate, maxPages = 20, fetchDetails = false } = req.body;
const items = await fetchBidAnnounceByDateRange(startDate, endDate, maxPages, fetchDetails);
res.json({ success: true, data: items });
} catch (error) {
res.status(500).json({ success: false, error: error.message });
}
});
// 生成招标公告报告(含金额统计)
app.post('/api/bid-announce/report', async (req, res) => {
try {
const { startDate, endDate, threshold = 0, maxPages = 20 } = req.body;
// 采集招标公告(包含详情)
const items = await fetchBidAnnounceByDateRange(startDate, endDate, maxPages, true);
if (items.length === 0) {
return res.json({
success: true,
data: {
summary: {
total_count: 0,
filtered_count: 0,
threshold: threshold > 0 ? `${threshold}元` : '无',
total_amount: '0.00元',
generated_at: new Date().toISOString(),
date_range: { startDate, endDate },
report_type: '招标公告'
},
projects: [],
},
});
}
// 按阈值筛选(阈值单位为元)
const filtered = threshold > 0
? items.filter(item => item.estimatedAmount && item.estimatedAmount >= threshold)
: items.filter(item => item.estimatedAmount);
// 计算总金额
const total = filtered.reduce((sum, item) => sum + (item.estimatedAmount || 0), 0);
const report = {
summary: {
total_count: items.length,
filtered_count: filtered.length,
has_amount_count: items.filter(i => i.estimatedAmount).length,
threshold: threshold > 0 ? `${(threshold / 10000).toFixed(2)}万元` : '无',
total_amount: `${(total / 10000).toFixed(2)}万元`,
total_amount_yuan: total,
generated_at: new Date().toISOString(),
date_range: { startDate, endDate },
report_type: '招标公告'
},
projects: filtered.map((item) => ({
title: item.title,
bidCode: item.bidCode,
tenderee: item.tenderee,
date: item.date,
duration: item.duration,
estimatedAmount: item.estimatedAmount ? {
amount: item.estimatedAmount,
amountWan: (item.estimatedAmount / 10000).toFixed(2),
unit: '元'
} : null,
url: item.href,
})),
};
res.json({ success: true, data: report });
} catch (error) {
res.status(500).json({ success: false, error: error.message });
}
});
// 发送招标公告报告邮件
app.post('/api/bid-announce/send-email', async (req, res) => {
try {
const { emailConfig, report } = req.body;
if (!emailConfig || !emailConfig.smtpHost || !emailConfig.smtpUser || !emailConfig.smtpPass) {
return res.status(400).json({
success: false,
error: '邮件配置不完整,请填写SMTP服务器、用户名和密码',
});
}
if (!emailConfig.recipients || emailConfig.recipients.trim() === '') {
return res.status(400).json({
success: false,
error: '请至少指定一个收件人',
});
}
if (!report) {
return res.status(400).json({
success: false,
error: '没有可发送的报告数据',
});
}
// 使用招标公告专用的邮件发送
const result = await sendBidAnnounceReportEmail(emailConfig, report);
res.json({
success: true,
message: '招标公告报告邮件发送成功',
messageId: result.messageId,
});
} catch (error) {
console.error('发送招标公告邮件API错误:', error);
res.status(500).json({
success: false,
error: error.message,
});
}
});
// 发送综合报告邮件(中标+招标)
app.post('/api/send-combined-email', async (req, res) => {
try {
const { emailConfig, winningReport, bidReport } = req.body;
if (!emailConfig || !emailConfig.smtpHost || !emailConfig.smtpUser || !emailConfig.smtpPass) {
return res.status(400).json({
success: false,
error: '邮件配置不完整,请填写SMTP服务器、用户名和密码',
});
}
if (!emailConfig.recipients || emailConfig.recipients.trim() === '') {
return res.status(400).json({
success: false,
error: '请至少指定一个收件人',
});
}
if (!winningReport && !bidReport) {
return res.status(400).json({
success: false,
error: '没有可发送的报告数据',
});
}
// 发送综合邮件
const result = await sendCombinedReportEmail(emailConfig, winningReport, bidReport);
res.json({
success: true,
message: '综合报告邮件发送成功',
messageId: result.messageId,
});
} catch (error) {
console.error('发送综合邮件API错误:', error);
res.status(500).json({
success: false,
error: error.message,
});
}
});
// 发送报告邮件
app.post('/api/send-email', async (req, res) => {
try {
const { emailConfig, report } = req.body;
// 验证必需的配置参数
if (!emailConfig || !emailConfig.smtpHost || !emailConfig.smtpUser || !emailConfig.smtpPass) {
return res.status(400).json({
success: false,
error: '邮件配置不完整,请填写SMTP服务器、用户名和密码',
});
}
if (!emailConfig.recipients || emailConfig.recipients.trim() === '') {
return res.status(400).json({
success: false,
error: '请至少指定一个收件人',
});
}
if (!report) {
return res.status(400).json({
success: false,
error: '没有可发送的报告数据',
});
}
// 发送邮件
const result = await sendReportEmail(emailConfig, report);
res.json({
success: true,
message: '邮件发送成功',
messageId: result.messageId,
});
} catch (error) {
console.error('发送邮件API错误:', error);
res.status(500).json({
success: false,
error: error.message,
});
}
});
// 获取配置
app.get('/api/config', async (req, res) => {
try {
const { readFileSync } = await import('fs');
const { join } = await import('path');
const { fileURLToPath } = await import('url');
const { dirname } = await import('path');
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const configPath = join(__dirname, '..', 'config.json');
const configContent = readFileSync(configPath, 'utf-8');
const config = JSON.parse(configContent);
// 不返回敏感信息(密码)
if (config.email && config.email.smtpPass) {
config.email.smtpPass = '***已配置***';
}
res.json({ success: true, data: config });
} catch (error) {
res.status(500).json({ success: false, error: error.message });
}
});
// 更新配置
app.post('/api/config', async (req, res) => {
try {
const { writeFileSync, readFileSync } = await import('fs');
const { join } = await import('path');
const { fileURLToPath } = await import('url');
const { dirname } = await import('path');
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const configPath = join(__dirname, '..', 'config.json');
const newConfig = req.body;
// 读取旧配置以保留敏感信息
const oldConfigContent = readFileSync(configPath, 'utf-8');
const oldConfig = JSON.parse(oldConfigContent);
// 如果密码字段是占位符,保留原密码
if (newConfig.email && newConfig.email.smtpPass === '***已配置***') {
newConfig.email.smtpPass = oldConfig.email?.smtpPass || '';
}
// 保存配置
writeFileSync(configPath, JSON.stringify(newConfig, null, 2), 'utf-8');
// 重新加载定时任务(如果定时任务配置有变化)
reloadScheduler();
res.json({ success: true, message: '配置已保存并重新加载定时任务' });
} catch (error) {
res.status(500).json({ success: false, error: error.message });
}
});
// 获取定时任务状态
app.get('/api/scheduler/status', async (req, res) => {
try {
const status = getSchedulerStatus();
res.json({ success: true, data: status });
} catch (error) {
res.status(500).json({ success: false, error: error.message });
}
});
// 手动触发定时任务的API(用于测试)
app.post('/api/run-scheduled-task', async (req, res) => {
try {
console.log('手动触发定时任务...');
// 在后台执行任务,不阻塞响应
runTaskNow().catch(err => {
console.error('定时任务执行失败:', err);
});
res.json({
success: true,
message: '定时任务已触发,正在后台执行...'
});
} catch (error) {
res.status(500).json({
success: false,
error: error.message
});
}
});
app.listen(PORT, () => {
console.log(`Server running at http://localhost:${PORT}`);
// 启动定时任务
console.log('正在初始化定时任务...');
initScheduler();
});