import 'dotenv/config'; import express from 'express'; import cors from 'cors'; import axios from 'axios'; import * as cheerio from 'cheerio'; import iconv from 'iconv-lite'; import { sendReportEmail, sendBidAnnounceReportEmail, sendCombinedReportEmail } from './emailService.js'; import { initScheduler, runTaskNow, reloadScheduler, getSchedulerStatus } from './scheduler.js'; import { log } from 'console'; const app = express(); const PORT = process.env.PORT || 5000; app.use(cors()); app.use(express.json()); app.use(express.static('public')); // 南京市公共资源交易平台 - 交通水务中标结果公示 const BASE_URL = 'https://njggzy.nanjing.gov.cn/njweb/jtsw/069008/'; // 南京市公共资源交易平台 - 交通水务招标公告 const BID_ANNOUNCE_BASE_URL = 'https://njggzy.nanjing.gov.cn/njweb/jtsw/069001/'; // 获取分页URL (实际数据在moreinfosl3.html中,分页为2.html, 3.html...) function getPageUrl(pageIndex) { if (pageIndex === 1) { return `${BASE_URL}moreinfosl3.html`; } return `${BASE_URL}${pageIndex}.html`; } // 获取招标公告分页URL // 数据通过AJAX加载,第1页是 moreinfo5dc.html,第2页起是 2.html, 3.html... function getBidAnnouncePageUrl(pageIndex) { if (pageIndex === 1) { return `${BID_ANNOUNCE_BASE_URL}moreinfo5dc.html`; } return `${BID_ANNOUNCE_BASE_URL}${pageIndex}.html`; } // 检查日期是否在范围内 function isDateInRange(dateStr, startDate, endDate) { if (!dateStr) return false; const date = new Date(dateStr); if (isNaN(date.getTime())) return false; if (startDate && date < new Date(startDate)) return false; if (endDate && date > new Date(endDate)) return false; return true; } // 按时间范围采集多页列表 async function fetchListByDateRange(startDate, endDate, maxPages = 50) { const allItems = []; let shouldContinue = true; let pageIndex = 1; console.log(`开始按时间范围采集: ${startDate || '不限'} 至 ${endDate || '不限'}`); while (shouldContinue && pageIndex <= maxPages) { const pageUrl = getPageUrl(pageIndex); console.log(`正在采集第 ${pageIndex} 页: ${pageUrl}`); try { const html = await fetchHtml(pageUrl); const items = parseList(html); if (items.length === 0) { console.log(`第 ${pageIndex} 页没有数据,停止采集`); break; } let hasItemsInRange = false; let allItemsBeforeRange = true; for (const item of items) { if (isDateInRange(item.date, startDate, endDate)) { allItems.push(item); hasItemsInRange = true; allItemsBeforeRange = false; } else if (startDate && new Date(item.date) < new Date(startDate)) { allItemsBeforeRange = allItemsBeforeRange && true; } else { allItemsBeforeRange = false; } } if (allItemsBeforeRange && startDate) { console.log(`第 ${pageIndex} 页所有项目都早于起始日期,停止采集`); shouldContinue = false; } console.log(`第 ${pageIndex} 页找到 ${items.length} 条,符合条件 ${hasItemsInRange ? '有' : '无'}`); pageIndex++; if (shouldContinue && pageIndex <= maxPages) { await new Promise(resolve => setTimeout(resolve, 500)); } } catch (err) { console.error(`采集第 ${pageIndex} 页失败: ${err.message}`); break; } } console.log(`总共采集了 ${pageIndex - 1} 页,找到 ${allItems.length} 条符合条件的公告`); return allItems; } const http = axios.create({ responseType: 'arraybuffer', timeout: 15000, headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', }, }); function pickEncoding(contentType = '') { const match = /charset=([^;]+)/i.exec(contentType); if (!match) return 'utf-8'; const charset = match[1].trim().toLowerCase(); if (charset.includes('gb')) return 'gbk'; return charset; } async function fetchHtml(url) { const res = await http.get(url); const encoding = pickEncoding(res.headers['content-type']); const html = iconv.decode(res.data, encoding || 'utf-8'); return html; } // 解析列表页HTML,提取中标结果信息 function parseList(html) { const $ = cheerio.load(html); const items = []; // 解析南京公共资源交易平台的列表结构(交通水务中标结果公示) //
  • $('li.ewb-info-item2').each((_, row) => { const $row = $(row); const cells = $row.find('div.ewb-info-num2'); if (cells.length >= 5) { // 获取各字段 const bidNo = $(cells[0]).find('p').attr('title') || $(cells[0]).find('p').text().trim(); const projectName = $(cells[1]).find('p').attr('title') || $(cells[1]).find('p').text().trim(); const bidName = $(cells[2]).find('p').attr('title') || $(cells[2]).find('p').text().trim(); const winningPrice = $(cells[3]).find('p').text().trim(); // 中标价格 const winningDate = $(cells[4]).find('p').text().trim(); // 中标日期 // 从onclick提取详情链接 const onclick = $row.attr('onclick') || ''; const hrefMatch = onclick.match(/window\.open\(['"]([^'"]+)['"]\)/); let href = ''; if (hrefMatch) { href = hrefMatch[1]; // 转换为绝对URL if (href.startsWith('/')) { href = `https://njggzy.nanjing.gov.cn${href}`; } } // 验证日期格式 (YYYY-MM-DD) if (!/^\d{4}-\d{2}-\d{2}$/.test(winningDate)) return; // 解析中标价格 const price = parseFloat(winningPrice); if (isNaN(price)) return; items.push({ bidNo, // 标段编号 title: projectName, // 项目名称 bidName, // 标段名称 winningBid: { // 中标金额 amount: price, unit: '万元' }, date: winningDate, // 中标日期 href }); } }); return items; } // 解析招标公告列表页HTML function parseBidAnnounceList(html) { const $ = cheerio.load(html); const items = []; // 解析南京公共资源交易平台的招标公告列表结构 //
  • $('li.ewb-info-item2').each((_, row) => { const $row = $(row); const onclick = $row.attr('onclick') || ''; // 提取详情链接 const hrefMatch = onclick.match(/window\.open\(['"]([^'"]+)['"]\)/); if (!hrefMatch) return; let href = hrefMatch[1]; if (href.startsWith('/')) { href = `https://njggzy.nanjing.gov.cn${href}`; } // 获取标题(从p标签的title属性或文本) const $titleP = $row.find('.ewb-info-num2').first().find('p'); const title = $titleP.attr('title') || $titleP.text().trim(); // 获取日期 const $dateP = $row.find('.ewb-info-num2').last().find('p'); const dateText = $dateP.text().trim(); const dateMatch = dateText.match(/\d{4}-\d{2}-\d{2}/); const date = dateMatch ? dateMatch[0] : ''; if (title && date) { items.push({ title, date, href, estimatedAmount: null }); } }); return items; } // 解析招标公告详情页,获取合同估算价 async function fetchBidAnnounceDetail(url) { try { const html = await fetchHtml(url); const $ = cheerio.load(html); // 获取页面全部文本 const bodyText = $('body').text(); // 查找合同估算价 (格式如: 合同估算价:4,300,000.00 元) const amountMatch = bodyText.match(/合同估算价[::]\s*([\d,]+\.?\d*)\s*元/); let estimatedAmount = null; if (amountMatch) { // 去掉逗号,转换为数字(单位:元) const amountStr = amountMatch[1].replace(/,/g, ''); estimatedAmount = parseFloat(amountStr); } // 获取标段编码 const bidCodeMatch = bodyText.match(/标段编码[::]\s*([A-Za-z0-9\-]+)/); const bidCode = bidCodeMatch ? bidCodeMatch[1] : null; // 获取招标人 const tendereeMatch = bodyText.match(/招标人[为是][::]?\s*([^\s,,。]+)/); const tenderee = tendereeMatch ? tendereeMatch[1] : null; // 获取计划工期 const durationMatch = bodyText.match(/计划工期[::]\s*(\d+)\s*日历天/); const duration = durationMatch ? parseInt(durationMatch[1]) : null; return { estimatedAmount, bidCode, tenderee, duration, url }; } catch (error) { console.error(`获取招标详情失败 ${url}: ${error.message}`); return { estimatedAmount: null, url }; } } // 按时间范围采集招标公告 async function fetchBidAnnounceByDateRange(startDate, endDate, maxPages = 20, fetchDetails = true) { const allItems = []; let shouldContinue = true; let pageIndex = 1; console.log(`开始采集招标公告: ${startDate || '不限'} 至 ${endDate || '不限'}`); while (shouldContinue && pageIndex <= maxPages) { const pageUrl = getBidAnnouncePageUrl(pageIndex); console.log(`正在采集招标公告第 ${pageIndex} 页: ${pageUrl}`); try { const html = await fetchHtml(pageUrl); const items = parseBidAnnounceList(html); if (items.length === 0) { console.log(`第 ${pageIndex} 页没有数据,停止采集`); break; } let hasItemsInRange = false; let allItemsBeforeRange = true; for (const item of items) { if (isDateInRange(item.date, startDate, endDate)) { allItems.push(item); hasItemsInRange = true; allItemsBeforeRange = false; } else if (startDate && new Date(item.date) < new Date(startDate)) { allItemsBeforeRange = allItemsBeforeRange && true; } else { allItemsBeforeRange = false; } } if (allItemsBeforeRange && startDate) { console.log(`第 ${pageIndex} 页所有项目都早于起始日期,停止采集`); shouldContinue = false; } console.log(`第 ${pageIndex} 页找到 ${items.length} 条,符合条件 ${hasItemsInRange ? '有' : '无'}`); pageIndex++; if (shouldContinue && pageIndex <= maxPages) { await new Promise(resolve => setTimeout(resolve, 500)); } } catch (err) { console.error(`采集第 ${pageIndex} 页失败: ${err.message}`); break; } } console.log(`总共采集了 ${pageIndex - 1} 页,找到 ${allItems.length} 条符合条件的招标公告`); // 如果需要获取详情(合同估算价) if (fetchDetails && allItems.length > 0) { console.log(`开始获取 ${allItems.length} 条招标公告的详情...`); for (let i = 0; i < allItems.length; i++) { const item = allItems[i]; console.log(`获取详情 ${i + 1}/${allItems.length}: ${item.title.substring(0, 30)}...`); const detail = await fetchBidAnnounceDetail(item.href); item.estimatedAmount = detail.estimatedAmount; item.bidCode = detail.bidCode; item.tenderee = detail.tenderee; item.duration = detail.duration; // 添加延迟避免请求过快 if (i < allItems.length - 1) { await new Promise(resolve => setTimeout(resolve, 300)); } } console.log('详情获取完成'); } return allItems; } // API 路由 // 获取列表 app.get('/api/list', async (req, res) => { try { const page = parseInt(req.query.page) || 1; const pageUrl = getPageUrl(page); const html = await fetchHtml(pageUrl); const items = parseList(html); res.json({ success: true, data: items, page }); } catch (error) { res.status(500).json({ success: false, error: error.message }); } }); // 按时间范围获取列表 app.post('/api/list-daterange', async (req, res) => { try { const { startDate, endDate, maxPages = 50 } = req.body; const items = await fetchListByDateRange(startDate, endDate, maxPages); res.json({ success: true, data: items }); } catch (error) { res.status(500).json({ success: false, error: error.message }); } }); // 生成报告 app.post('/api/report', async (req, res) => { try { const { limit = 50, threshold = 50 } = req.body; // 采集列表 const items = []; let pageIndex = 1; const maxPagesToFetch = Math.ceil(limit / 10) + 1; while (items.length < limit && pageIndex <= maxPagesToFetch) { const pageUrl = getPageUrl(pageIndex); console.log(`正在采集第 ${pageIndex} 页: ${pageUrl}`); try { const html = await fetchHtml(pageUrl); const pageItems = parseList(html); if (pageItems.length === 0) { console.log(`第 ${pageIndex} 页没有数据,停止采集`); break; } items.push(...pageItems); pageIndex++; if (items.length < limit && pageIndex <= maxPagesToFetch) { await new Promise(resolve => setTimeout(resolve, 500)); } } catch (err) { console.error(`采集第 ${pageIndex} 页失败: ${err.message}`); break; } } const results = items.slice(0, limit); // 按阈值筛选 const filtered = results.filter((item) => { return item.winningBid && item.winningBid.amount > threshold; }); const total = filtered.reduce( (sum, item) => sum + (item.winningBid?.amount || 0), 0 ); const report = { summary: { total_count: results.length, filtered_count: filtered.length, threshold: `${threshold}万元`, total_amount: `${total.toFixed(2)}万元`, generated_at: new Date().toISOString(), }, projects: filtered.map((item) => ({ bidNo: item.bidNo, title: item.title, bidName: item.bidName, date: item.date, winningBid: item.winningBid, url: item.href, })), }; res.json({ success: true, data: report }); } catch (error) { res.status(500).json({ success: false, error: error.message }); } }); // 按时间范围生成报告 app.post('/api/report-daterange', async (req, res) => { try { const { startDate, endDate, threshold = 50, maxPages = 50 } = req.body; // 按时间范围采集列表 const items = await fetchListByDateRange(startDate, endDate, maxPages); if (items.length === 0) { return res.json({ success: true, data: { summary: { total_count: 0, filtered_count: 0, threshold: `${threshold}万元`, total_amount: '0.00万元', generated_at: new Date().toISOString(), date_range: { startDate, endDate }, }, projects: [], }, }); } // 按阈值筛选 const filtered = items.filter((item) => { return item.winningBid && item.winningBid.amount > threshold; }); const total = filtered.reduce( (sum, item) => sum + (item.winningBid?.amount || 0), 0 ); const report = { summary: { total_count: items.length, filtered_count: filtered.length, threshold: `${threshold}万元`, total_amount: `${total.toFixed(2)}万元`, generated_at: new Date().toISOString(), date_range: { startDate, endDate }, }, projects: filtered.map((item) => ({ bidNo: item.bidNo, title: item.title, bidName: item.bidName, date: item.date, winningBid: item.winningBid, url: item.href, })), }; res.json({ success: true, data: report }); } catch (error) { res.status(500).json({ success: false, error: error.message }); } }); // ========== 招标公告相关API ========== // 获取招标公告列表(简单列表,按页码) app.get('/api/bid-announce/list', async (req, res) => { try { const page = parseInt(req.query.page) || 1; const pageUrl = getBidAnnouncePageUrl(page); const html = await fetchHtml(pageUrl); const items = parseBidAnnounceList(html); res.json({ success: true, data: items, page }); } catch (error) { res.status(500).json({ success: false, error: error.message }); } }); // 获取招标公告列表(按时间范围) app.post('/api/bid-announce/list', async (req, res) => { try { const { startDate, endDate, maxPages = 20, fetchDetails = false } = req.body; const items = await fetchBidAnnounceByDateRange(startDate, endDate, maxPages, fetchDetails); res.json({ success: true, data: items }); } catch (error) { res.status(500).json({ success: false, error: error.message }); } }); // 生成招标公告报告(含金额统计) app.post('/api/bid-announce/report', async (req, res) => { try { const { startDate, endDate, threshold = 0, maxPages = 20 } = req.body; // 采集招标公告(包含详情) const items = await fetchBidAnnounceByDateRange(startDate, endDate, maxPages, true); if (items.length === 0) { return res.json({ success: true, data: { summary: { total_count: 0, filtered_count: 0, threshold: threshold > 0 ? `${threshold}元` : '无', total_amount: '0.00元', generated_at: new Date().toISOString(), date_range: { startDate, endDate }, report_type: '招标公告' }, projects: [], }, }); } // 按阈值筛选(阈值单位为元) const filtered = threshold > 0 ? items.filter(item => item.estimatedAmount && item.estimatedAmount >= threshold) : items.filter(item => item.estimatedAmount); // 计算总金额 const total = filtered.reduce((sum, item) => sum + (item.estimatedAmount || 0), 0); const report = { summary: { total_count: items.length, filtered_count: filtered.length, has_amount_count: items.filter(i => i.estimatedAmount).length, threshold: threshold > 0 ? `${(threshold / 10000).toFixed(2)}万元` : '无', total_amount: `${(total / 10000).toFixed(2)}万元`, total_amount_yuan: total, generated_at: new Date().toISOString(), date_range: { startDate, endDate }, report_type: '招标公告' }, projects: filtered.map((item) => ({ title: item.title, bidCode: item.bidCode, tenderee: item.tenderee, date: item.date, duration: item.duration, estimatedAmount: item.estimatedAmount ? { amount: item.estimatedAmount, amountWan: (item.estimatedAmount / 10000).toFixed(2), unit: '元' } : null, url: item.href, })), }; res.json({ success: true, data: report }); } catch (error) { res.status(500).json({ success: false, error: error.message }); } }); // 发送招标公告报告邮件 app.post('/api/bid-announce/send-email', async (req, res) => { try { const { emailConfig, report } = req.body; if (!emailConfig || !emailConfig.smtpHost || !emailConfig.smtpUser || !emailConfig.smtpPass) { return res.status(400).json({ success: false, error: '邮件配置不完整,请填写SMTP服务器、用户名和密码', }); } if (!emailConfig.recipients || emailConfig.recipients.trim() === '') { return res.status(400).json({ success: false, error: '请至少指定一个收件人', }); } if (!report) { return res.status(400).json({ success: false, error: '没有可发送的报告数据', }); } // 使用招标公告专用的邮件发送 const result = await sendBidAnnounceReportEmail(emailConfig, report); res.json({ success: true, message: '招标公告报告邮件发送成功', messageId: result.messageId, }); } catch (error) { console.error('发送招标公告邮件API错误:', error); res.status(500).json({ success: false, error: error.message, }); } }); // 发送综合报告邮件(中标+招标) app.post('/api/send-combined-email', async (req, res) => { try { const { emailConfig, winningReport, bidReport } = req.body; if (!emailConfig || !emailConfig.smtpHost || !emailConfig.smtpUser || !emailConfig.smtpPass) { return res.status(400).json({ success: false, error: '邮件配置不完整,请填写SMTP服务器、用户名和密码', }); } if (!emailConfig.recipients || emailConfig.recipients.trim() === '') { return res.status(400).json({ success: false, error: '请至少指定一个收件人', }); } if (!winningReport && !bidReport) { return res.status(400).json({ success: false, error: '没有可发送的报告数据', }); } // 发送综合邮件 const result = await sendCombinedReportEmail(emailConfig, winningReport, bidReport); res.json({ success: true, message: '综合报告邮件发送成功', messageId: result.messageId, }); } catch (error) { console.error('发送综合邮件API错误:', error); res.status(500).json({ success: false, error: error.message, }); } }); // 发送报告邮件 app.post('/api/send-email', async (req, res) => { try { const { emailConfig, report } = req.body; // 验证必需的配置参数 if (!emailConfig || !emailConfig.smtpHost || !emailConfig.smtpUser || !emailConfig.smtpPass) { return res.status(400).json({ success: false, error: '邮件配置不完整,请填写SMTP服务器、用户名和密码', }); } if (!emailConfig.recipients || emailConfig.recipients.trim() === '') { return res.status(400).json({ success: false, error: '请至少指定一个收件人', }); } if (!report) { return res.status(400).json({ success: false, error: '没有可发送的报告数据', }); } // 发送邮件 const result = await sendReportEmail(emailConfig, report); res.json({ success: true, message: '邮件发送成功', messageId: result.messageId, }); } catch (error) { console.error('发送邮件API错误:', error); res.status(500).json({ success: false, error: error.message, }); } }); // 获取配置 app.get('/api/config', async (req, res) => { try { const { readFileSync } = await import('fs'); const { join } = await import('path'); const { fileURLToPath } = await import('url'); const { dirname } = await import('path'); const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); const configPath = join(__dirname, '..', 'config.json'); const configContent = readFileSync(configPath, 'utf-8'); const config = JSON.parse(configContent); // 不返回敏感信息(密码) if (config.email && config.email.smtpPass) { config.email.smtpPass = '***已配置***'; } res.json({ success: true, data: config }); } catch (error) { res.status(500).json({ success: false, error: error.message }); } }); // 更新配置 app.post('/api/config', async (req, res) => { try { const { writeFileSync, readFileSync } = await import('fs'); const { join } = await import('path'); const { fileURLToPath } = await import('url'); const { dirname } = await import('path'); const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); const configPath = join(__dirname, '..', 'config.json'); const newConfig = req.body; // 读取旧配置以保留敏感信息 const oldConfigContent = readFileSync(configPath, 'utf-8'); const oldConfig = JSON.parse(oldConfigContent); // 如果密码字段是占位符,保留原密码 if (newConfig.email && newConfig.email.smtpPass === '***已配置***') { newConfig.email.smtpPass = oldConfig.email?.smtpPass || ''; } // 保存配置 writeFileSync(configPath, JSON.stringify(newConfig, null, 2), 'utf-8'); // 重新加载定时任务(如果定时任务配置有变化) reloadScheduler(); res.json({ success: true, message: '配置已保存并重新加载定时任务' }); } catch (error) { res.status(500).json({ success: false, error: error.message }); } }); // 获取定时任务状态 app.get('/api/scheduler/status', async (req, res) => { try { const status = getSchedulerStatus(); res.json({ success: true, data: status }); } catch (error) { res.status(500).json({ success: false, error: error.message }); } }); // 手动触发定时任务的API(用于测试) app.post('/api/run-scheduled-task', async (req, res) => { try { console.log('手动触发定时任务...'); // 在后台执行任务,不阻塞响应 runTaskNow().catch(err => { console.error('定时任务执行失败:', err); }); res.json({ success: true, message: '定时任务已触发,正在后台执行...' }); } catch (error) { res.status(500).json({ success: false, error: error.message }); } }); app.listen(PORT, () => { console.log(`Server running at http://localhost:${PORT}`); // 启动定时任务 console.log('正在初始化定时任务...'); initScheduler(); });