Compare commits
8 Commits
a904137b60
...
agent_new2
| Author | SHA1 | Date | |
|---|---|---|---|
| a2408fa952 | |||
| 61c93882d6 | |||
| 4653b1d7b9 | |||
| ad659c4ff0 | |||
| e3766b86be | |||
| ed03bd2032 | |||
| fb70356f5d | |||
| f2c856ab05 |
5
.env
Normal file
5
.env
Normal file
@@ -0,0 +1,5 @@
|
||||
# 服务器端口配置
|
||||
PORT=5000
|
||||
|
||||
# Firecrawl API Key（在 https://www.firecrawl.dev/app/api-keys 获取）
|
||||
FIRECRAWL_API_KEY=fc-595dd922780442f8a907202666a522ef
|
||||
@@ -1,11 +1,15 @@
|
||||
# 服务器端口配置
|
||||
PORT=5000
|
||||
|
||||
# Firecrawl API Key（在 https://www.firecrawl.dev/app/api-keys 获取）
|
||||
FIRECRAWL_API_KEY=fc-your-api-key-here
|
||||
|
||||
# 环境说明：
|
||||
# - 开发环境：通常使用 5000
|
||||
# - 生产环境：可以使用 80、8080 等
|
||||
#
|
||||
# 使用方法：
|
||||
# 1. 复制此文件为 .env
|
||||
# 2. 修改端口号
|
||||
# 3. 启动服务时会自动读取
|
||||
# 2. 填写 FIRECRAWL_API_KEY
|
||||
# 3. 修改端口号（可选）
|
||||
# 4. 启动服务时会自动读取
|
||||
|
||||
@@ -3,7 +3,7 @@ name: Deploy Vue App
|
||||
# 触发条件：推送到 master 分支时执行
|
||||
on:
|
||||
push:
|
||||
branches: ['master']
|
||||
branches: ["南京公共资源交易中心"]
|
||||
|
||||
env:
|
||||
IP: 106.15.181.192
|
||||
@@ -13,11 +13,20 @@ jobs:
|
||||
# 运行在已注册的 runner 上（确保 runner 标签匹配，默认无标签可省略）
|
||||
runs-on: [test]
|
||||
steps:
|
||||
# 步骤 4：部署到服务器
|
||||
- name: Deploy to server
|
||||
# 步骤 1：拉取代码
|
||||
- name: Checkout code
|
||||
uses: https://gitee.com/skr2005/checkout@v4
|
||||
# 步骤 2：停止服务
|
||||
- name: Stop service
|
||||
run: net stop tool-node
|
||||
|
||||
# 步骤 3：拷贝源码到目标目录
|
||||
- name: Copy source to target
|
||||
run: xcopy /E /I /Y "${{ github.workspace }}\*" "D:\tools\tool-node\"
|
||||
|
||||
# 步骤 4：安装依赖并启动服务
|
||||
- name: Install and start service
|
||||
run: |
|
||||
cd D:\tools\tool-node
|
||||
npm install
|
||||
git pull
|
||||
net stop tool-node
|
||||
net start tool-node
|
||||
|
||||
2
.gitignore
vendored
2
.gitignore
vendored
@@ -8,7 +8,7 @@ yarn-error.log*
|
||||
pnpm-debug.log*
|
||||
|
||||
# 环境变量文件
|
||||
.env
|
||||
# .env (已注释，不再忽略)
|
||||
.env.local
|
||||
.env.*.local
|
||||
|
||||
|
||||
889
config.json
889
config.json
@@ -2,9 +2,8 @@
|
||||
"scheduler": {
|
||||
"enabled": false,
|
||||
"cronTime": "0 9 * * *",
|
||||
"threshold": 100000,
|
||||
"description": "每天9点采集当日大于10亿的项目",
|
||||
"timeRange": "today"
|
||||
"threshold": 0,
|
||||
"description": "每天9点采集当日项目"
|
||||
},
|
||||
"email": {
|
||||
"smtpHost": "smtp.qq.com",
|
||||
@@ -12,5 +11,887 @@
|
||||
"smtpUser": "1076597680@qq.com",
|
||||
"smtpPass": "nfrjdiraqddsjeeh",
|
||||
"recipients": "5482498@qq.com"
|
||||
}
|
||||
},
|
||||
"scrapers": [
|
||||
{
|
||||
"id": "scraper-1772762494299",
|
||||
"city": "南京市",
|
||||
"url": "https://njggzy.nanjing.gov.cn/njweb/fjsz/buildService1.html",
|
||||
"section": "房建市政",
|
||||
"subsection": "工程类、服务类",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今天的招标公告信息，包括:标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期(YYYY-MM-DD格式)、详情页完整URL，如果没有该日期数据，直接忽略并输出结果",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772958569164",
|
||||
"city": "南京市",
|
||||
"url": "https://njggzy.nanjing.gov.cn/njweb/jtsw/traffic.html",
|
||||
"section": "交通水务",
|
||||
"subsection": "交通",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772958590218",
|
||||
"city": "南京市",
|
||||
"url": "https://njggzy.nanjing.gov.cn/njweb/jtsw/069005/traffic5.html",
|
||||
"section": "交通水务",
|
||||
"subsection": "水务",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772958710168",
|
||||
"city": "无锡市",
|
||||
"url": "https://ggzyjy.wuxi.gov.cn/wxsggzyjyzxzl/jyxx/jsgc/index.shtml",
|
||||
"section": "建设工程",
|
||||
"subsection": "工程类、非工程类",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772958756969",
|
||||
"city": "无锡市",
|
||||
"url": "https://ggzyjy.wuxi.gov.cn/wxsggzyjyzxzl/jyxx/slgc/index.shtml",
|
||||
"section": "水利工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上2026-03-05招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772958789571",
|
||||
"city": "无锡市",
|
||||
"url": "https://ggzyjy.wuxi.gov.cn/wxsggzyjyzxzl/jyxx/jtgc/index.shtml",
|
||||
"section": "交通工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772958889688",
|
||||
"city": "徐州市",
|
||||
"url": "https://ggzy.zwb.xz.gov.cn/jyxx/003001/003001014/list.html",
|
||||
"section": "建设工程",
|
||||
"subsection": "",
|
||||
"type": "招标文件提前公示",
|
||||
"prompt": "从页面中提取今日的招标文件提前公示信息，具体字段包括：标题、项目金额（包含合同预估价、最高投标限价等所有涉及金额的信息）、发布日期（严格按照 YYYY-MM-DD 格式）、详情页完整 URL。若当前为分页第一页且未检索到任何符合 “今日发布” 条件的公示信息，直接返回 “无数据”，无需执行后续翻页及提取操作；若第一页存在有效数据，则正常提取对应字段信息",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772958933270",
|
||||
"city": "徐州市",
|
||||
"url": "https://ggzy.zwb.xz.gov.cn/jyxx/003001/003001001/list.html",
|
||||
"section": "建设工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告/资审公告",
|
||||
"prompt": "从页面中提取今日的招标公告/资审公告提前公示信息，具体字段包括：标题、项目金额（包含合同预估价、最高投标限价等所有涉及金额的信息）、发布日期（严格按照 YYYY-MM-DD 格式）、详情页完整 URL。若当前为分页第一页且未检索到任何符合 “今日发布” 条件的公示信息，直接返回 “无数据”，无需执行后续翻页及提取操作；若第一页存在有效数据，则正常提取对应字段信息",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772958989431",
|
||||
"city": "徐州市",
|
||||
"url": "https://ggzy.zwb.xz.gov.cn/jyxx/003002/003002005/list.html",
|
||||
"section": "交通工程",
|
||||
"subsection": "",
|
||||
"type": "招标计划",
|
||||
"prompt": "提取页面上今日的招标计划信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772959084636",
|
||||
"city": "徐州市",
|
||||
"url": "https://ggzy.zwb.xz.gov.cn/jyxx/003002/003002001/list.html",
|
||||
"section": "交通工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772959127769",
|
||||
"city": "徐州市",
|
||||
"url": "https://ggzy.zwb.xz.gov.cn/jyxx/003003/003003005/list.html",
|
||||
"section": "水务工程",
|
||||
"subsection": "",
|
||||
"type": "招标计划",
|
||||
"prompt": "提取页面上今日的招标计划信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772959152540",
|
||||
"city": "徐州市",
|
||||
"url": "https://ggzy.zwb.xz.gov.cn/jyxx/003003/003003001/list.html",
|
||||
"section": "水务工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772959320386",
|
||||
"city": "常州市",
|
||||
"url": "http://ggzy.xzsp.changzhou.gov.cn/jyzx/001001/tradeInfonew.html?category=001001",
|
||||
"section": "建设工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告/资审公告",
|
||||
"prompt": "提取页面上今日的招标公告/资审公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772959351660",
|
||||
"city": "常州市",
|
||||
"url": "http://ggzy.xzsp.changzhou.gov.cn/jyzx/001001/tradeInfonew.html?category=001001",
|
||||
"section": "交通工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772959481049",
|
||||
"city": "常州市",
|
||||
"url": "http://ggzy.xzsp.changzhou.gov.cn/jyzx/001001/tradeInfonew.html?category=001001",
|
||||
"section": "水利工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772959722015",
|
||||
"city": "苏州市",
|
||||
"url": "http://ggzy.suzhou.gov.cn/jyxx/003001/tradeInfo.html",
|
||||
"section": "建设工程",
|
||||
"subsection": "",
|
||||
"type": "招标预计划",
|
||||
"prompt": "提取页面上今日的招标预计划信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772959755349",
|
||||
"city": "苏州市",
|
||||
"url": "http://ggzy.suzhou.gov.cn/jyxx/003001/tradeInfo.html",
|
||||
"section": "建设工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告/资审公告",
|
||||
"prompt": "提取页面上今日的招标公告/资审公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772959817179",
|
||||
"city": "苏州市",
|
||||
"url": "http://ggzy.suzhou.gov.cn/jyxx/003001/tradeInfo.html",
|
||||
"section": "交通工程",
|
||||
"subsection": "",
|
||||
"type": "招标预计划",
|
||||
"prompt": "提取页面上今日的招标预计划信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772959842489",
|
||||
"city": "苏州市",
|
||||
"url": "http://ggzy.suzhou.gov.cn/jyxx/003001/tradeInfo.html",
|
||||
"section": "交通工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772959877226",
|
||||
"city": "苏州市",
|
||||
"url": "http://ggzy.suzhou.gov.cn/jyxx/003001/tradeInfo.html",
|
||||
"section": "水利工程",
|
||||
"subsection": "",
|
||||
"type": "招标预计划",
|
||||
"prompt": "提取页面上今日的招标预计划信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772959894660",
|
||||
"city": "苏州市",
|
||||
"url": "http://ggzy.suzhou.gov.cn/jyxx/003001/tradeInfo.html",
|
||||
"section": "水务工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772959960153",
|
||||
"city": "南通市",
|
||||
"url": "https://ggzyjy.nantong.gov.cn/jyxx/003001/003001009/tradeInfo.html",
|
||||
"section": "建设工程",
|
||||
"subsection": "",
|
||||
"type": "招标计划",
|
||||
"prompt": "提取页面上今日的招标计划信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772959988900",
|
||||
"city": "南通市",
|
||||
"url": "https://ggzyjy.nantong.gov.cn/jyxx/003001/003001001/tradeInfo.html",
|
||||
"section": "建设工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告/资审公告",
|
||||
"prompt": "提取页面上今日的招标公告/资审公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772960042784",
|
||||
"city": "南通市",
|
||||
"url": "https://ggzyjy.nantong.gov.cn/jyxx/003005/003005001/tradeInfo.html",
|
||||
"section": "交通工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告/招标计划",
|
||||
"prompt": "提取页面上今日的招标公告/招标计划信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772960066902",
|
||||
"city": "南通市",
|
||||
"url": "https://ggzyjy.nantong.gov.cn/jyxx/003006/003006001/tradeInfo.html",
|
||||
"section": "水利工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告/招标计划",
|
||||
"prompt": "提取页面上今日的招标公告/招标计划信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772960133857",
|
||||
"city": "连云港市",
|
||||
"url": "https://ggzy.lyg.gov.cn/lygweb/jyxx/001001/001001008/tradeInfo.html",
|
||||
"section": "建设工程",
|
||||
"subsection": "",
|
||||
"type": "招标计划",
|
||||
"prompt": "提取页面上今日的招标计划信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772960157269",
|
||||
"city": "连云港市",
|
||||
"url": "https://ggzy.lyg.gov.cn/lygweb/jyxx/001001/001001001/tradeInfo.html",
|
||||
"section": "建设工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772960194684",
|
||||
"city": "连云港市",
|
||||
"url": "https://ggzy.lyg.gov.cn/lygweb/jyxx/001003/001003004/tradeInfo.html",
|
||||
"section": "水利工程",
|
||||
"subsection": "",
|
||||
"type": "招标计划",
|
||||
"prompt": "提取页面上今日的招标计划信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772960218045",
|
||||
"city": "连云港市",
|
||||
"url": "https://ggzy.lyg.gov.cn/lygweb/jyxx/001003/001003001/tradeInfo.html",
|
||||
"section": "水务工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772960262581",
|
||||
"city": "连云港市",
|
||||
"url": "https://ggzy.lyg.gov.cn/lygweb/jyxx/001002/001002004/tradeInfo.html",
|
||||
"section": "交通工程",
|
||||
"subsection": "",
|
||||
"type": "招标计划",
|
||||
"prompt": "提取页面上今日的招标计划信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772960280984",
|
||||
"city": "连云港市",
|
||||
"url": "https://ggzy.lyg.gov.cn/lygweb/jyxx/001002/001002001/tradeInfo.html",
|
||||
"section": "交通工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772960353235",
|
||||
"city": "淮安市",
|
||||
"url": "https://ggzy.huaian.gov.cn/jyxx/001001/001001009/tradeInfo.html",
|
||||
"section": "建设工程",
|
||||
"subsection": "",
|
||||
"type": "招标计划",
|
||||
"prompt": "提取页面上今日的招标计划信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772960376613",
|
||||
"city": "淮安市",
|
||||
"url": "https://ggzy.huaian.gov.cn/jyxx/001001/001001001/tradeInfo.html",
|
||||
"section": "建设工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772960419655",
|
||||
"city": "淮安市",
|
||||
"url": "https://ggzy.huaian.gov.cn/jyxx/001002/001002009/tradeInfo.html",
|
||||
"section": "交通工程",
|
||||
"subsection": "",
|
||||
"type": "招标计划",
|
||||
"prompt": "提取页面上今日的招标计划信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772960532565",
|
||||
"city": "淮安市",
|
||||
"url": "https://ggzy.huaian.gov.cn/jyxx/001002/001002001/tradeInfo.html",
|
||||
"section": "交通工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772960574486",
|
||||
"city": "淮安市",
|
||||
"url": "https://ggzy.huaian.gov.cn/jyxx/001003/001003006/tradeInfo.html",
|
||||
"section": "水利工程",
|
||||
"subsection": "",
|
||||
"type": "招标计划",
|
||||
"prompt": "提取页面上今日的招标计划信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772960597160",
|
||||
"city": "淮安市",
|
||||
"url": "https://ggzy.huaian.gov.cn/jyxx/001003/001003001/tradeInfo.html",
|
||||
"section": "水利工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772960644255",
|
||||
"city": "淮安市",
|
||||
"url": "https://ggzy.huaian.gov.cn/jyxx/001009/001009005/tradeInfo.html",
|
||||
"section": "土地整治",
|
||||
"subsection": "",
|
||||
"type": "招标计划",
|
||||
"prompt": "提取页面上今日的招标计划信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772960674724",
|
||||
"city": "淮安市",
|
||||
"url": "https://ggzy.huaian.gov.cn/jyxx/001009/001009001/tradeInfo.html",
|
||||
"section": "土地整治",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772960714492",
|
||||
"city": "淮安市",
|
||||
"url": "https://ggzy.huaian.gov.cn/jyxx/001010/001010005/tradeInfo.html",
|
||||
"section": "农田建设",
|
||||
"subsection": "",
|
||||
"type": "招标计划",
|
||||
"prompt": "提取页面上今日的招标计划信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772960745082",
|
||||
"city": "淮安市",
|
||||
"url": "https://ggzy.huaian.gov.cn/jyxx/001010/001010001/tradeInfo.html",
|
||||
"section": "农田建设",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772960817101",
|
||||
"city": "盐城市",
|
||||
"url": "https://ycggzy.jszwfw.gov.cn/tradeInfor?secondId=19&secondCode=transactionInfo",
|
||||
"section": "工程建设",
|
||||
"subsection": "",
|
||||
"type": "招标计划",
|
||||
"prompt": "提取页面上今日的招标计划信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772960871290",
|
||||
"city": "盐城市",
|
||||
"url": "https://ycggzy.jszwfw.gov.cn/tradeInfor?secondId=19&secondCode=transactionInfo",
|
||||
"section": "工程建设",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772961673500",
|
||||
"city": "盐城市",
|
||||
"url": "https://ycggzy.jszwfw.gov.cn/tradeInfor?secondId=19&secondCode=transactionInfo",
|
||||
"section": "交通工程",
|
||||
"subsection": "",
|
||||
"type": "招标计划",
|
||||
"prompt": "提取页面上今日的招标计划信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772961722432",
|
||||
"city": "盐城市",
|
||||
"url": "https://ycggzy.jszwfw.gov.cn/tradeInfor?secondId=19&secondCode=transactionInfo",
|
||||
"section": "交通工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772961991141",
|
||||
"city": "盐城市",
|
||||
"url": "https://ycggzy.jszwfw.gov.cn/tradeInfor?secondId=19&secondCode=transactionInfo",
|
||||
"section": "水利工程",
|
||||
"subsection": "",
|
||||
"type": "招标计划",
|
||||
"prompt": "提取页面上今日的招标计划信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772962017043",
|
||||
"city": "盐城市",
|
||||
"url": "https://ycggzy.jszwfw.gov.cn/tradeInfor?secondId=19&secondCode=transactionInfo",
|
||||
"section": "水利工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772962073734",
|
||||
"city": "盐城市",
|
||||
"url": "https://ycggzy.jszwfw.gov.cn/tradeInfor?secondId=19&secondCode=transactionInfo",
|
||||
"section": "农业农村",
|
||||
"subsection": "",
|
||||
"type": "招标计划",
|
||||
"prompt": "提取页面上今日的招标计划信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772962143448",
|
||||
"city": "盐城市",
|
||||
"url": "https://ycggzy.jszwfw.gov.cn/tradeInfor?secondId=19&secondCode=transactionInfo",
|
||||
"section": "农业农村",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772962220152",
|
||||
"city": "扬州市",
|
||||
"url": "https://ggzyjyzx.yangzhou.gov.cn/jyxx/fjsz/zbgg/index.html",
|
||||
"section": "房建市政",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772962257442",
|
||||
"city": "扬州市",
|
||||
"url": "https://ggzyjyzx.yangzhou.gov.cn/jyxx/jtgc/zbgg/index.html",
|
||||
"section": "交通工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772962297502",
|
||||
"city": "扬州市",
|
||||
"url": "https://ggzyjyzx.yangzhou.gov.cn/jyxx/slgc/zbgg/index.html",
|
||||
"section": "水利工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772962349287",
|
||||
"city": "扬州市",
|
||||
"url": "https://ggzyjyzx.yangzhou.gov.cn/jyxx/nygc/nyzbgg/index.html",
|
||||
"section": "农业工程",
|
||||
"subsection": "",
|
||||
"type": "农业招标公告",
|
||||
"prompt": "提取页面上今日的农业招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772962408213",
|
||||
"city": "镇江市",
|
||||
"url": "https://ggzy.zhenjiang.gov.cn/jyxx/tradeInfonew.html?type=gcjs",
|
||||
"section": "建设工程",
|
||||
"subsection": "",
|
||||
"type": "招标计划",
|
||||
"prompt": "提取页面上今日的招标计划信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772962433571",
|
||||
"city": "镇江市",
|
||||
"url": "https://ggzy.zhenjiang.gov.cn/jyxx/tradeInfonew.html?type=gcjs",
|
||||
"section": "建设工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772962491815",
|
||||
"city": "镇江市",
|
||||
"url": "https://ggzy.zhenjiang.gov.cn/jyxx/tradeInfonew.html?type=gcjs",
|
||||
"section": "交通工程",
|
||||
"subsection": "",
|
||||
"type": "招标计划",
|
||||
"prompt": "提取页面上今日的招标计划信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772962512616",
|
||||
"city": "镇江市",
|
||||
"url": "https://ggzy.zhenjiang.gov.cn/jyxx/tradeInfonew.html?type=gcjs",
|
||||
"section": "交通工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772962542402",
|
||||
"city": "镇江市",
|
||||
"url": "https://ggzy.zhenjiang.gov.cn/jyxx/tradeInfonew.html?type=gcjs",
|
||||
"section": "水利工程",
|
||||
"subsection": "",
|
||||
"type": "招标计划",
|
||||
"prompt": "提取页面上今日的招标计划信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772962560473",
|
||||
"city": "镇江市",
|
||||
"url": "https://ggzy.zhenjiang.gov.cn/jyxx/tradeInfonew.html?type=gcjs",
|
||||
"section": "水利工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772962599558",
|
||||
"city": "泰州市",
|
||||
"url": "http://ggzy.taizhou.gov.cn/jyxx/001001/001001001/secondPagejyxx.html",
|
||||
"section": "建设工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772962630223",
|
||||
"city": "泰州市",
|
||||
"url": "http://ggzy.taizhou.gov.cn/jyxx/001013/001013001/secondPagejyxx.html",
|
||||
"section": "能源工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772962664564",
|
||||
"city": "泰州市",
|
||||
"url": "http://ggzy.taizhou.gov.cn/jyxx/001002/001002001/secondPagejyxx.html",
|
||||
"section": "交通工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772962689903",
|
||||
"city": "泰州市",
|
||||
"url": "http://ggzy.taizhou.gov.cn/jyxx/001003/001003001/secondPagejyxx.html",
|
||||
"section": "水利工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772962734224",
|
||||
"city": "泰州市",
|
||||
"url": "http://ggzy.taizhou.gov.cn/jyxx/001012/001012001/secondPagejyxx.html",
|
||||
"section": "农业工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772962859477",
|
||||
"city": "宿迁市",
|
||||
"url": "https://ggzy.xzspj.suqian.gov.cn/jyxx/001010/tradeInfo.html",
|
||||
"section": "招标计划提前发布",
|
||||
"subsection": "",
|
||||
"type": "招标计划提前发布",
|
||||
"prompt": "提取页面上今日的招标计划提前发布信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772962901103",
|
||||
"city": "宿迁市",
|
||||
"url": "https://ggzy.xzspj.suqian.gov.cn/jyxx/001001/001001010/tradeInfo.html",
|
||||
"section": "建设工程",
|
||||
"subsection": "",
|
||||
"type": "招标预公告",
|
||||
"prompt": "提取页面上今日的招标预公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772962919812",
|
||||
"city": "宿迁市",
|
||||
"url": "https://ggzy.xzspj.suqian.gov.cn/jyxx/001001/001001001/tradeInfo.html",
|
||||
"section": "建设工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告/资审公告",
|
||||
"prompt": "提取页面上今日的招标公告/资审公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772962967658",
|
||||
"city": "宿迁市",
|
||||
"url": "https://ggzy.xzspj.suqian.gov.cn/jyxx/001002/001002001/tradeInfo.html",
|
||||
"section": "交通工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告/资审公告",
|
||||
"prompt": "提取页面上今日的招标公告/资审公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772963005494",
|
||||
"city": "宿迁市",
|
||||
"url": "https://ggzy.xzspj.suqian.gov.cn/jyxx/001003/001003001/tradeInfo.html",
|
||||
"section": "水利工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告/资审公告",
|
||||
"prompt": "提取页面上今日的招标公告/资审公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772963398342",
|
||||
"city": "江苏省公共资源交易平台",
|
||||
"url": "http://jsggzy.jszwfw.gov.cn/jyxx/tradeInfonew.html?type=jsgc",
|
||||
"section": "建设工程",
|
||||
"subsection": "",
|
||||
"type": "招标计划/招标计划变更公告",
|
||||
"prompt": "提取页面上今日的招标计划/招标计划变更公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772963423954",
|
||||
"city": "江苏省公共资源交易平台",
|
||||
"url": "http://jsggzy.jszwfw.gov.cn/jyxx/tradeInfonew.html?type=jsgc",
|
||||
"section": "建设工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告/资审公告",
|
||||
"prompt": "提取页面上今日的招标公告/资审公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772963466949",
|
||||
"city": "江苏省公共资源交易平台",
|
||||
"url": "http://jsggzy.jszwfw.gov.cn/jyxx/tradeInfonew.html?type=jsgc",
|
||||
"section": "交通工程",
|
||||
"subsection": "",
|
||||
"type": "招标计划/招标计划变更公告",
|
||||
"prompt": "提取页面上今日的招标计划/招标计划变更公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772963511551",
|
||||
"city": "江苏省公共资源交易平台",
|
||||
"url": "http://jsggzy.jszwfw.gov.cn/jyxx/tradeInfonew.html?type=jsgc",
|
||||
"section": "交通工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772963581266",
|
||||
"city": "江苏省公共资源交易平台",
|
||||
"url": "http://jsggzy.jszwfw.gov.cn/jyxx/tradeInfonew.html?type=jsgc",
|
||||
"section": "水利工程",
|
||||
"subsection": "",
|
||||
"type": "招标计划/招标计划变更公告",
|
||||
"prompt": "提取页面上今日的招标计划/招标计划变更公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772963597905",
|
||||
"city": "江苏省公共资源交易平台",
|
||||
"url": "http://jsggzy.jszwfw.gov.cn/jyxx/tradeInfonew.html?type=jsgc",
|
||||
"section": "水利工程",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772963721156",
|
||||
"city": "江苏省建设工程招标网",
|
||||
"url": "http://www.jszb.com.cn/JSZB/YW_info/ZhaoBiaoGG/MoreInfo_ZBGG.aspx?Type=%B7%BF%CE%DD%BD%A8%D6%FE%CA%A9%B9%A4",
|
||||
"section": "房屋建筑施工",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772963750768",
|
||||
"city": "江苏省建设工程招标网",
|
||||
"url": "http://www.jszb.com.cn/JSZB/YW_info/ZhaoBiaoGG/MoreInfo_ZBGG.aspx?Type=%CA%D0%D5%FE%B9%A4%B3%CC%CA%A9%B9%A4",
|
||||
"section": "市政工程施工",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772965504453",
|
||||
"city": "江苏省建设工程招标网",
|
||||
"url": "http://www.jszb.com.cn/JSZB/YW_info/ZhaoBiaoGG/MoreInfo_ZBGG.aspx?Type=%B5%A5%B6%C0%D7%B0%CA%CE%D7%B0%D0%DE%CA%A9%B9%A4",
|
||||
"section": "单独装饰装修施工",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772965525132",
|
||||
"city": "江苏省建设工程招标网",
|
||||
"url": "http://www.jszb.com.cn/JSZB/YW_info/ZhaoBiaoGG/MoreInfo_ZBGG.aspx?Type=%C9%E8%BC%C6",
|
||||
"section": "设计",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
},
|
||||
{
|
||||
"id": "scraper-1772965538927",
|
||||
"city": "江苏省建设工程招标网",
|
||||
"url": "http://www.jszb.com.cn/JSZB/YW_info/ZhaoBiaoGG/MoreInfo_ZBGG.aspx?Type=%CB%AE%C0%FB",
|
||||
"section": "水利",
|
||||
"subsection": "",
|
||||
"type": "招标公告",
|
||||
"prompt": "提取页面上2026年3月3日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL",
|
||||
"enabled": true,
|
||||
"model": "spark-1-mini"
|
||||
}
|
||||
]
|
||||
}
|
||||
64
package-lock.json
generated
64
package-lock.json
generated
@@ -8,14 +8,38 @@
|
||||
"name": "njggzy-scraper",
|
||||
"version": "2.0.0",
|
||||
"dependencies": {
|
||||
"axios": "^1.6.8",
|
||||
"@mendable/firecrawl-js": "^4.15.2",
|
||||
"cheerio": "^1.0.0-rc.12",
|
||||
"cors": "^2.8.5",
|
||||
"dotenv": "^17.2.3",
|
||||
"express": "^5.2.1",
|
||||
"iconv-lite": "^0.6.3",
|
||||
"node-cron": "^4.2.1",
|
||||
"nodemailer": "^7.0.11"
|
||||
"nodemailer": "^7.0.11",
|
||||
"zod": "^4.3.6"
|
||||
}
|
||||
},
|
||||
"node_modules/@mendable/firecrawl-js": {
|
||||
"version": "4.15.2",
|
||||
"resolved": "https://registry.npmmirror.com/@mendable/firecrawl-js/-/firecrawl-js-4.15.2.tgz",
|
||||
"integrity": "sha512-J+lfnJpd00irDhy5ZJE58lsdqbc1fC1d7X6/UyF4VFASEGy1GDpR0FuVweasEpFfOhEGS5DZ+dq8Ui21zIFrOw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"axios": "^1.13.5",
|
||||
"typescript-event-target": "^1.1.1",
|
||||
"zod": "^3.23.8",
|
||||
"zod-to-json-schema": "^3.23.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=22.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@mendable/firecrawl-js/node_modules/zod": {
|
||||
"version": "3.25.76",
|
||||
"resolved": "https://registry.npmmirror.com/zod/-/zod-3.25.76.tgz",
|
||||
"integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==",
|
||||
"license": "MIT",
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/colinhacks"
|
||||
}
|
||||
},
|
||||
"node_modules/accepts": {
|
||||
@@ -63,13 +87,13 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/axios": {
|
||||
"version": "1.13.2",
|
||||
"resolved": "https://registry.npmmirror.com/axios/-/axios-1.13.2.tgz",
|
||||
"integrity": "sha512-VPk9ebNqPcy5lRGuSlKx752IlDatOjT9paPlm8A7yOuW2Fbvp4X3JznJtT4f0GzGLLiWE9W8onz51SqLYwzGaA==",
|
||||
"version": "1.13.6",
|
||||
"resolved": "https://registry.npmmirror.com/axios/-/axios-1.13.6.tgz",
|
||||
"integrity": "sha512-ChTCHMouEe2kn713WHbQGcuYrr6fXTBiu460OTwWrWob16g1bXn4vtz07Ope7ewMozJAnEquLk5lWQWtBig9DQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"follow-redirects": "^1.15.6",
|
||||
"form-data": "^4.0.4",
|
||||
"follow-redirects": "^1.15.11",
|
||||
"form-data": "^4.0.5",
|
||||
"proxy-from-env": "^1.1.0"
|
||||
}
|
||||
},
|
||||
@@ -1331,6 +1355,12 @@
|
||||
"url": "https://opencollective.com/express"
|
||||
}
|
||||
},
|
||||
"node_modules/typescript-event-target": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmmirror.com/typescript-event-target/-/typescript-event-target-1.1.2.tgz",
|
||||
"integrity": "sha512-TvkrTUpv7gCPlcnSoEwUVUBwsdheKm+HF5u2tPAKubkIGMfovdSizCTaZRY/NhR8+Ijy8iZZUapbVQAsNrkFrw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/undici": {
|
||||
"version": "7.16.0",
|
||||
"resolved": "https://registry.npmmirror.com/undici/-/undici-7.16.0.tgz",
|
||||
@@ -1384,6 +1414,24 @@
|
||||
"resolved": "https://registry.npmmirror.com/wrappy/-/wrappy-1.0.2.tgz",
|
||||
"integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/zod": {
|
||||
"version": "4.3.6",
|
||||
"resolved": "https://registry.npmmirror.com/zod/-/zod-4.3.6.tgz",
|
||||
"integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==",
|
||||
"license": "MIT",
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/colinhacks"
|
||||
}
|
||||
},
|
||||
"node_modules/zod-to-json-schema": {
|
||||
"version": "3.25.1",
|
||||
"resolved": "https://registry.npmmirror.com/zod-to-json-schema/-/zod-to-json-schema-3.25.1.tgz",
|
||||
"integrity": "sha512-pM/SU9d3YAggzi6MtR4h7ruuQlqKtad8e9S0fmxcMi+ueAK5Korys/aWcV9LIIHTVbj01NdzxcnXSN+O74ZIVA==",
|
||||
"license": "ISC",
|
||||
"peerDependencies": {
|
||||
"zod": "^3.25 || ^4"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,13 +8,12 @@
|
||||
"start": "node src/server.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"axios": "^1.6.8",
|
||||
"cheerio": "^1.0.0-rc.12",
|
||||
"@mendable/firecrawl-js": "latest",
|
||||
"cors": "^2.8.5",
|
||||
"dotenv": "^17.2.3",
|
||||
"express": "^5.2.1",
|
||||
"iconv-lite": "^0.6.3",
|
||||
"node-cron": "^4.2.1",
|
||||
"nodemailer": "^7.0.11"
|
||||
"nodemailer": "^7.0.11",
|
||||
"zod": "^3.24.2"
|
||||
}
|
||||
}
|
||||
529
public/app.js
529
public/app.js
@@ -90,7 +90,7 @@ function displayList(items, container) {
|
||||
<div class="meta">标段编号: ${item.bidNo}</div>
|
||||
<div class="meta">标段名称: ${item.bidName}</div>
|
||||
<div class="meta">发布日期: ${item.date}</div>
|
||||
<span class="budget">${item.budget.amount}${item.budget.unit}</span>
|
||||
<span class="budget">${item.winningBid.amount}${item.winningBid.unit}</span>
|
||||
${item.href ? `<br><a href="${item.href}" target="_blank">查看详情 →</a>` : ''}
|
||||
</div>
|
||||
`).join('')}
|
||||
@@ -189,7 +189,7 @@ function displayReport(report, container) {
|
||||
<div class="meta">标段编号: ${project.bidNo || '-'}</div>
|
||||
<div class="meta">标段名称: ${project.bidName || '-'}</div>
|
||||
<div class="meta">发布日期: ${project.date}</div>
|
||||
<span class="budget">${project.budget.amount}${project.budget.unit}</span>
|
||||
<span class="budget">${project.winningBid.amount}${project.winningBid.unit}</span>
|
||||
${project.url ? `<br><a href="${project.url}" target="_blank">查看详情 →</a>` : ''}
|
||||
</div>
|
||||
`).join('')}
|
||||
@@ -202,10 +202,34 @@ function displayReport(report, container) {
|
||||
async function exportReport() {
|
||||
if (!currentReport) return;
|
||||
|
||||
// 检查docx库是否加载
|
||||
// 按需动态加载docx库
|
||||
if (!window.docx) {
|
||||
alert('Word导出库正在加载中,请稍后再试...');
|
||||
return;
|
||||
try {
|
||||
// 显示加载提示
|
||||
const loadingMsg = document.createElement('div');
|
||||
loadingMsg.textContent = '正在加载导出库...';
|
||||
loadingMsg.style.cssText = 'position:fixed;top:50%;left:50%;transform:translate(-50%,-50%);background:#fff;padding:20px;border-radius:8px;box-shadow:0 4px 12px rgba(0,0,0,0.15);z-index:9999;';
|
||||
document.body.appendChild(loadingMsg);
|
||||
|
||||
// 动态加载docx库
|
||||
await new Promise((resolve, reject) => {
|
||||
const script = document.createElement('script');
|
||||
script.src = 'https://cdn.jsdelivr.net/npm/docx@7.8.2/build/index.js';
|
||||
script.onload = resolve;
|
||||
script.onerror = () => {
|
||||
// 降级到unpkg
|
||||
script.src = 'https://unpkg.com/docx@7.8.2/build/index.js';
|
||||
script.onload = resolve;
|
||||
script.onerror = reject;
|
||||
};
|
||||
document.head.appendChild(script);
|
||||
});
|
||||
|
||||
document.body.removeChild(loadingMsg);
|
||||
} catch (error) {
|
||||
alert('导出库加载失败,请检查网络连接后重试');
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const report = currentReport;
|
||||
@@ -319,7 +343,7 @@ async function exportReport() {
|
||||
new Paragraph({
|
||||
children: [
|
||||
new TextRun({ text: '合同估算价: ', bold: true }),
|
||||
new TextRun({ text: `${project.budget.amount}${project.budget.unit}` })
|
||||
new TextRun({ text: `${project.winningBid.amount}${project.winningBid.unit}` })
|
||||
],
|
||||
spacing: { after: 50 }
|
||||
}),
|
||||
@@ -356,8 +380,8 @@ async function exportReport() {
|
||||
|
||||
// ========== 邮件功能 ==========
|
||||
|
||||
// 保存邮件配置到localStorage
|
||||
function saveEmailConfig() {
|
||||
// 保存邮件配置到服务器
|
||||
async function saveEmailConfig() {
|
||||
const config = {
|
||||
smtpHost: document.getElementById('smtpHost').value,
|
||||
smtpPort: parseInt(document.getElementById('smtpPort').value) || 587,
|
||||
@@ -367,29 +391,107 @@ function saveEmailConfig() {
|
||||
};
|
||||
|
||||
// 验证配置
|
||||
if (!config.smtpHost || !config.smtpUser || !config.smtpPass || !config.recipients) {
|
||||
showEmailStatus('请填写所有必填项', 'error');
|
||||
if (!config.smtpHost || !config.smtpUser || !config.recipients) {
|
||||
showEmailStatus('请填写SMTP服务器、发件人邮箱和收件人', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
// 保存到localStorage
|
||||
localStorage.setItem('emailConfig', JSON.stringify(config));
|
||||
showEmailStatus('邮件配置已保存', 'success');
|
||||
// 如果密码为空,可能是要保持原密码不变
|
||||
const smtpPassInput = document.getElementById('smtpPass');
|
||||
if (!config.smtpPass && smtpPassInput.placeholder.includes('已配置')) {
|
||||
// 使用占位符表示保持原密码
|
||||
config.smtpPass = '***已配置***';
|
||||
} else if (!config.smtpPass) {
|
||||
showEmailStatus('请填写SMTP密码', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
showEmailStatus('正在保存配置...', 'info');
|
||||
|
||||
try {
|
||||
// 先获取当前服务器配置
|
||||
const getResponse = await fetch(`${API_BASE}/config`);
|
||||
const getData = await getResponse.json();
|
||||
|
||||
let fullConfig = { email: config };
|
||||
|
||||
// 如果服务器有其他配置(如scheduler),保留它们
|
||||
if (getData.success && getData.data) {
|
||||
fullConfig = {
|
||||
...getData.data,
|
||||
email: config
|
||||
};
|
||||
}
|
||||
|
||||
// 保存到服务器
|
||||
const saveResponse = await fetch(`${API_BASE}/config`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(fullConfig)
|
||||
});
|
||||
|
||||
const saveData = await saveResponse.json();
|
||||
|
||||
if (saveData.success) {
|
||||
// 同时保存到localStorage作为备份
|
||||
localStorage.setItem('emailConfig', JSON.stringify(config));
|
||||
showEmailStatus('邮件配置已保存到服务器', 'success');
|
||||
} else {
|
||||
showEmailStatus(`保存失败: ${saveData.error}`, 'error');
|
||||
}
|
||||
} catch (error) {
|
||||
showEmailStatus(`保存失败: ${error.message}`, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
// 从localStorage加载邮件配置
|
||||
function loadEmailConfig() {
|
||||
const configStr = localStorage.getItem('emailConfig');
|
||||
if (configStr) {
|
||||
try {
|
||||
const config = JSON.parse(configStr);
|
||||
// 从服务器加载邮件配置
|
||||
async function loadEmailConfig() {
|
||||
try {
|
||||
// 从服务器获取配置
|
||||
const response = await fetch(`${API_BASE}/config`);
|
||||
const data = await response.json();
|
||||
|
||||
if (data.success && data.data && data.data.email) {
|
||||
const config = data.data.email;
|
||||
document.getElementById('smtpHost').value = config.smtpHost || '';
|
||||
document.getElementById('smtpPort').value = config.smtpPort || 587;
|
||||
document.getElementById('smtpUser').value = config.smtpUser || '';
|
||||
document.getElementById('smtpPass').value = config.smtpPass || '';
|
||||
// 如果密码是占位符,保持输入框为空或显示占位符
|
||||
document.getElementById('smtpPass').value = config.smtpPass === '***已配置***' ? '' : (config.smtpPass || '');
|
||||
if (config.smtpPass === '***已配置***') {
|
||||
document.getElementById('smtpPass').placeholder = '***已配置*** (留空保持不变)';
|
||||
}
|
||||
document.getElementById('recipients').value = config.recipients || '';
|
||||
} catch (e) {
|
||||
console.error('加载邮件配置失败:', e);
|
||||
|
||||
// 同时保存到localStorage作为备份
|
||||
localStorage.setItem('emailConfig', JSON.stringify(config));
|
||||
} else {
|
||||
// 如果服务器没有配置,尝试从localStorage加载
|
||||
const configStr = localStorage.getItem('emailConfig');
|
||||
if (configStr) {
|
||||
const config = JSON.parse(configStr);
|
||||
document.getElementById('smtpHost').value = config.smtpHost || '';
|
||||
document.getElementById('smtpPort').value = config.smtpPort || 587;
|
||||
document.getElementById('smtpUser').value = config.smtpUser || '';
|
||||
document.getElementById('smtpPass').value = config.smtpPass || '';
|
||||
document.getElementById('recipients').value = config.recipients || '';
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('从服务器加载邮件配置失败:', error);
|
||||
// 失败时尝试从localStorage加载
|
||||
const configStr = localStorage.getItem('emailConfig');
|
||||
if (configStr) {
|
||||
try {
|
||||
const config = JSON.parse(configStr);
|
||||
document.getElementById('smtpHost').value = config.smtpHost || '';
|
||||
document.getElementById('smtpPort').value = config.smtpPort || 587;
|
||||
document.getElementById('smtpUser').value = config.smtpUser || '';
|
||||
document.getElementById('smtpPass').value = config.smtpPass || '';
|
||||
document.getElementById('recipients').value = config.recipients || '';
|
||||
} catch (e) {
|
||||
console.error('从localStorage加载邮件配置失败:', e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -585,50 +687,34 @@ function cronToFriendlyText(cronTime) {
|
||||
// 加载定时任务配置
|
||||
async function loadSchedulerConfig() {
|
||||
try {
|
||||
// 从服务器获取配置
|
||||
const response = await fetch(`${API_BASE}/config`);
|
||||
const data = await response.json();
|
||||
|
||||
if (data.success && data.data) {
|
||||
const config = data.data;
|
||||
|
||||
// 填充表单
|
||||
if (config.scheduler) {
|
||||
document.getElementById('schedulerEnabled').checked = config.scheduler.enabled || false;
|
||||
const cronTime = config.scheduler.cronTime || '0 9 * * *';
|
||||
document.getElementById('schedulerCronInput').value = cronTime;
|
||||
document.getElementById('schedulerThresholdInput').value = config.scheduler.threshold || 10000;
|
||||
document.getElementById('schedulerThresholdInput').value = config.scheduler.threshold ?? 0;
|
||||
document.getElementById('schedulerDescription').value = config.scheduler.description || '';
|
||||
|
||||
// 时间段配置
|
||||
document.getElementById('schedulerTimeRange').value = config.scheduler.timeRange || 'thisMonth';
|
||||
|
||||
// 反向映射Cron表达式到预设选择器
|
||||
const presetSelector = document.getElementById('schedulerCronPreset');
|
||||
const customGroup = document.getElementById('customCronGroup');
|
||||
|
||||
// 预设值列表
|
||||
const presets = [
|
||||
'0 9 * * *',
|
||||
'0 6 * * *',
|
||||
'0 12 * * *',
|
||||
'0 18 * * *',
|
||||
'0 9,18 * * *',
|
||||
'0 */6 * * *',
|
||||
'0 */12 * * *',
|
||||
'0 9 * * 1',
|
||||
'0 9 1 * *'
|
||||
'0 9 * * *', '0 6 * * *', '0 12 * * *', '0 18 * * *',
|
||||
'0 9,18 * * *', '0 */6 * * *', '0 */12 * * *', '0 9 * * 1', '0 9 1 * *'
|
||||
];
|
||||
|
||||
// 检查是否匹配预设值
|
||||
if (presets.includes(cronTime)) {
|
||||
presetSelector.value = cronTime;
|
||||
customGroup.style.display = 'none';
|
||||
} else {
|
||||
// 自定义时间 - 尝试解析为 "分 时 * * *" 格式
|
||||
presetSelector.value = 'custom';
|
||||
customGroup.style.display = 'block';
|
||||
|
||||
const cronParts = cronTime.split(/\s+/);
|
||||
if (cronParts.length >= 2) {
|
||||
document.getElementById('customMinute').value = cronParts[0];
|
||||
@@ -637,7 +723,6 @@ async function loadSchedulerConfig() {
|
||||
}
|
||||
}
|
||||
|
||||
// 更新状态显示
|
||||
await updateSchedulerStatus();
|
||||
}
|
||||
} catch (error) {
|
||||
@@ -673,9 +758,14 @@ function updateCustomCron() {
|
||||
cronInput.value = `${minute} ${hour} * * *`;
|
||||
}
|
||||
|
||||
document.addEventListener('DOMContentLoaded', function() {
|
||||
loadEmailConfig();
|
||||
loadSchedulerConfig();
|
||||
document.addEventListener('DOMContentLoaded', function () {
|
||||
// 并行加载配置,提高加载速度
|
||||
Promise.all([
|
||||
loadEmailConfig().catch(err => console.error('加载邮件配置失败:', err)),
|
||||
loadSchedulerConfig().catch(err => console.error('加载定时任务配置失败:', err))
|
||||
]).then(() => {
|
||||
console.log('配置加载完成');
|
||||
});
|
||||
|
||||
// 添加自定义时间输入框的事件监听
|
||||
const customHour = document.getElementById('customHour');
|
||||
@@ -705,8 +795,12 @@ async function updateSchedulerStatus() {
|
||||
// 更新执行计划
|
||||
if (status.config) {
|
||||
document.getElementById('schedulerCronTime').textContent = cronToFriendlyText(status.config.cronTime);
|
||||
const thresholdBillion = (status.config.threshold / 10000).toFixed(1);
|
||||
document.getElementById('schedulerThreshold').textContent = `${status.config.threshold}万元 (${thresholdBillion}亿)`;
|
||||
}
|
||||
|
||||
// 更新已启用来源数
|
||||
const enabledCountEl = document.getElementById('schedulerEnabledCount');
|
||||
if (enabledCountEl) {
|
||||
enabledCountEl.textContent = `${status.enabledScrapers ?? '-'} 个`;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
@@ -719,9 +813,8 @@ async function saveSchedulerConfig() {
|
||||
const schedulerConfig = {
|
||||
enabled: document.getElementById('schedulerEnabled').checked,
|
||||
cronTime: document.getElementById('schedulerCronInput').value,
|
||||
threshold: parseInt(document.getElementById('schedulerThresholdInput').value),
|
||||
threshold: parseInt(document.getElementById('schedulerThresholdInput').value) || 0,
|
||||
description: document.getElementById('schedulerDescription').value,
|
||||
timeRange: document.getElementById('schedulerTimeRange').value
|
||||
};
|
||||
|
||||
// 验证Cron表达式格式(简单验证)
|
||||
@@ -731,36 +824,16 @@ async function saveSchedulerConfig() {
|
||||
return;
|
||||
}
|
||||
|
||||
// 从localStorage获取邮件配置
|
||||
const emailConfigStr = localStorage.getItem('emailConfig');
|
||||
let emailConfig = {};
|
||||
|
||||
if (emailConfigStr) {
|
||||
try {
|
||||
emailConfig = JSON.parse(emailConfigStr);
|
||||
} catch (e) {
|
||||
console.error('解析邮件配置失败:', e);
|
||||
}
|
||||
}
|
||||
|
||||
// 如果邮件配置为空,提示用户
|
||||
if (!emailConfig.smtpHost || !emailConfig.smtpUser) {
|
||||
if (confirm('检测到邮件配置未完成,定时任务需要邮件配置才能发送报告。\n\n是否继续保存定时任务配置(不保存邮件配置)?')) {
|
||||
// 继续保存,但不包含邮件配置
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// 构建完整配置对象
|
||||
const fullConfig = {
|
||||
scheduler: schedulerConfig,
|
||||
email: emailConfig
|
||||
};
|
||||
|
||||
showSchedulerStatus('正在保存配置...', 'info');
|
||||
|
||||
try {
|
||||
// 先获取当前服务器配置(保留 email/scrapers 等字段)
|
||||
const getResponse = await fetch(`${API_BASE}/config`);
|
||||
const getData = await getResponse.json();
|
||||
const currentCfg = (getData.success && getData.data) ? getData.data : {};
|
||||
|
||||
const fullConfig = { ...currentCfg, scheduler: schedulerConfig };
|
||||
|
||||
const response = await fetch(`${API_BASE}/config`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
@@ -771,7 +844,6 @@ async function saveSchedulerConfig() {
|
||||
|
||||
if (data.success) {
|
||||
showSchedulerStatus('配置已保存,定时任务已重新加载!', 'success');
|
||||
// 刷新状态显示
|
||||
await updateSchedulerStatus();
|
||||
} else {
|
||||
showSchedulerStatus(`保存失败: ${data.error}`, 'error');
|
||||
@@ -834,3 +906,306 @@ function showSchedulerStatus(message, type) {
|
||||
}, 3000);
|
||||
}
|
||||
}
|
||||
|
||||
// ========== 招标公告列表功能 ==========
|
||||
let currentBidListPage = 1;
|
||||
|
||||
// 获取招标公告列表
|
||||
async function fetchBidList(pageNum) {
|
||||
const page = pageNum || parseInt(document.getElementById('bidListPage').value) || 1;
|
||||
const loading = document.getElementById('bidListLoading');
|
||||
const results = document.getElementById('bidListResults');
|
||||
const pagination = document.getElementById('bidListPagination');
|
||||
|
||||
currentBidListPage = page;
|
||||
document.getElementById('bidListPage').value = page;
|
||||
|
||||
loading.classList.add('active');
|
||||
results.innerHTML = '';
|
||||
pagination.style.display = 'none';
|
||||
|
||||
try {
|
||||
const response = await fetch(`${API_BASE}/bid-announce/list?page=${page}`);
|
||||
const data = await response.json();
|
||||
|
||||
if (data.success) {
|
||||
displayBidList(data.data, results);
|
||||
updateBidListPagination(page, data.data.length > 0);
|
||||
} else {
|
||||
results.innerHTML = `<div class="error">错误: ${data.error}</div>`;
|
||||
}
|
||||
} catch (error) {
|
||||
results.innerHTML = `<div class="error">请求失败: ${error.message}</div>`;
|
||||
} finally {
|
||||
loading.classList.remove('active');
|
||||
}
|
||||
}
|
||||
|
||||
function goToBidListPage(page) {
|
||||
if (page < 1) return;
|
||||
fetchBidList(page);
|
||||
}
|
||||
|
||||
function updateBidListPagination(page, hasData) {
|
||||
const pagination = document.getElementById('bidListPagination');
|
||||
const currentPageSpan = document.getElementById('bidCurrentPage');
|
||||
const prevBtn = document.getElementById('bidPrevPage');
|
||||
const firstBtn = document.getElementById('bidFirstPage');
|
||||
const nextBtn = document.getElementById('bidNextPage');
|
||||
|
||||
if (hasData) {
|
||||
pagination.style.display = 'flex';
|
||||
currentPageSpan.textContent = page;
|
||||
|
||||
prevBtn.disabled = page <= 1;
|
||||
firstBtn.disabled = page <= 1;
|
||||
nextBtn.disabled = !hasData;
|
||||
}
|
||||
}
|
||||
|
||||
function displayBidList(items, container) {
|
||||
if (items.length === 0) {
|
||||
container.innerHTML = '<p>没有找到招标公告</p>';
|
||||
return;
|
||||
}
|
||||
|
||||
const html = `
|
||||
<div class="simple-list">
|
||||
<h3 style="margin-bottom: 15px; color: #e67e22;">找到 ${items.length} 条招标公告</h3>
|
||||
${items.map((item, index) => `
|
||||
<div class="list-item" style="border-left-color: #e67e22;">
|
||||
<h3>${index + 1}. ${item.title}</h3>
|
||||
<div class="meta">发布日期: ${item.date}</div>
|
||||
${item.href ? `<a href="${item.href}" target="_blank" style="color: #e67e22;">查看详情 →</a>` : ''}
|
||||
</div>
|
||||
`).join('')}
|
||||
</div>
|
||||
`;
|
||||
|
||||
container.innerHTML = html;
|
||||
}
|
||||
|
||||
// ========== 综合报告功能 ==========
|
||||
let currentWinningReport = null;
|
||||
let currentBidReport = null;
|
||||
|
||||
// 初始化报告日期
|
||||
function initReportDates() {
|
||||
const today = new Date();
|
||||
const firstDayOfMonth = new Date(today.getFullYear(), today.getMonth(), 1);
|
||||
|
||||
document.getElementById('startDate').value = firstDayOfMonth.toISOString().split('T')[0];
|
||||
document.getElementById('endDate').value = today.toISOString().split('T')[0];
|
||||
}
|
||||
|
||||
// 生成综合报告（同时包含中标和招标）
|
||||
async function generateCombinedReport() {
|
||||
const startDate = document.getElementById('startDate').value;
|
||||
const endDate = document.getElementById('endDate').value;
|
||||
const maxPages = parseInt(document.getElementById('maxPages').value) || 10;
|
||||
const winningThreshold = parseFloat(document.getElementById('reportThreshold').value) * 10000 || 0; // 转换为元
|
||||
const bidThreshold = parseFloat(document.getElementById('bidReportThreshold').value) * 10000 || 0;
|
||||
|
||||
if (!startDate && !endDate) {
|
||||
alert('请至少填写开始日期或结束日期');
|
||||
return;
|
||||
}
|
||||
|
||||
const loading = document.getElementById('reportLoading');
|
||||
const loadingText = document.getElementById('reportLoadingText');
|
||||
const results = document.getElementById('reportResults');
|
||||
const sendBtn = document.getElementById('sendEmailBtn');
|
||||
|
||||
loading.classList.add('active');
|
||||
results.innerHTML = '';
|
||||
sendBtn.style.display = 'none';
|
||||
|
||||
try {
|
||||
// 1. 先获取中标报告
|
||||
loadingText.textContent = '正在采集中标公示...';
|
||||
const winningResponse = await fetch(`${API_BASE}/report-daterange`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ startDate, endDate, threshold: winningThreshold, maxPages })
|
||||
});
|
||||
const winningData = await winningResponse.json();
|
||||
|
||||
// 2. 再获取招标报告
|
||||
loadingText.textContent = '正在采集招标公告...';
|
||||
const bidResponse = await fetch(`${API_BASE}/bid-announce/report`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ startDate, endDate, maxPages, threshold: bidThreshold })
|
||||
});
|
||||
const bidData = await bidResponse.json();
|
||||
|
||||
// 3. 显示综合报告
|
||||
if (winningData.success && bidData.success) {
|
||||
currentWinningReport = winningData.data;
|
||||
currentBidReport = bidData.data;
|
||||
displayCombinedReport(winningData.data, bidData.data, results);
|
||||
sendBtn.style.display = 'inline-block';
|
||||
} else {
|
||||
let errorMsg = '';
|
||||
if (!winningData.success) errorMsg += `中标报告错误: ${winningData.error}\n`;
|
||||
if (!bidData.success) errorMsg += `招标报告错误: ${bidData.error}`;
|
||||
results.innerHTML = `<div class="error">${errorMsg}</div>`;
|
||||
}
|
||||
} catch (error) {
|
||||
results.innerHTML = `<div class="error">请求失败: ${error.message}</div>`;
|
||||
} finally {
|
||||
loading.classList.remove('active');
|
||||
loadingText.textContent = '正在生成报告...';
|
||||
}
|
||||
}
|
||||
|
||||
// 显示综合报告
|
||||
function displayCombinedReport(winningReport, bidReport, container) {
|
||||
const html = `
|
||||
<!-- 中标公示部分 -->
|
||||
<div class="summary" style="background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);">
|
||||
<h2>中标公示报告</h2>
|
||||
<div class="stat">
|
||||
<div class="stat-label">总项目数</div>
|
||||
<div class="stat-value">${winningReport.summary.total_count}</div>
|
||||
</div>
|
||||
<div class="stat">
|
||||
<div class="stat-label">符合条件</div>
|
||||
<div class="stat-value">${winningReport.summary.filtered_count}</div>
|
||||
</div>
|
||||
<div class="stat">
|
||||
<div class="stat-label">总金额</div>
|
||||
<div class="stat-value">${winningReport.summary.total_amount}</div>
|
||||
</div>
|
||||
<div class="stat">
|
||||
<div class="stat-label">阈值</div>
|
||||
<div class="stat-value">${winningReport.summary.threshold}</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
${winningReport.projects.length === 0 ? '<p style="color: #999; margin-bottom: 20px;">暂无符合条件的中标项目</p>' : `
|
||||
<h3 style="margin-bottom: 15px;">中标项目列表 (${winningReport.projects.length} 条)</h3>
|
||||
<div class="simple-list" style="margin-bottom: 30px;">
|
||||
${winningReport.projects.map((project, index) => `
|
||||
<div class="list-item">
|
||||
<h3>${index + 1}. ${project.title}</h3>
|
||||
<div class="meta">标段编号: ${project.bidNo || '-'}</div>
|
||||
<div class="meta">标段名称: ${project.bidName || '-'}</div>
|
||||
<div class="meta">发布日期: ${project.date}</div>
|
||||
<span class="budget">${project.winningBid.amount}${project.winningBid.unit}</span>
|
||||
${project.url ? `<br><a href="${project.url}" target="_blank">查看详情 →</a>` : ''}
|
||||
</div>
|
||||
`).join('')}
|
||||
</div>
|
||||
`}
|
||||
|
||||
<!-- 招标公告部分 -->
|
||||
<div class="summary" style="background: linear-gradient(135deg, #e67e22 0%, #d35400 100%); margin-top: 30px;">
|
||||
<h2>招标公告报告</h2>
|
||||
<div class="stat">
|
||||
<div class="stat-label">总公告数量</div>
|
||||
<div class="stat-value">${bidReport.summary.total_count} 条</div>
|
||||
</div>
|
||||
<div class="stat">
|
||||
<div class="stat-label">有金额信息</div>
|
||||
<div class="stat-value">${bidReport.summary.has_amount_count || bidReport.summary.filtered_count} 条</div>
|
||||
</div>
|
||||
<div class="stat">
|
||||
<div class="stat-label">金额阈值</div>
|
||||
<div class="stat-value">${bidReport.summary.threshold || '无'}</div>
|
||||
</div>
|
||||
<div class="stat">
|
||||
<div class="stat-label">合同估算总额</div>
|
||||
<div class="stat-value">${bidReport.summary.total_amount}</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
${bidReport.projects.length === 0 ? '<p style="color: #999;">暂无符合条件的招标项目</p>' : `
|
||||
<h3 style="margin-bottom: 15px; color: #e67e22;">招标项目详情 (${bidReport.projects.length} 条)</h3>
|
||||
<div class="simple-list">
|
||||
${bidReport.projects.map((item, index) => `
|
||||
<div class="list-item" style="border-left-color: #e67e22;">
|
||||
<h3>${index + 1}. ${item.title}</h3>
|
||||
<div class="meta">发布日期: ${item.date}</div>
|
||||
${item.bidCode ? `<div class="meta">标段编码: ${item.bidCode}</div>` : ''}
|
||||
${item.tenderee ? `<div class="meta">招标人: ${item.tenderee}</div>` : ''}
|
||||
${item.duration ? `<div class="meta">工期: ${item.duration} 日历天</div>` : ''}
|
||||
${item.estimatedAmount ? `
|
||||
<span class="budget" style="background: #e67e22;">合同估算价: ${item.estimatedAmount.amountWan} 万元</span>
|
||||
` : ''}
|
||||
<br><a href="${item.url}" target="_blank" style="color: #e67e22;">查看详情 →</a>
|
||||
</div>
|
||||
`).join('')}
|
||||
</div>
|
||||
`}
|
||||
`;
|
||||
|
||||
container.innerHTML = html;
|
||||
}
|
||||
|
||||
// 发送综合报告邮件
|
||||
async function sendCombinedReportByEmail() {
|
||||
if (!currentWinningReport && !currentBidReport) {
|
||||
alert('请先生成报告');
|
||||
return;
|
||||
}
|
||||
|
||||
// 从localStorage获取邮件配置
|
||||
const emailConfigStr = localStorage.getItem('emailConfig');
|
||||
if (!emailConfigStr) {
|
||||
alert('请先在"邮件配置"页面配置邮件服务器信息');
|
||||
return;
|
||||
}
|
||||
|
||||
let emailConfig;
|
||||
try {
|
||||
emailConfig = JSON.parse(emailConfigStr);
|
||||
} catch (e) {
|
||||
alert('邮件配置解析失败,请重新配置');
|
||||
return;
|
||||
}
|
||||
|
||||
if (!emailConfig.smtpHost || !emailConfig.smtpUser || !emailConfig.smtpPass || !emailConfig.recipients) {
|
||||
alert('邮件配置不完整,请检查SMTP服务器、用户名、密码和收件人');
|
||||
return;
|
||||
}
|
||||
|
||||
if (!confirm(`确定要将综合报告发送到以下邮箱吗?\n\n${emailConfig.recipients}`)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const sendBtn = document.getElementById('sendEmailBtn');
|
||||
const originalText = sendBtn.textContent;
|
||||
sendBtn.disabled = true;
|
||||
sendBtn.textContent = '正在发送...';
|
||||
|
||||
try {
|
||||
const response = await fetch(`${API_BASE}/send-combined-email`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
emailConfig,
|
||||
winningReport: currentWinningReport,
|
||||
bidReport: currentBidReport
|
||||
})
|
||||
});
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
if (data.success) {
|
||||
alert('综合报告邮件发送成功!');
|
||||
} else {
|
||||
alert(`邮件发送失败: ${data.error}`);
|
||||
}
|
||||
} catch (error) {
|
||||
alert(`请求失败: ${error.message}`);
|
||||
} finally {
|
||||
sendBtn.disabled = false;
|
||||
sendBtn.textContent = originalText;
|
||||
}
|
||||
}
|
||||
|
||||
// 页面加载时初始化报告日期
|
||||
document.addEventListener('DOMContentLoaded', function () {
|
||||
initReportDates();
|
||||
});
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>南京公共资源交易平台 - 合同估算价采集工具</title>
|
||||
<title>南京公共资源交易平台 - 中标价格采集工具</title>
|
||||
<style>
|
||||
* {
|
||||
margin: 0;
|
||||
@@ -95,7 +96,8 @@
|
||||
color: #333;
|
||||
}
|
||||
|
||||
.form-group input, .form-group select {
|
||||
.form-group input,
|
||||
.form-group select {
|
||||
width: 100%;
|
||||
padding: 12px;
|
||||
border: 2px solid #e0e0e0;
|
||||
@@ -104,7 +106,8 @@
|
||||
transition: border 0.3s;
|
||||
}
|
||||
|
||||
.form-group input:focus, .form-group select:focus {
|
||||
.form-group input:focus,
|
||||
.form-group select:focus {
|
||||
outline: none;
|
||||
border-color: #667eea;
|
||||
}
|
||||
@@ -153,8 +156,13 @@
|
||||
}
|
||||
|
||||
@keyframes spin {
|
||||
0% { transform: rotate(0deg); }
|
||||
100% { transform: rotate(360deg); }
|
||||
0% {
|
||||
transform: rotate(0deg);
|
||||
}
|
||||
|
||||
100% {
|
||||
transform: rotate(360deg);
|
||||
}
|
||||
}
|
||||
|
||||
.results {
|
||||
@@ -323,114 +331,414 @@
|
||||
font-size: 14px;
|
||||
margin: 0 10px;
|
||||
}
|
||||
|
||||
/* ===== 抓取来源配置页样式 ===== */
|
||||
.scrapers-toolbar {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
margin-bottom: 20px;
|
||||
flex-wrap: wrap;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.scrapers-toolbar h2 {
|
||||
margin: 0;
|
||||
color: #667eea;
|
||||
font-size: 20px;
|
||||
}
|
||||
|
||||
.btn-add {
|
||||
background: linear-gradient(135deg, #11998e 0%, #38ef7d 100%);
|
||||
color: white;
|
||||
border: none;
|
||||
padding: 10px 22px;
|
||||
border-radius: 8px;
|
||||
font-size: 15px;
|
||||
cursor: pointer;
|
||||
font-weight: 600;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
transition: all 0.2s;
|
||||
box-shadow: 0 2px 8px rgba(17, 153, 142, 0.3);
|
||||
}
|
||||
|
||||
.btn-add:hover {
|
||||
transform: translateY(-1px);
|
||||
box-shadow: 0 4px 14px rgba(17, 153, 142, 0.4);
|
||||
}
|
||||
|
||||
.scrapers-table-wrap {
|
||||
overflow-x: auto;
|
||||
border-radius: 12px;
|
||||
border: 1px solid #e8eaf0;
|
||||
box-shadow: 0 2px 12px rgba(102, 126, 234, 0.06);
|
||||
}
|
||||
|
||||
.scrapers-table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
font-size: 14px;
|
||||
min-width: 800px;
|
||||
}
|
||||
|
||||
.scrapers-table thead tr {
|
||||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||||
color: white;
|
||||
}
|
||||
|
||||
.scrapers-table th {
|
||||
padding: 14px 14px;
|
||||
text-align: left;
|
||||
font-weight: 600;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.scrapers-table tbody tr {
|
||||
border-bottom: 1px solid #f0f0f5;
|
||||
transition: background 0.15s;
|
||||
}
|
||||
|
||||
.scrapers-table tbody tr:last-child {
|
||||
border-bottom: none;
|
||||
}
|
||||
|
||||
.scrapers-table tbody tr:hover {
|
||||
background: #f5f7ff;
|
||||
}
|
||||
|
||||
.scrapers-table td {
|
||||
padding: 12px 14px;
|
||||
vertical-align: top;
|
||||
color: #333;
|
||||
}
|
||||
|
||||
.scrapers-table td.prompt-cell {
|
||||
max-width: 220px;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
color: #666;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.tag {
|
||||
display: inline-block;
|
||||
padding: 2px 10px;
|
||||
border-radius: 20px;
|
||||
font-size: 12px;
|
||||
font-weight: 600;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.tag-type {
|
||||
background: #e8f4fd;
|
||||
color: #1a73c8;
|
||||
}
|
||||
|
||||
.tag-enabled {
|
||||
background: #e4f9ee;
|
||||
color: #1a8a4a;
|
||||
}
|
||||
|
||||
.tag-disabled {
|
||||
background: #feeaea;
|
||||
color: #c0392b;
|
||||
}
|
||||
|
||||
.url-cell a {
|
||||
color: #667eea;
|
||||
text-decoration: none;
|
||||
font-size: 12px;
|
||||
word-break: break-all;
|
||||
}
|
||||
|
||||
.url-cell a:hover {
|
||||
text-decoration: underline;
|
||||
}
|
||||
|
||||
.action-btns {
|
||||
display: flex;
|
||||
gap: 6px;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.btn-sm {
|
||||
padding: 5px 12px;
|
||||
border-radius: 6px;
|
||||
border: none;
|
||||
font-size: 12px;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
transition: all 0.15s;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.btn-edit {
|
||||
background: #fff3cd;
|
||||
color: #856404;
|
||||
}
|
||||
|
||||
.btn-edit:hover {
|
||||
background: #ffc107;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.btn-delete {
|
||||
background: #fdeaea;
|
||||
color: #c0392b;
|
||||
}
|
||||
|
||||
.btn-delete:hover {
|
||||
background: #e74c3c;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.btn-run {
|
||||
background: #e8f4fd;
|
||||
color: #1a73c8;
|
||||
}
|
||||
|
||||
.btn-run:hover {
|
||||
background: #667eea;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.btn-toggle-on {
|
||||
background: #e4f9ee;
|
||||
color: #1a8a4a;
|
||||
}
|
||||
|
||||
.btn-toggle-on:hover {
|
||||
background: #27ae60;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.btn-toggle-off {
|
||||
background: #feeaea;
|
||||
color: #c0392b;
|
||||
}
|
||||
|
||||
.btn-toggle-off:hover {
|
||||
background: #e74c3c;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.empty-state {
|
||||
text-align: center;
|
||||
padding: 60px 20px;
|
||||
color: #aaa;
|
||||
}
|
||||
|
||||
.empty-state svg {
|
||||
margin-bottom: 12px;
|
||||
opacity: 0.4;
|
||||
}
|
||||
|
||||
/* 弹窗 */
|
||||
.modal-overlay {
|
||||
display: none;
|
||||
position: fixed;
|
||||
inset: 0;
|
||||
background: rgba(0, 0, 0, 0.45);
|
||||
z-index: 1000;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.modal-overlay.show {
|
||||
display: flex;
|
||||
}
|
||||
|
||||
.modal-box {
|
||||
background: white;
|
||||
border-radius: 16px;
|
||||
padding: 32px;
|
||||
width: 600px;
|
||||
max-width: 95vw;
|
||||
max-height: 90vh;
|
||||
overflow-y: auto;
|
||||
box-shadow: 0 20px 60px rgba(0, 0, 0, 0.3);
|
||||
animation: modalIn 0.2s ease;
|
||||
}
|
||||
|
||||
@keyframes modalIn {
|
||||
from {
|
||||
opacity: 0;
|
||||
transform: scale(0.95) translateY(-10px);
|
||||
}
|
||||
|
||||
to {
|
||||
opacity: 1;
|
||||
transform: scale(1) translateY(0);
|
||||
}
|
||||
}
|
||||
|
||||
.modal-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
margin-bottom: 24px;
|
||||
}
|
||||
|
||||
.modal-header h3 {
|
||||
margin: 0;
|
||||
color: #333;
|
||||
font-size: 18px;
|
||||
}
|
||||
|
||||
.modal-close {
|
||||
background: none;
|
||||
border: none;
|
||||
font-size: 24px;
|
||||
cursor: pointer;
|
||||
color: #999;
|
||||
line-height: 1;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.modal-close:hover {
|
||||
color: #333;
|
||||
}
|
||||
|
||||
.modal-form .form-row {
|
||||
display: grid;
|
||||
grid-template-columns: 1fr 1fr;
|
||||
gap: 14px;
|
||||
}
|
||||
|
||||
.modal-form .form-group {
|
||||
margin-bottom: 16px;
|
||||
}
|
||||
|
||||
.modal-form .form-group label {
|
||||
display: block;
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
color: #555;
|
||||
margin-bottom: 6px;
|
||||
}
|
||||
|
||||
.modal-form .form-group input,
|
||||
.modal-form .form-group select,
|
||||
.modal-form .form-group textarea {
|
||||
width: 100%;
|
||||
padding: 10px 12px;
|
||||
border: 1.5px solid #e0e0e0;
|
||||
border-radius: 8px;
|
||||
font-size: 14px;
|
||||
font-family: inherit;
|
||||
transition: border 0.2s;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
.modal-form .form-group input:focus,
|
||||
.modal-form .form-group select:focus,
|
||||
.modal-form .form-group textarea:focus {
|
||||
outline: none;
|
||||
border-color: #667eea;
|
||||
box-shadow: 0 0 0 3px rgba(102, 126, 234, 0.12);
|
||||
}
|
||||
|
||||
.modal-form .form-group textarea {
|
||||
resize: vertical;
|
||||
min-height: 90px;
|
||||
}
|
||||
|
||||
.modal-footer {
|
||||
display: flex;
|
||||
justify-content: flex-end;
|
||||
gap: 10px;
|
||||
margin-top: 20px;
|
||||
}
|
||||
|
||||
.btn-cancel {
|
||||
background: #f0f0f0;
|
||||
color: #555;
|
||||
border: none;
|
||||
padding: 10px 24px;
|
||||
border-radius: 8px;
|
||||
font-size: 14px;
|
||||
cursor: pointer;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.btn-cancel:hover {
|
||||
background: #e0e0e0;
|
||||
}
|
||||
|
||||
.btn-save {
|
||||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||||
color: white;
|
||||
border: none;
|
||||
padding: 10px 28px;
|
||||
border-radius: 8px;
|
||||
font-size: 14px;
|
||||
cursor: pointer;
|
||||
font-weight: 600;
|
||||
transition: all 0.2s;
|
||||
}
|
||||
|
||||
.btn-save:hover {
|
||||
box-shadow: 0 4px 14px rgba(102, 126, 234, 0.4);
|
||||
}
|
||||
|
||||
.run-result {
|
||||
margin-top: 16px;
|
||||
padding: 14px;
|
||||
background: #f7f8ff;
|
||||
border-radius: 8px;
|
||||
border: 1px solid #e0e5ff;
|
||||
font-size: 13px;
|
||||
max-height: 300px;
|
||||
overflow-y: auto;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-all;
|
||||
color: #333;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<div class="container">
|
||||
<div class="header">
|
||||
<h1>南京公共资源交易平台</h1>
|
||||
<p>房建市政招标公告 - 合同估算价采集工具</p>
|
||||
<p>交通水务中标结果公示 - 中标价格采集工具</p>
|
||||
</div>
|
||||
|
||||
<div class="tabs">
|
||||
<button class="tab active" onclick="switchTab('list')">公告列表</button>
|
||||
<button class="tab" onclick="switchTab('report')">生成报告</button>
|
||||
<button class="tab" onclick="switchTab('scheduler')">定时任务</button>
|
||||
<button class="tab active" onclick="switchTab('scheduler')">定时任务</button>
|
||||
<button class="tab" onclick="switchTab('email')">邮件配置</button>
|
||||
<button class="tab" onclick="switchTab('scrapers')">抓取来源</button>
|
||||
<a href="/results.html" target="_blank" class="tab" style="text-decoration:none;color:inherit;">📊 抓取结果</a>
|
||||
</div>
|
||||
|
||||
<div class="content">
|
||||
<!-- 公告列表 -->
|
||||
<div id="list" class="tab-content active">
|
||||
<div class="form-group">
|
||||
<label>页码 (第1页为最新公告)</label>
|
||||
<input type="number" id="listPage" value="1" min="1" max="300">
|
||||
</div>
|
||||
<button class="btn" onclick="fetchList()">获取公告列表</button>
|
||||
|
||||
<div id="listLoading" class="loading">
|
||||
<div class="spinner"></div>
|
||||
<p>正在采集...</p>
|
||||
</div>
|
||||
|
||||
<div id="listResults" class="results"></div>
|
||||
|
||||
<div id="listPagination" class="pagination" style="display:none;">
|
||||
<button onclick="goToListPage(1)" id="listFirstPage">首页</button>
|
||||
<button onclick="goToListPage(currentListPage - 1)" id="listPrevPage">上一页</button>
|
||||
<span class="page-info">第 <span id="listCurrentPage">1</span> 页</span>
|
||||
<button onclick="goToListPage(currentListPage + 1)" id="listNextPage">下一页</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 生成报告 -->
|
||||
<div id="report" class="tab-content">
|
||||
<div class="form-group">
|
||||
<div class="checkbox-wrapper" onclick="document.getElementById('useDateRange').click();">
|
||||
<input type="checkbox" id="useDateRange" onchange="toggleDateRange()" onclick="event.stopPropagation();">
|
||||
<label for="useDateRange">按时间范围采集</label>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="dateRangeFields" style="display:none;">
|
||||
<div class="form-group">
|
||||
<label>开始日期</label>
|
||||
<input type="date" id="startDate">
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label>结束日期</label>
|
||||
<input type="date" id="endDate">
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label>最大采集页数</label>
|
||||
<input type="number" id="maxPages" value="10" min="1" max="50">
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="normalFields">
|
||||
<div class="form-group">
|
||||
<label>采集数量</label>
|
||||
<input type="number" id="reportLimit" value="50" min="1" max="200">
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label>金额阈值 (万元) - 只显示大于此金额的项目</label>
|
||||
<input type="number" id="reportThreshold" value="100000" min="0" step="100">
|
||||
</div>
|
||||
|
||||
<button class="btn" onclick="generateReport()">生成报告</button>
|
||||
<button class="btn export-btn" onclick="exportReport()" id="exportBtn" style="display:none;">导出Word</button>
|
||||
<button class="btn" onclick="sendReportByEmail()" id="sendEmailBtn" style="display:none; background: linear-gradient(135deg, #f093fb 0%, #f5576c 100%);">发送邮件</button>
|
||||
|
||||
<div id="reportLoading" class="loading">
|
||||
<div class="spinner"></div>
|
||||
<p>正在生成报告...</p>
|
||||
</div>
|
||||
|
||||
<div id="reportResults" class="results"></div>
|
||||
</div>
|
||||
|
||||
<!-- 定时任务 -->
|
||||
<div id="scheduler" class="tab-content">
|
||||
<div id="scheduler" class="tab-content active">
|
||||
<h2 style="margin-bottom: 20px; color: #667eea;">定时任务配置</h2>
|
||||
<p style="color: #666; margin-bottom: 20px;">配置定时任务自动采集大于指定金额的项目并发送邮件报告</p>
|
||||
|
||||
<!-- 任务状态 -->
|
||||
<div id="schedulerStatus" style="margin-bottom: 30px; padding: 20px; background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); color: white; border-radius: 8px;">
|
||||
<div id="schedulerStatus"
|
||||
style="margin-bottom: 30px; padding: 20px; background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); color: white; border-radius: 8px;">
|
||||
<h3 style="margin-top: 0; margin-bottom: 15px;">任务状态</h3>
|
||||
<div style="display: flex; gap: 30px; flex-wrap: wrap;">
|
||||
<div>
|
||||
<div style="opacity: 0.9; font-size: 14px;">运行状态</div>
|
||||
<div style="font-size: 20px; font-weight: bold; margin-top: 5px;" id="schedulerRunningStatus">加载中...</div>
|
||||
<div style="font-size: 20px; font-weight: bold; margin-top: 5px;"
|
||||
id="schedulerRunningStatus">加载中...</div>
|
||||
</div>
|
||||
<div>
|
||||
<div style="opacity: 0.9; font-size: 14px;">执行时间</div>
|
||||
<div style="font-size: 20px; font-weight: bold; margin-top: 5px;" id="schedulerCronTime">-</div>
|
||||
<div style="font-size: 20px; font-weight: bold; margin-top: 5px;" id="schedulerCronTime">-
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<div style="opacity: 0.9; font-size: 14px;">金额阈值</div>
|
||||
<div style="font-size: 20px; font-weight: bold; margin-top: 5px;" id="schedulerThreshold">-</div>
|
||||
<div style="opacity: 0.9; font-size: 14px;">已启用来源</div>
|
||||
<div style="font-size: 20px; font-weight: bold; margin-top: 5px;"
|
||||
id="schedulerEnabledCount">-</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -480,44 +788,35 @@
|
||||
<!-- 隐藏的Cron表达式字段 -->
|
||||
<input type="hidden" id="schedulerCronInput" value="0 9 * * *">
|
||||
|
||||
<div class="form-group">
|
||||
<label>采集时间段</label>
|
||||
<select id="schedulerTimeRange">
|
||||
<option value="today">今日</option>
|
||||
<option value="thisWeek">本周</option>
|
||||
<option value="thisMonth" selected>本月</option>
|
||||
</select>
|
||||
<small style="color: #666; display: block; margin-top: 5px;">
|
||||
今日:今天 | 本周:本周一至今 | 本月:本月1日至今
|
||||
</small>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label>金额阈值 (万元)</label>
|
||||
<input type="number" id="schedulerThresholdInput" value="100000" min="0" step="1000">
|
||||
<label>金额阈值（万元）- 邮件报告中只显示大于此金额的条目，0 表示不筛选</label>
|
||||
<input type="number" id="schedulerThresholdInput" value="0" min="0" step="1000">
|
||||
<small style="color: #666; display: block; margin-top: 5px;">
|
||||
10亿 = 100000万元 | 5亿 = 50000万元 | 1亿 = 10000万元
|
||||
10亿 = 100000万元 | 1亿 = 10000万元 | 0 = 不筛选，全部显示
|
||||
</small>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label>任务描述 (可选)</label>
|
||||
<input type="text" id="schedulerDescription" placeholder="例如: 每天9点采集大于1亿的项目">
|
||||
<input type="text" id="schedulerDescription" placeholder="例如: 每天9点自动抓取所有启用来源">
|
||||
</div>
|
||||
|
||||
<button class="btn" onclick="saveSchedulerConfig()">保存配置</button>
|
||||
<button class="btn" onclick="testSchedulerNow()" style="background: linear-gradient(135deg, #f093fb 0%, #f5576c 100%);">立即测试运行</button>
|
||||
<button class="btn" onclick="testSchedulerNow()"
|
||||
style="background: linear-gradient(135deg, #f093fb 0%, #f5576c 100%);">立即测试运行</button>
|
||||
<button class="btn" onclick="loadSchedulerConfig()" style="background: #6c757d;">刷新状态</button>
|
||||
|
||||
<div id="schedulerConfigStatus" style="margin-top: 20px;"></div>
|
||||
|
||||
<div style="margin-top: 30px; padding: 20px; background: #fff3cd; border-radius: 8px; border-left: 4px solid #ffc107;">
|
||||
<div
|
||||
style="margin-top: 30px; padding: 20px; background: #fff3cd; border-radius: 8px; border-left: 4px solid #ffc107;">
|
||||
<h3 style="margin-top: 0; color: #856404;">使用说明</h3>
|
||||
<ul style="line-height: 1.8; color: #856404;">
|
||||
<li><strong>数据来源:</strong> 南京公共资源交易平台 - 房建市政招标公告</li>
|
||||
<li><strong>采集内容:</strong> 标段编号、项目名称、标段名称、合同估算价、发布日期</li>
|
||||
<li><strong>筛选条件:</strong> 只保留合同估算价大于设定阈值的项目</li>
|
||||
<li><strong>邮件发送:</strong> 自动将筛选结果生成HTML报告并发送到配置的邮箱</li>
|
||||
<li><strong>数据来源:</strong> 运行「抓取来源」页中所有已启用的抓取配置</li>
|
||||
<li><strong>自动抓取:</strong> 按计划时间自动逐个运行所有启用的抓取来源，结果保存到「抓取结果」页</li>
|
||||
<li><strong>邮件通知:</strong> 抓取完成后自动将结果发送到配置的邮箱（需先完成邮件配置）</li>
|
||||
<li><strong>提示:</strong> 请前往「抓取来源」页配置并启用需要定时抓取的来源</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
@@ -553,11 +852,13 @@
|
||||
</div>
|
||||
|
||||
<button class="btn" onclick="saveEmailConfig()">保存配置</button>
|
||||
<button class="btn" onclick="testEmailConfig()" style="background: linear-gradient(135deg, #f093fb 0%, #f5576c 100%);">测试连接</button>
|
||||
<button class="btn" onclick="testEmailConfig()"
|
||||
style="background: linear-gradient(135deg, #f093fb 0%, #f5576c 100%);">测试连接</button>
|
||||
|
||||
<div id="emailConfigStatus" style="margin-top: 20px;"></div>
|
||||
|
||||
<div style="margin-top: 30px; padding: 20px; background: #f0f8ff; border-radius: 8px; border-left: 4px solid #667eea;">
|
||||
<div
|
||||
style="margin-top: 30px; padding: 20px; background: #f0f8ff; border-radius: 8px; border-left: 4px solid #667eea;">
|
||||
<h3 style="margin-top: 0; color: #667eea;">常用邮箱配置参考</h3>
|
||||
<ul style="line-height: 1.8; color: #666;">
|
||||
<li><strong>QQ邮箱:</strong> smtp.qq.com, 端口 587 或 465, 需要使用授权码</li>
|
||||
@@ -571,9 +872,357 @@
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 抓取来源配置 -->
|
||||
<div id="scrapers" class="tab-content" style="padding:30px ;">
|
||||
<div class="scrapers-toolbar">
|
||||
<h2>抓取来源配置</h2>
|
||||
<div style="display:flex;gap:10px;flex-wrap:wrap;">
|
||||
<button class="btn-add" onclick="runAllScrapers()" id="btnRunAll"
|
||||
style="background:linear-gradient(135deg,#667eea 0%,#764ba2 100%);">
|
||||
<svg width="16" height="16" fill="none" stroke="currentColor" stroke-width="2.5"
|
||||
viewBox="0 0 24 24">
|
||||
<polygon points="5,3 19,12 5,21" />
|
||||
</svg>
|
||||
运行全部启用
|
||||
</button>
|
||||
<a href="/results.html" target="_blank" class="btn-add"
|
||||
style="background:linear-gradient(135deg,#11998e 0%,#38ef7d 100%);text-decoration:none;">
|
||||
<svg width="16" height="16" fill="none" stroke="currentColor" stroke-width="2.5"
|
||||
viewBox="0 0 24 24">
|
||||
<rect x="3" y="3" width="18" height="18" rx="3" />
|
||||
<path d="M3 9h18M9 21V9" />
|
||||
</svg>
|
||||
查看结果
|
||||
</a>
|
||||
<button class="btn-add" onclick="openScraperModal()" style="cursor:pointer;">
|
||||
<svg width="16" height="16" fill="none" stroke="currentColor" stroke-width="2.5"
|
||||
viewBox="0 0 24 24">
|
||||
<line x1="12" y1="5" x2="12" y2="19" />
|
||||
<line x1="5" y1="12" x2="19" y2="12" />
|
||||
</svg>
|
||||
新增来源
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p style="color:#888;font-size:13px;margin:-8px 0 18px;">通过配置 URL 和提示词，使用 Firecrawl Browser Sandbox
|
||||
抓取任意网页数据。结果会自动保存，可在「抓取结果」页查看历史。</p>
|
||||
|
||||
<div class="scrapers-table-wrap">
|
||||
<table class="scrapers-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th style="width:80px">城市</th>
|
||||
<th style="width:80px">板块</th>
|
||||
<th style="width:70px">子板块</th>
|
||||
<th style="width:80px">类型</th>
|
||||
<th>链接地址</th>
|
||||
<th>提示词</th>
|
||||
<th style="width:70px">AI模型</th>
|
||||
<th style="width:60px">状态</th>
|
||||
<th style="width:180px">操作</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="scrapersTbody">
|
||||
<tr id="scrapers-empty-row">
|
||||
<td colspan="9" class="empty-state">
|
||||
<svg width="48" height="48" fill="none" stroke="currentColor" stroke-width="1.5"
|
||||
viewBox="0 0 24 24">
|
||||
<rect x="3" y="3" width="18" height="18" rx="3" />
|
||||
<path d="M3 9h18M9 21V9" />
|
||||
</svg>
|
||||
<div>暂无配置，点击「新增来源」添加抓取任务</div>
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<!-- 批量运行状态 -->
|
||||
<div id="batchRunStatus"
|
||||
style="display:none;margin-top:16px;padding:14px;background:#f7f8ff;border-radius:8px;border:1px solid #e0e5ff;font-size:13px;color:#333;">
|
||||
</div>
|
||||
|
||||
<!-- 测试结果展示 -->
|
||||
<div id="scraperRunResult" style="display:none;">
|
||||
<div style="margin-top:20px;font-size:14px;font-weight:600;color:#333;margin-bottom:8px;">📋 测试抓取结果
|
||||
</div>
|
||||
<div class="run-result" id="scraperRunResultContent"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script src="https://unpkg.com/docx@7.8.2/build/index.js"></script>
|
||||
<script src="app.js"></script>
|
||||
<!-- 新增/编辑弹窗 -->
|
||||
<div class="modal-overlay" id="scraperModal">
|
||||
<div class="modal-box">
|
||||
<div class="modal-header">
|
||||
<h3 id="scraperModalTitle">新增抓取来源</h3>
|
||||
<button class="modal-close" onclick="closeScraperModal()">×</button>
|
||||
</div>
|
||||
<form class="modal-form" onsubmit="saveScraperItem(event)">
|
||||
<input type="hidden" id="scraperEditId">
|
||||
<div class="form-row">
|
||||
<div class="form-group">
|
||||
<label>城市 *</label>
|
||||
<input type="text" id="scraperCity" placeholder="例: 南京市" required>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label>板块</label>
|
||||
<input type="text" id="scraperSection" placeholder="例: 交通水务">
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-row">
|
||||
<div class="form-group">
|
||||
<label>子板块</label>
|
||||
<input type="text" id="scraperSubsection" placeholder="例: 建设工程">
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label>类型</label>
|
||||
<input type="text" id="scraperType" placeholder="例: 招标公告">
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label>链接地址 *</label>
|
||||
<input type="url" id="scraperUrl" placeholder="https://..." required>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label>提示词（Agent 指令）*</label>
|
||||
<textarea id="scraperPrompt"
|
||||
placeholder="提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL"
|
||||
required></textarea>
|
||||
</div>
|
||||
<div class="form-row">
|
||||
<div class="form-group">
|
||||
<label>AI 模型</label>
|
||||
<select id="scraperModel">
|
||||
<option value="spark-1-mini">spark-1-mini（默认）</option>
|
||||
<option value="spark-2">spark-2</option>
|
||||
<option value="gpt-4o-mini">gpt-4o-mini</option>
|
||||
<option value="claude-3-haiku">claude-3-haiku</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="form-group" style="display:flex;align-items:flex-end;padding-bottom:2px;">
|
||||
<div class="checkbox-wrapper" style="width:100%;"
|
||||
onclick="document.getElementById('scraperEnabled').click();">
|
||||
<input type="checkbox" id="scraperEnabled" checked onclick="event.stopPropagation();">
|
||||
<label for="scraperEnabled">启用此来源</label>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn-cancel" onclick="closeScraperModal()">取消</button>
|
||||
<button type="submit" class="btn-save">保存</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- docx库已改为按需加载,只在用户点击导出时才加载,提升首屏加载速度 -->
|
||||
<script src="app.js" defer></script>
|
||||
<script>
|
||||
// ===== 抓取来源配置 JS =====
|
||||
let scrapersList = [];
|
||||
|
||||
async function loadScrapers() {
|
||||
try {
|
||||
const res = await fetch('/api/scrapers');
|
||||
const json = await res.json();
|
||||
scrapersList = json.data || [];
|
||||
renderScrapers();
|
||||
} catch (e) {
|
||||
console.error('加载抓取来源失败:', e);
|
||||
}
|
||||
}
|
||||
|
||||
function renderScrapers() {
|
||||
const tbody = document.getElementById('scrapersTbody');
|
||||
if (scrapersList.length === 0) {
|
||||
tbody.innerHTML = `<tr id="scrapers-empty-row"><td colspan="9" class="empty-state">
|
||||
<svg width="48" height="48" fill="none" stroke="currentColor" stroke-width="1.5" viewBox="0 0 24 24"><rect x="3" y="3" width="18" height="18" rx="3"/><path d="M3 9h18M9 21V9"/></svg>
|
||||
<div>暂无配置，点击「新增来源」添加抓取任务</div></td></tr>`;
|
||||
return;
|
||||
}
|
||||
tbody.innerHTML = scrapersList.map(s => `
|
||||
<tr>
|
||||
<td>${s.city || '-'}</td>
|
||||
<td>${s.section || '-'}</td>
|
||||
<td>${s.subsection || '-'}</td>
|
||||
<td><span class="tag tag-type">${s.type || ''}</span></td>
|
||||
<td class="url-cell"><a href="${s.url}" target="_blank" title="${s.url}">${s.url.replace(/^https?:\/\//, '').substring(0, 35)}${s.url.length > 40 ? '...' : ''}</a></td>
|
||||
<td class="prompt-cell" title="${(s.prompt || '').replace(/"/g, '"')}">${s.prompt || '-'}</td>
|
||||
<td style="font-size:12px;color:#888;">${s.model || 'spark-1-mini'}</td>
|
||||
<td><span class="tag ${s.enabled ? 'tag-enabled' : 'tag-disabled'}">${s.enabled ? '启用' : '禁用'}</span></td>
|
||||
<td>
|
||||
<div class="action-btns">
|
||||
<button class="btn-sm btn-edit" onclick="openScraperModal('${s.id}')" title="编辑">编辑</button>
|
||||
<button class="btn-sm btn-run" onclick="runScraper('${s.id}', this)" title="测试运行">测试</button>
|
||||
<button class="btn-sm ${s.enabled ? 'btn-toggle-on' : 'btn-toggle-off'}" onclick="toggleScraper('${s.id}', ${!s.enabled})" title="切换启用状态">${s.enabled ? '禁用' : '启用'}</button>
|
||||
<button class="btn-sm btn-delete" onclick="deleteScraper('${s.id}')" title="删除">删除</button>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
`).join('');
|
||||
}
|
||||
|
||||
function openScraperModal(id) {
|
||||
const item = id ? scrapersList.find(s => s.id === id) : null;
|
||||
document.getElementById('scraperModalTitle').textContent = item ? '编辑抓取来源' : '新增抓取来源';
|
||||
document.getElementById('scraperEditId').value = item ? item.id : '';
|
||||
document.getElementById('scraperCity').value = item ? item.city : '';
|
||||
document.getElementById('scraperSection').value = item ? item.section : '';
|
||||
document.getElementById('scraperSubsection').value = item ? item.subsection : '';
|
||||
document.getElementById('scraperType').value = item ? item.type : '招标公告';
|
||||
document.getElementById('scraperUrl').value = item ? item.url : '';
|
||||
document.getElementById('scraperPrompt').value = item ? item.prompt : '提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等等）、发布日期（YYYY-MM-DD格式）、详情页完整URL';
|
||||
document.getElementById('scraperModel').value = item ? (item.model || 'spark-1-mini') : 'spark-1-mini';
|
||||
document.getElementById('scraperEnabled').checked = item ? item.enabled : true;
|
||||
document.getElementById('scraperRunResult').style.display = 'none';
|
||||
document.getElementById('scraperModal').classList.add('show');
|
||||
}
|
||||
|
||||
function closeScraperModal() {
|
||||
document.getElementById('scraperModal').classList.remove('show');
|
||||
}
|
||||
|
||||
async function saveScraperItem(e) {
|
||||
e.preventDefault();
|
||||
const id = document.getElementById('scraperEditId').value;
|
||||
const data = {
|
||||
city: document.getElementById('scraperCity').value.trim(),
|
||||
url: document.getElementById('scraperUrl').value.trim(),
|
||||
section: document.getElementById('scraperSection').value.trim(),
|
||||
subsection: document.getElementById('scraperSubsection').value.trim(),
|
||||
type: document.getElementById('scraperType').value,
|
||||
prompt: document.getElementById('scraperPrompt').value.trim(),
|
||||
model: document.getElementById('scraperModel').value,
|
||||
enabled: document.getElementById('scraperEnabled').checked,
|
||||
};
|
||||
try {
|
||||
const url = id ? `/api/scrapers/${id}` : '/api/scrapers';
|
||||
const method = id ? 'PUT' : 'POST';
|
||||
const res = await fetch(url, { method, headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(data) });
|
||||
const json = await res.json();
|
||||
if (!json.success) throw new Error(json.error);
|
||||
closeScraperModal();
|
||||
await loadScrapers();
|
||||
} catch (err) {
|
||||
alert('保存失败: ' + err.message);
|
||||
}
|
||||
}
|
||||
|
||||
async function deleteScraper(id) {
|
||||
const item = scrapersList.find(s => s.id === id);
|
||||
if (!confirm(`确定要删除「${item?.city} - ${item?.type}」吗？`)) return;
|
||||
try {
|
||||
const res = await fetch(`/api/scrapers/${id}`, { method: 'DELETE' });
|
||||
const json = await res.json();
|
||||
if (!json.success) throw new Error(json.error);
|
||||
await loadScrapers();
|
||||
} catch (err) {
|
||||
alert('删除失败: ' + err.message);
|
||||
}
|
||||
}
|
||||
|
||||
async function toggleScraper(id, enabled) {
|
||||
try {
|
||||
const res = await fetch(`/api/scrapers/${id}`, {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ enabled })
|
||||
});
|
||||
const json = await res.json();
|
||||
if (!json.success) throw new Error(json.error);
|
||||
await loadScrapers();
|
||||
} catch (err) {
|
||||
alert('操作失败: ' + err.message);
|
||||
}
|
||||
}
|
||||
|
||||
async function runScraper(id, btnEl) {
|
||||
const item = scrapersList.find(s => s.id === id);
|
||||
const resultDiv = document.getElementById('scraperRunResult');
|
||||
const contentDiv = document.getElementById('scraperRunResultContent');
|
||||
resultDiv.style.display = 'block';
|
||||
contentDiv.textContent = `正在测试抓取「${item?.city} - ${item?.type}」，请稍候...`;
|
||||
resultDiv.scrollIntoView({ behavior: 'smooth', block: 'nearest' });
|
||||
|
||||
const originalText = btnEl ? btnEl.textContent : '';
|
||||
if (btnEl) {
|
||||
btnEl.disabled = true;
|
||||
btnEl.textContent = '测试中...';
|
||||
}
|
||||
try {
|
||||
const res = await fetch(`/api/scrapers/${id}/run`, { method: 'POST' });
|
||||
const json = await res.json();
|
||||
if (!json.success) throw new Error(json.error);
|
||||
contentDiv.textContent = JSON.stringify(json.data, null, 2);
|
||||
} catch (err) {
|
||||
contentDiv.textContent = '❌ 测试失败: ' + err.message;
|
||||
} finally {
|
||||
if (btnEl) {
|
||||
btnEl.disabled = false;
|
||||
btnEl.textContent = originalText || '测试';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 切换到抓取来源 Tab 时自动加载
|
||||
const _origSwitchTab = typeof switchTab === 'function' ? switchTab : null;
|
||||
document.addEventListener('DOMContentLoaded', () => {
|
||||
// 拦截 tab 切换，在进入 scrapers tab 时加载数据
|
||||
document.querySelectorAll('.tab').forEach(btn => {
|
||||
if (btn.textContent.trim() === '抓取来源') {
|
||||
btn.addEventListener('click', () => { loadScrapers(); });
|
||||
}
|
||||
});
|
||||
// 点击弹窗遮罩关闭
|
||||
document.getElementById('scraperModal').addEventListener('click', function (e) {
|
||||
if (e.target === this) closeScraperModal();
|
||||
});
|
||||
});
|
||||
|
||||
// 批量运行所有已启用来源
|
||||
async function runAllScrapers() {
|
||||
const enabled = scrapersList.filter(s => s.enabled);
|
||||
if (enabled.length === 0) {
|
||||
alert('没有已启用的抓取来源，请先在列表中启用至少一个来源。');
|
||||
return;
|
||||
}
|
||||
if (!confirm(`确定要运行全部 ${enabled.length} 个已启用的抓取来源吗？\n结果将自动保存，可在「抓取结果」页查看。`)) return;
|
||||
|
||||
const btn = document.getElementById('btnRunAll');
|
||||
const statusDiv = document.getElementById('batchRunStatus');
|
||||
btn.disabled = true;
|
||||
btn.textContent = '运行中...';
|
||||
statusDiv.style.display = 'block';
|
||||
statusDiv.innerHTML = `⏳ 正在运行 ${enabled.length} 个抓取来源，请稍候...`;
|
||||
|
||||
try {
|
||||
const res = await fetch('/api/scrape/run', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({}),
|
||||
});
|
||||
const json = await res.json();
|
||||
if (!json.success) throw new Error(json.error);
|
||||
|
||||
const results = json.data || [];
|
||||
const ok = results.filter(r => !r.error).length;
|
||||
const err = results.filter(r => r.error).length;
|
||||
statusDiv.innerHTML = `
|
||||
✅ 批量抓取完成！成功 <strong>${ok}</strong> 条，失败 <strong>${err}</strong> 条。
|
||||
<a href="/results.html" target="_blank" style="color:#667eea;font-weight:600;text-decoration:underline;">点击查看抓取结果 →</a>
|
||||
`;
|
||||
} catch (e) {
|
||||
statusDiv.innerHTML = `❌ 批量运行失败: ${e.message}`;
|
||||
} finally {
|
||||
btn.disabled = false;
|
||||
btn.innerHTML = `<svg width="16" height="16" fill="none" stroke="currentColor" stroke-width="2.5" viewBox="0 0 24 24"><polygon points="5,3 19,12 5,21"/></svg> 运行全部启用`;
|
||||
}
|
||||
}
|
||||
</script>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
||||
1273
public/results.html
Normal file
1273
public/results.html
Normal file
File diff suppressed because it is too large
Load Diff
1
results.json
Normal file
1
results.json
Normal file
@@ -0,0 +1 @@
|
||||
[]
|
||||
1
scrapegraph-service/.env
Normal file
1
scrapegraph-service/.env
Normal file
@@ -0,0 +1 @@
|
||||
|
||||
@@ -21,7 +21,7 @@ export async function sendReportEmail(emailConfig, report) {
|
||||
const info = await transporter.sendMail({
|
||||
from: `"公告采集系统" <${emailConfig.smtpUser}>`,
|
||||
to: emailConfig.recipients,
|
||||
subject: `采购公告分析报告 - ${new Date().toLocaleDateString('zh-CN')}`,
|
||||
subject: `交通水务中标结果报告 - ${new Date().toLocaleDateString('zh-CN')}`,
|
||||
html: htmlContent,
|
||||
});
|
||||
|
||||
@@ -35,6 +35,523 @@ export async function sendReportEmail(emailConfig, report) {
|
||||
}
|
||||
}
|
||||
|
||||
// 发送招标公告报告邮件
|
||||
export async function sendBidAnnounceReportEmail(emailConfig, report) {
|
||||
try {
|
||||
const transporter = nodemailer.createTransport({
|
||||
host: emailConfig.smtpHost,
|
||||
port: emailConfig.smtpPort || 587,
|
||||
secure: emailConfig.smtpPort === 465,
|
||||
auth: {
|
||||
user: emailConfig.smtpUser,
|
||||
pass: emailConfig.smtpPass,
|
||||
},
|
||||
});
|
||||
|
||||
const htmlContent = generateBidAnnounceReportHtml(report);
|
||||
|
||||
const info = await transporter.sendMail({
|
||||
from: `"公告采集系统" <${emailConfig.smtpUser}>`,
|
||||
to: emailConfig.recipients,
|
||||
subject: `交通水务招标公告报告 - ${new Date().toLocaleDateString('zh-CN')}`,
|
||||
html: htmlContent,
|
||||
});
|
||||
|
||||
return {
|
||||
success: true,
|
||||
messageId: info.messageId,
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('发送招标公告邮件失败:', error);
|
||||
throw new Error(`邮件发送失败: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// 生成招标公告报告HTML
|
||||
function generateBidAnnounceReportHtml(report) {
|
||||
const { summary, projects } = report;
|
||||
|
||||
return `
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>交通水务招标公告报告</title>
|
||||
<style>
|
||||
body {
|
||||
font-family: 'PingFang SC', 'Microsoft YaHei', Arial, sans-serif;
|
||||
line-height: 1.6;
|
||||
color: #333;
|
||||
max-width: 900px;
|
||||
margin: 0 auto;
|
||||
padding: 20px;
|
||||
background-color: #f5f5f5;
|
||||
}
|
||||
.container {
|
||||
background: white;
|
||||
border-radius: 8px;
|
||||
padding: 30px;
|
||||
box-shadow: 0 2px 10px rgba(0,0,0,0.1);
|
||||
}
|
||||
h1 {
|
||||
color: #e67e22;
|
||||
border-bottom: 3px solid #e67e22;
|
||||
padding-bottom: 10px;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
.summary {
|
||||
background: linear-gradient(135deg, #e67e22 0%, #d35400 100%);
|
||||
color: white;
|
||||
padding: 20px;
|
||||
border-radius: 8px;
|
||||
margin-bottom: 30px;
|
||||
}
|
||||
.summary h2 {
|
||||
margin-top: 0;
|
||||
margin-bottom: 15px;
|
||||
font-size: 18px;
|
||||
}
|
||||
.stat-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(180px, 1fr));
|
||||
gap: 15px;
|
||||
}
|
||||
.stat {
|
||||
background: rgba(255,255,255,0.15);
|
||||
padding: 12px;
|
||||
border-radius: 6px;
|
||||
}
|
||||
.stat-label {
|
||||
font-size: 13px;
|
||||
opacity: 0.9;
|
||||
margin-bottom: 5px;
|
||||
}
|
||||
.stat-value {
|
||||
font-size: 20px;
|
||||
font-weight: bold;
|
||||
}
|
||||
.project-list {
|
||||
margin-top: 20px;
|
||||
}
|
||||
.project-item {
|
||||
background: #f9f9f9;
|
||||
border-left: 4px solid #e67e22;
|
||||
padding: 15px;
|
||||
margin-bottom: 15px;
|
||||
border-radius: 4px;
|
||||
}
|
||||
.project-item h3 {
|
||||
color: #333;
|
||||
margin: 0 0 10px 0;
|
||||
font-size: 15px;
|
||||
line-height: 1.4;
|
||||
}
|
||||
.project-meta {
|
||||
color: #666;
|
||||
font-size: 13px;
|
||||
margin: 5px 0;
|
||||
}
|
||||
.amount {
|
||||
display: inline-block;
|
||||
background: #e67e22;
|
||||
color: white;
|
||||
padding: 4px 12px;
|
||||
border-radius: 4px;
|
||||
font-weight: bold;
|
||||
margin-top: 8px;
|
||||
font-size: 14px;
|
||||
}
|
||||
.project-link {
|
||||
color: #e67e22;
|
||||
text-decoration: none;
|
||||
font-size: 12px;
|
||||
word-break: break-all;
|
||||
}
|
||||
.footer {
|
||||
margin-top: 30px;
|
||||
padding-top: 20px;
|
||||
border-top: 1px solid #e0e0e0;
|
||||
color: #999;
|
||||
font-size: 12px;
|
||||
text-align: center;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<h1>南京公共资源交易平台 - 交通水务招标公告报告</h1>
|
||||
|
||||
<div class="summary">
|
||||
<h2>报告摘要</h2>
|
||||
<div class="stat-grid">
|
||||
<div class="stat">
|
||||
<div class="stat-label">总公告数量</div>
|
||||
<div class="stat-value">${summary.total_count} 条</div>
|
||||
</div>
|
||||
<div class="stat">
|
||||
<div class="stat-label">有金额信息</div>
|
||||
<div class="stat-value">${summary.has_amount_count || summary.filtered_count} 条</div>
|
||||
</div>
|
||||
<div class="stat">
|
||||
<div class="stat-label">符合筛选</div>
|
||||
<div class="stat-value">${summary.filtered_count} 条</div>
|
||||
</div>
|
||||
<div class="stat">
|
||||
<div class="stat-label">金额阈值</div>
|
||||
<div class="stat-value">${summary.threshold || '无'}</div>
|
||||
</div>
|
||||
<div class="stat">
|
||||
<div class="stat-label">合同估算总额</div>
|
||||
<div class="stat-value">${summary.total_amount}</div>
|
||||
</div>
|
||||
</div>
|
||||
${summary.date_range ? `
|
||||
<div style="margin-top: 15px; padding-top: 15px; border-top: 1px solid rgba(255,255,255,0.2);">
|
||||
<div class="stat-label">时间范围</div>
|
||||
<div style="font-size: 14px; margin-top: 5px;">
|
||||
${summary.date_range.startDate || '不限'} 至 ${summary.date_range.endDate || '不限'}
|
||||
</div>
|
||||
</div>
|
||||
` : ''}
|
||||
</div>
|
||||
|
||||
<h2>招标项目详情</h2>
|
||||
<div class="project-list">
|
||||
${projects.length === 0 ? '<p style="color: #999; text-align: center; padding: 20px;">暂无符合条件的项目</p>' : ''}
|
||||
${projects.map((project, index) => `
|
||||
<div class="project-item">
|
||||
<h3>${index + 1}. ${project.title}</h3>
|
||||
<div class="project-meta">
|
||||
<strong>发布日期:</strong> ${project.date}
|
||||
${project.bidCode ? ` | <strong>标段编码:</strong> ${project.bidCode}` : ''}
|
||||
</div>
|
||||
${project.tenderee ? `
|
||||
<div class="project-meta">
|
||||
<strong>招标人:</strong> ${project.tenderee}
|
||||
${project.duration ? ` | <strong>工期:</strong> ${project.duration}日历天` : ''}
|
||||
</div>
|
||||
` : ''}
|
||||
${project.estimatedAmount ? `
|
||||
<div class="amount">
|
||||
合同估算价: ${project.estimatedAmount.amountWan} 万元
|
||||
</div>
|
||||
` : ''}
|
||||
<div style="margin-top: 10px;">
|
||||
<a href="${project.url}" class="project-link" target="_blank">${project.url}</a>
|
||||
</div>
|
||||
</div>
|
||||
`).join('')}
|
||||
</div>
|
||||
|
||||
<div class="footer">
|
||||
<p>报告生成时间: ${new Date(summary.generated_at).toLocaleString('zh-CN')}</p>
|
||||
<p>本报告由公告采集系统自动生成</p>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
`;
|
||||
}
|
||||
|
||||
// 发送综合报告邮件（中标+招标）
|
||||
export async function sendCombinedReportEmail(emailConfig, winningReport, bidReport) {
|
||||
try {
|
||||
const transporter = nodemailer.createTransport({
|
||||
host: emailConfig.smtpHost,
|
||||
port: emailConfig.smtpPort || 587,
|
||||
secure: emailConfig.smtpPort === 465,
|
||||
auth: {
|
||||
user: emailConfig.smtpUser,
|
||||
pass: emailConfig.smtpPass,
|
||||
},
|
||||
});
|
||||
|
||||
const htmlContent = generateCombinedReportHtml(winningReport, bidReport);
|
||||
|
||||
const info = await transporter.sendMail({
|
||||
from: `"公告采集系统" <${emailConfig.smtpUser}>`,
|
||||
to: emailConfig.recipients,
|
||||
subject: `交通水务综合报告 - ${new Date().toLocaleDateString('zh-CN')}`,
|
||||
html: htmlContent,
|
||||
});
|
||||
|
||||
return {
|
||||
success: true,
|
||||
messageId: info.messageId,
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('发送综合邮件失败:', error);
|
||||
throw new Error(`邮件发送失败: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// 生成综合报告HTML（中标+招标）
|
||||
function generateCombinedReportHtml(winningReport, bidReport) {
|
||||
return `
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>交通水务综合报告</title>
|
||||
<style>
|
||||
body {
|
||||
font-family: 'PingFang SC', 'Microsoft YaHei', Arial, sans-serif;
|
||||
line-height: 1.6;
|
||||
color: #333;
|
||||
max-width: 900px;
|
||||
margin: 0 auto;
|
||||
padding: 20px;
|
||||
background-color: #f5f5f5;
|
||||
}
|
||||
.container {
|
||||
background: white;
|
||||
border-radius: 8px;
|
||||
padding: 30px;
|
||||
box-shadow: 0 2px 10px rgba(0,0,0,0.1);
|
||||
}
|
||||
h1 {
|
||||
color: #333;
|
||||
border-bottom: 3px solid #667eea;
|
||||
padding-bottom: 10px;
|
||||
margin-bottom: 30px;
|
||||
text-align: center;
|
||||
}
|
||||
.section {
|
||||
margin-bottom: 40px;
|
||||
}
|
||||
.section-title {
|
||||
font-size: 20px;
|
||||
font-weight: bold;
|
||||
margin-bottom: 15px;
|
||||
padding: 10px 15px;
|
||||
border-radius: 6px;
|
||||
}
|
||||
.section-title.winning {
|
||||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||||
color: white;
|
||||
}
|
||||
.section-title.bid {
|
||||
background: linear-gradient(135deg, #e67e22 0%, #d35400 100%);
|
||||
color: white;
|
||||
}
|
||||
.summary {
|
||||
padding: 15px;
|
||||
border-radius: 8px;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
.summary.winning {
|
||||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||||
color: white;
|
||||
}
|
||||
.summary.bid {
|
||||
background: linear-gradient(135deg, #e67e22 0%, #d35400 100%);
|
||||
color: white;
|
||||
}
|
||||
.stat-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(150px, 1fr));
|
||||
gap: 15px;
|
||||
}
|
||||
.stat {
|
||||
background: rgba(255,255,255,0.15);
|
||||
padding: 10px;
|
||||
border-radius: 6px;
|
||||
}
|
||||
.stat-label {
|
||||
font-size: 12px;
|
||||
opacity: 0.9;
|
||||
margin-bottom: 5px;
|
||||
}
|
||||
.stat-value {
|
||||
font-size: 18px;
|
||||
font-weight: bold;
|
||||
}
|
||||
.project-list {
|
||||
margin-top: 15px;
|
||||
}
|
||||
.project-item {
|
||||
background: #f9f9f9;
|
||||
padding: 12px;
|
||||
margin-bottom: 10px;
|
||||
border-radius: 4px;
|
||||
}
|
||||
.project-item.winning {
|
||||
border-left: 4px solid #667eea;
|
||||
}
|
||||
.project-item.bid {
|
||||
border-left: 4px solid #e67e22;
|
||||
}
|
||||
.project-item h3 {
|
||||
color: #333;
|
||||
margin: 0 0 8px 0;
|
||||
font-size: 14px;
|
||||
line-height: 1.4;
|
||||
}
|
||||
.project-meta {
|
||||
color: #666;
|
||||
font-size: 12px;
|
||||
margin: 3px 0;
|
||||
}
|
||||
.amount {
|
||||
display: inline-block;
|
||||
color: white;
|
||||
padding: 3px 10px;
|
||||
border-radius: 4px;
|
||||
font-weight: bold;
|
||||
margin-top: 6px;
|
||||
font-size: 13px;
|
||||
}
|
||||
.amount.winning {
|
||||
background: #667eea;
|
||||
}
|
||||
.amount.bid {
|
||||
background: #e67e22;
|
||||
}
|
||||
.project-link {
|
||||
text-decoration: none;
|
||||
font-size: 11px;
|
||||
word-break: break-all;
|
||||
}
|
||||
.project-link.winning {
|
||||
color: #667eea;
|
||||
}
|
||||
.project-link.bid {
|
||||
color: #e67e22;
|
||||
}
|
||||
.footer {
|
||||
margin-top: 30px;
|
||||
padding-top: 20px;
|
||||
border-top: 1px solid #e0e0e0;
|
||||
color: #999;
|
||||
font-size: 12px;
|
||||
text-align: center;
|
||||
}
|
||||
.no-data {
|
||||
color: #999;
|
||||
text-align: center;
|
||||
padding: 20px;
|
||||
font-style: italic;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<h1>南京公共资源交易平台 - 交通水务综合报告</h1>
|
||||
|
||||
${bidReport ? `
|
||||
<!-- 招标公告部分 -->
|
||||
<div class="section">
|
||||
<div class="section-title bid">招标公告</div>
|
||||
<div class="summary bid">
|
||||
<div class="stat-grid">
|
||||
<div class="stat">
|
||||
<div class="stat-label">总公告数量</div>
|
||||
<div class="stat-value">${bidReport.summary.total_count} 条</div>
|
||||
</div>
|
||||
<div class="stat">
|
||||
<div class="stat-label">有金额信息</div>
|
||||
<div class="stat-value">${bidReport.summary.has_amount_count || bidReport.summary.filtered_count} 条</div>
|
||||
</div>
|
||||
<div class="stat">
|
||||
<div class="stat-label">符合筛选</div>
|
||||
<div class="stat-value">${bidReport.summary.filtered_count} 条</div>
|
||||
</div>
|
||||
<div class="stat">
|
||||
<div class="stat-label">金额阈值</div>
|
||||
<div class="stat-value">${bidReport.summary.threshold || '无'}</div>
|
||||
</div>
|
||||
<div class="stat">
|
||||
<div class="stat-label">合同估算总额</div>
|
||||
<div class="stat-value">${bidReport.summary.total_amount}</div>
|
||||
</div>
|
||||
</div>
|
||||
${bidReport.summary.date_range ? `
|
||||
<div style="margin-top: 10px; padding-top: 10px; border-top: 1px solid rgba(255,255,255,0.2); font-size: 13px;">
|
||||
时间范围: ${bidReport.summary.date_range.startDate || '不限'} 至 ${bidReport.summary.date_range.endDate || '不限'}
|
||||
</div>
|
||||
` : ''}
|
||||
</div>
|
||||
<div class="project-list">
|
||||
${bidReport.projects.length === 0 ? '<p class="no-data">暂无符合条件的招标项目</p>' : ''}
|
||||
${bidReport.projects.map((project, index) => `
|
||||
<div class="project-item bid">
|
||||
<h3>${index + 1}. ${project.title}</h3>
|
||||
<div class="project-meta"><strong>发布日期:</strong> ${project.date}</div>
|
||||
${project.bidCode ? `<div class="project-meta"><strong>标段编码:</strong> ${project.bidCode}</div>` : ''}
|
||||
${project.tenderee ? `<div class="project-meta"><strong>招标人:</strong> ${project.tenderee}</div>` : ''}
|
||||
${project.duration ? `<div class="project-meta"><strong>工期:</strong> ${project.duration}日历天</div>` : ''}
|
||||
${project.estimatedAmount ? `
|
||||
<div class="amount bid">合同估算价: ${project.estimatedAmount.amountWan} 万元</div>
|
||||
` : ''}
|
||||
<div style="margin-top: 8px;">
|
||||
<a href="${project.url}" class="project-link bid" target="_blank">${project.url}</a>
|
||||
</div>
|
||||
</div>
|
||||
`).join('')}
|
||||
</div>
|
||||
</div>
|
||||
` : ''}
|
||||
|
||||
${winningReport ? `
|
||||
<!-- 中标公示部分 -->
|
||||
<div class="section">
|
||||
<div class="section-title winning">中标结果公示</div>
|
||||
<div class="summary winning">
|
||||
<div class="stat-grid">
|
||||
<div class="stat">
|
||||
<div class="stat-label">总项目数</div>
|
||||
<div class="stat-value">${winningReport.summary.total_count} 条</div>
|
||||
</div>
|
||||
<div class="stat">
|
||||
<div class="stat-label">符合条件</div>
|
||||
<div class="stat-value">${winningReport.summary.filtered_count} 条</div>
|
||||
</div>
|
||||
<div class="stat">
|
||||
<div class="stat-label">金额阈值</div>
|
||||
<div class="stat-value">${winningReport.summary.threshold}</div>
|
||||
</div>
|
||||
<div class="stat">
|
||||
<div class="stat-label">总金额</div>
|
||||
<div class="stat-value">${winningReport.summary.total_amount}</div>
|
||||
</div>
|
||||
</div>
|
||||
${winningReport.summary.date_range ? `
|
||||
<div style="margin-top: 10px; padding-top: 10px; border-top: 1px solid rgba(255,255,255,0.2); font-size: 13px;">
|
||||
时间范围: ${winningReport.summary.date_range.startDate || '不限'} 至 ${winningReport.summary.date_range.endDate || '不限'}
|
||||
</div>
|
||||
` : ''}
|
||||
</div>
|
||||
<div class="project-list">
|
||||
${winningReport.projects.length === 0 ? '<p class="no-data">暂无符合条件的中标项目</p>' : ''}
|
||||
${winningReport.projects.map((project, index) => `
|
||||
<div class="project-item winning">
|
||||
<h3>${index + 1}. ${project.title}</h3>
|
||||
<div class="project-meta"><strong>中标日期:</strong> ${project.date}</div>
|
||||
${project.bidNo ? `<div class="project-meta"><strong>标段编号:</strong> ${project.bidNo}</div>` : ''}
|
||||
${project.winningBid ? `
|
||||
<div class="amount winning">中标金额: ${project.winningBid.amount.toFixed(2)} ${project.winningBid.unit}</div>
|
||||
` : ''}
|
||||
<div style="margin-top: 8px;">
|
||||
<a href="${project.url}" class="project-link winning" target="_blank">${project.url}</a>
|
||||
</div>
|
||||
</div>
|
||||
`).join('')}
|
||||
</div>
|
||||
</div>
|
||||
` : ''}
|
||||
|
||||
<div class="footer">
|
||||
<p>报告生成时间: ${new Date().toLocaleString('zh-CN')}</p>
|
||||
<p>本报告由公告采集系统自动生成</p>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
`;
|
||||
}
|
||||
|
||||
// 生成HTML格式的报告
|
||||
function generateReportHtml(report) {
|
||||
const { summary, projects } = report;
|
||||
@@ -45,7 +562,7 @@ function generateReportHtml(report) {
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>采购公告分析报告</title>
|
||||
<title>交通水务中标结果报告</title>
|
||||
<style>
|
||||
body {
|
||||
font-family: 'PingFang SC', 'Microsoft YaHei', Arial, sans-serif;
|
||||
@@ -147,7 +664,7 @@ function generateReportHtml(report) {
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<h1>南京公共工程建设中心 - 采购公告分析报告</h1>
|
||||
<h1>南京公共资源交易平台 - 交通水务中标结果报告</h1>
|
||||
|
||||
<div class="summary">
|
||||
<h2>报告摘要</h2>
|
||||
@@ -186,15 +703,17 @@ function generateReportHtml(report) {
|
||||
<div class="project-item">
|
||||
<h3>${index + 1}. ${project.title}</h3>
|
||||
<div class="project-meta">
|
||||
<strong>发布日期:</strong> ${project.date}
|
||||
${project.publish_time ? ` | <strong>发布时间:</strong> ${project.publish_time}` : ''}
|
||||
<strong>中标日期:</strong> ${project.date}
|
||||
</div>
|
||||
${project.budget ? `
|
||||
${project.winningBid ? `
|
||||
<div class="budget">
|
||||
中标金额: ${project.winningBid.amount.toFixed(2)} ${project.winningBid.unit}
|
||||
</div>
|
||||
` : (project.budget ? `
|
||||
<div class="budget">
|
||||
预算金额: ${project.budget.amount.toFixed(2)} ${project.budget.unit}
|
||||
${project.budget.originalUnit !== project.budget.unit ? ` (原始: ${project.budget.originalUnit})` : ''}
|
||||
</div>
|
||||
` : ''}
|
||||
` : '')}
|
||||
<div style="margin-top: 10px;">
|
||||
<a href="${project.url}" class="project-link" target="_blank">${project.url}</a>
|
||||
</div>
|
||||
@@ -211,3 +730,166 @@ function generateReportHtml(report) {
|
||||
</html>
|
||||
`;
|
||||
}
|
||||
|
||||
// ========== 通用抓取结果邮件（定时任务使用） ==========
|
||||
|
||||
export async function sendScraperResultsEmail(emailConfig, results) {
|
||||
try {
|
||||
const transporter = nodemailer.createTransport({
|
||||
host: emailConfig.smtpHost,
|
||||
port: emailConfig.smtpPort || 587,
|
||||
secure: emailConfig.smtpPort === 465,
|
||||
auth: {
|
||||
user: emailConfig.smtpUser,
|
||||
pass: emailConfig.smtpPass,
|
||||
},
|
||||
});
|
||||
|
||||
const htmlContent = generateScraperResultsHtml(results);
|
||||
const successCount = results.filter(r => !r.error).length;
|
||||
|
||||
const info = await transporter.sendMail({
|
||||
from: `"公告采集系统" <${emailConfig.smtpUser}>`,
|
||||
to: emailConfig.recipients,
|
||||
subject: `公告采集结果报告(${successCount}条) - ${new Date().toLocaleDateString('zh-CN')}`,
|
||||
html: htmlContent,
|
||||
});
|
||||
|
||||
return { success: true, messageId: info.messageId };
|
||||
} catch (error) {
|
||||
console.error('发送抓取结果邮件失败:', error);
|
||||
throw new Error(`邮件发送失败: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
function generateScraperResultsHtml(results) {
|
||||
const successResults = results.filter(r => !r.error);
|
||||
const failResults = results.filter(r => r.error);
|
||||
const generatedAt = new Date().toLocaleString('zh-CN');
|
||||
|
||||
// 把所有成功来源的 items 展开，附带来源信息
|
||||
const allRows = [];
|
||||
for (const r of successResults) {
|
||||
const items = r.data?.result || [];
|
||||
for (const item of items) {
|
||||
allRows.push({
|
||||
section: [r.section, r.subsection].filter(Boolean).join(' · ') || r.city || '-',
|
||||
type: r.type || '-',
|
||||
title: item.title || '-',
|
||||
date: item.date || '-',
|
||||
amount: item.amount || '未公开',
|
||||
url: item.url || '',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// 按日期降序排列
|
||||
allRows.sort((a, b) => {
|
||||
if (a.date === b.date) return 0;
|
||||
return a.date > b.date ? -1 : 1;
|
||||
});
|
||||
|
||||
const totalItems = allRows.length;
|
||||
|
||||
// 行颜色交替
|
||||
const rowHtml = allRows.length === 0
|
||||
? `<tr><td colspan="6" style="text-align:center;color:#999;padding:30px;font-size:14px;">暂无数据</td></tr>`
|
||||
: allRows.map((row, i) => `
|
||||
<tr style="background:${i % 2 === 0 ? '#fff' : '#f7f8ff'};">
|
||||
<td style="padding:9px 12px;border-bottom:1px solid #eaecf5;white-space:nowrap;color:#555;font-size:13px;">${row.section}</td>
|
||||
<td style="padding:9px 12px;border-bottom:1px solid #eaecf5;white-space:nowrap;">
|
||||
<span style="display:inline-block;padding:2px 8px;background:#e8f4fd;color:#1a73c8;border-radius:10px;font-size:11px;font-weight:600;">${row.type}</span>
|
||||
</td>
|
||||
<td style="padding:9px 12px;border-bottom:1px solid #eaecf5;font-size:13px;max-width:320px;">${row.title}</td>
|
||||
<td style="padding:9px 12px;border-bottom:1px solid #eaecf5;white-space:nowrap;font-size:13px;color:#555;">${row.date}</td>
|
||||
<td style="padding:9px 12px;border-bottom:1px solid #eaecf5;white-space:nowrap;font-size:13px;font-weight:600;color:${row.amount === '未公开' ? '#aaa' : '#e67e22'};">${row.amount}</td>
|
||||
<td style="padding:9px 12px;border-bottom:1px solid #eaecf5;text-align:center;">
|
||||
${row.url
|
||||
? `<a href="${row.url}" target="_blank" style="color:#667eea;font-size:12px;text-decoration:none;white-space:nowrap;">查看 →</a>`
|
||||
: '<span style="color:#ccc;font-size:12px;">-</span>'
|
||||
}
|
||||
</td>
|
||||
</tr>`).join('');
|
||||
|
||||
// 失败来源列表
|
||||
const failHtml = failResults.length === 0 ? '' : `
|
||||
<div style="margin-top:24px;">
|
||||
<div style="font-size:14px;font-weight:600;color:#c0392b;margin-bottom:10px;">⚠️ 抓取失败的来源（${failResults.length} 个）</div>
|
||||
${failResults.map(r => `
|
||||
<div style="background:#fdeaea;border-left:3px solid #e74c3c;padding:10px 14px;border-radius:4px;margin-bottom:8px;font-size:13px;">
|
||||
<strong>${r.city || ''}${r.section ? ' · ' + r.section : ''}${r.type ? ' · ' + r.type : ''}</strong>
|
||||
<div style="color:#999;font-size:12px;margin-top:4px;">${r.url}</div>
|
||||
<div style="color:#c0392b;margin-top:4px;">❌ ${r.error}</div>
|
||||
</div>`).join('')}
|
||||
</div>`;
|
||||
|
||||
return `
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>公告采集结果报告</title>
|
||||
</head>
|
||||
<body style="font-family:'PingFang SC','Microsoft YaHei',Arial,sans-serif;line-height:1.6;color:#333;margin:0;padding:20px;background:#f0f2f8;">
|
||||
<div style="max-width:960px;margin:0 auto;background:white;border-radius:10px;overflow:hidden;box-shadow:0 4px 20px rgba(0,0,0,.1);">
|
||||
|
||||
<!-- 标题栏 -->
|
||||
<div style="background:linear-gradient(135deg,#667eea 0%,#764ba2 100%);padding:24px 30px;color:white;">
|
||||
<h1 style="margin:0;font-size:20px;font-weight:700;">📋 公告采集结果报告</h1>
|
||||
<div style="margin-top:6px;opacity:.85;font-size:13px;">生成时间：${generatedAt}</div>
|
||||
</div>
|
||||
|
||||
<!-- 统计栏 -->
|
||||
<div style="display:flex;gap:0;border-bottom:1px solid #eaecf5;">
|
||||
<div style="flex:1;padding:16px 24px;text-align:center;border-right:1px solid #eaecf5;">
|
||||
<div style="font-size:28px;font-weight:700;color:#667eea;">${totalItems}</div>
|
||||
<div style="font-size:12px;color:#888;margin-top:2px;">公告总数</div>
|
||||
</div>
|
||||
<div style="flex:1;padding:16px 24px;text-align:center;border-right:1px solid #eaecf5;">
|
||||
<div style="font-size:28px;font-weight:700;color:#1a8a4a;">${successResults.length}</div>
|
||||
<div style="font-size:12px;color:#888;margin-top:2px;">成功来源</div>
|
||||
</div>
|
||||
<div style="flex:1;padding:16px 24px;text-align:center;border-right:1px solid #eaecf5;">
|
||||
<div style="font-size:28px;font-weight:700;color:#e67e22;">${allRows.filter(r => r.amount && r.amount !== '未公开').length}</div>
|
||||
<div style="font-size:12px;color:#888;margin-top:2px;">有金额</div>
|
||||
</div>
|
||||
<div style="flex:1;padding:16px 24px;text-align:center;">
|
||||
<div style="font-size:28px;font-weight:700;color:${failResults.length > 0 ? '#c0392b' : '#aaa'};">${failResults.length}</div>
|
||||
<div style="font-size:12px;color:#888;margin-top:2px;">失败来源</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 公告汇总表格 -->
|
||||
<div style="padding:24px 30px;">
|
||||
<div style="font-size:15px;font-weight:600;color:#333;margin-bottom:14px;">公告汇总（共 ${totalItems} 条）</div>
|
||||
<div style="overflow-x:auto;">
|
||||
<table style="width:100%;border-collapse:collapse;font-size:13px;">
|
||||
<thead>
|
||||
<tr style="background:linear-gradient(135deg,#667eea 0%,#764ba2 100%);color:white;">
|
||||
<th style="padding:10px 12px;text-align:left;font-weight:600;white-space:nowrap;">板块</th>
|
||||
<th style="padding:10px 12px;text-align:left;font-weight:600;white-space:nowrap;">类型</th>
|
||||
<th style="padding:10px 12px;text-align:left;font-weight:600;">公告标题</th>
|
||||
<th style="padding:10px 12px;text-align:left;font-weight:600;white-space:nowrap;">发布日期</th>
|
||||
<th style="padding:10px 12px;text-align:left;font-weight:600;white-space:nowrap;">项目金额</th>
|
||||
<th style="padding:10px 12px;text-align:center;font-weight:600;white-space:nowrap;">详情</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
${rowHtml}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
${failHtml}
|
||||
|
||||
<div style="margin-top:24px;padding-top:16px;border-top:1px solid #eaecf5;color:#aaa;font-size:12px;text-align:center;">
|
||||
本报告由公告采集系统自动生成 · ${generatedAt}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
`;
|
||||
}
|
||||
|
||||
275
src/firecrawlBrowserScraper.js
Normal file
275
src/firecrawlBrowserScraper.js
Normal file
@@ -0,0 +1,275 @@
|
||||
const DEFAULT_SCRAPER_PROMPT = '提取页面上今日的招标公告信息，包括：标题、项目金额（可能为合同预估价/最高投标限价等）、发布日期（YYYY-MM-DD格式）、详情页完整URL';
|
||||
const PAYLOAD_MARKER = '__FC_PAYLOAD__';
|
||||
|
||||
function pad2(value) {
|
||||
return String(value).padStart(2, '0');
|
||||
}
|
||||
|
||||
function formatDate(year, month, day) {
|
||||
return `${year}-${pad2(month)}-${pad2(day)}`;
|
||||
}
|
||||
|
||||
function getTodayInShanghai() {
|
||||
return new Intl.DateTimeFormat('en-CA', {
|
||||
timeZone: 'Asia/Shanghai',
|
||||
year: 'numeric',
|
||||
month: '2-digit',
|
||||
day: '2-digit',
|
||||
}).format(new Date());
|
||||
}
|
||||
|
||||
function parseTargetDate(prompt) {
|
||||
const text = String(prompt || '');
|
||||
if (!text) return null;
|
||||
|
||||
const fullDate = text.match(/(20\d{2})[-/.年](\d{1,2})[-/.月](\d{1,2})日?/);
|
||||
if (fullDate) {
|
||||
return formatDate(fullDate[1], fullDate[2], fullDate[3]);
|
||||
}
|
||||
|
||||
if (/(今天|今日|当日)/.test(text)) {
|
||||
return getTodayInShanghai();
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function normalizeDate(input) {
|
||||
if (!input) return '';
|
||||
const text = String(input).trim();
|
||||
if (!text) return '';
|
||||
|
||||
let m = text.match(/(20\d{2})[-/.年](\d{1,2})[-/.月](\d{1,2})日?/);
|
||||
if (m) return formatDate(m[1], m[2], m[3]);
|
||||
|
||||
m = text.match(/(\d{1,2})[-/.月](\d{1,2})日?/);
|
||||
if (m) {
|
||||
const currentYear = Number(getTodayInShanghai().slice(0, 4));
|
||||
return formatDate(currentYear, m[1], m[2]);
|
||||
}
|
||||
|
||||
return '';
|
||||
}
|
||||
|
||||
function extractDateFromText(text) {
|
||||
if (!text) return '';
|
||||
const m = String(text).match(/(20\d{2}[-/.年]\d{1,2}[-/.月]\d{1,2}日?)|(\d{1,2}[-/.月]\d{1,2}日?)/);
|
||||
return m ? normalizeDate(m[0]) : '';
|
||||
}
|
||||
|
||||
function extractAmountFromText(text) {
|
||||
if (!text) return null;
|
||||
const m = String(text).match(/([0-9][0-9,.\s]*(?:亿元|万元|万|元))/);
|
||||
if (!m) return null;
|
||||
return m[1].replace(/\s+/g, '').trim();
|
||||
}
|
||||
|
||||
function cleanText(text) {
|
||||
return String(text || '').replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
|
||||
function toFiniteNumber(value, fallback) {
|
||||
const n = Number(value);
|
||||
return Number.isFinite(n) ? n : fallback;
|
||||
}
|
||||
|
||||
function parsePayloadFromText(rawText) {
|
||||
if (!rawText) return null;
|
||||
const text = String(rawText);
|
||||
|
||||
const markerIndex = text.lastIndexOf(PAYLOAD_MARKER);
|
||||
if (markerIndex >= 0) {
|
||||
const tail = text.slice(markerIndex + PAYLOAD_MARKER.length);
|
||||
const firstLine = tail.split(/\r?\n/).find(line => line.trim());
|
||||
if (firstLine) {
|
||||
try {
|
||||
return JSON.parse(firstLine.trim());
|
||||
} catch {
|
||||
// Continue fallback parsing.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
return JSON.parse(text.trim());
|
||||
} catch {
|
||||
// Continue fallback parsing.
|
||||
}
|
||||
|
||||
const lines = text.split(/\r?\n/).map(line => line.trim()).filter(Boolean).reverse();
|
||||
for (const line of lines) {
|
||||
try {
|
||||
return JSON.parse(line);
|
||||
} catch {
|
||||
// Try next line.
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function parseBrowserExecutePayload(executeResult) {
|
||||
const sources = [executeResult?.result, executeResult?.stdout]
|
||||
.filter(value => typeof value === 'string' && value.trim().length > 0);
|
||||
|
||||
for (const source of sources) {
|
||||
const payload = parsePayloadFromText(source);
|
||||
if (payload && typeof payload === 'object') return payload;
|
||||
}
|
||||
|
||||
return { items: [] };
|
||||
}
|
||||
|
||||
function splitKeywords(input) {
|
||||
return String(input || '')
|
||||
.split(/[、/，,|\s]+/)
|
||||
.map(item => item.trim())
|
||||
.filter(item => item.length >= 2);
|
||||
}
|
||||
|
||||
function filterByTypeIfPossible(items, type) {
|
||||
const keywords = splitKeywords(type);
|
||||
if (keywords.length === 0) return items;
|
||||
|
||||
const filtered = items.filter(item => {
|
||||
const haystack = `${item.title} ${item.context || ''}`;
|
||||
return keywords.some(keyword => haystack.includes(keyword));
|
||||
});
|
||||
|
||||
return filtered.length > 0 ? filtered : items;
|
||||
}
|
||||
|
||||
function normalizeItems(rawItems, targetDate, scraperType) {
|
||||
const dedup = new Map();
|
||||
|
||||
for (const raw of rawItems) {
|
||||
const title = cleanText(raw?.title);
|
||||
const url = cleanText(raw?.url);
|
||||
if (!title || !url) continue;
|
||||
|
||||
const context = cleanText(raw?.context);
|
||||
const date = normalizeDate(raw?.date) || extractDateFromText(context);
|
||||
const amount = cleanText(raw?.amount) || extractAmountFromText(context) || null;
|
||||
const key = `${title}@@${url}`;
|
||||
|
||||
if (!dedup.has(key)) {
|
||||
dedup.set(key, { title, amount, date, url, context });
|
||||
}
|
||||
}
|
||||
|
||||
let items = Array.from(dedup.values());
|
||||
items = filterByTypeIfPossible(items, scraperType);
|
||||
|
||||
if (targetDate) {
|
||||
items = items.filter(item => item.date === targetDate);
|
||||
}
|
||||
|
||||
return items
|
||||
.map(({ title, amount, date, url }) => ({ title, amount, date, url }))
|
||||
.slice(0, 100);
|
||||
}
|
||||
|
||||
function buildBrowserScript(url) {
|
||||
return `
|
||||
const targetUrl = ${JSON.stringify(url)};
|
||||
await page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 60000 });
|
||||
await page.waitForTimeout(1500);
|
||||
|
||||
const payload = await page.evaluate(() => {
|
||||
const normalize = (value) => String(value || '').replace(/\\s+/g, ' ').trim();
|
||||
const blockedTitles = new Set(['首页', '尾页', '上一页', '下一页', '更多', '详情', '查看', '返回', '跳转']);
|
||||
|
||||
const links = Array.from(document.querySelectorAll('a[href]'));
|
||||
const rows = [];
|
||||
const seen = new Set();
|
||||
|
||||
for (const a of links) {
|
||||
const href = a.getAttribute('href') || '';
|
||||
if (!href || href.startsWith('javascript:') || href.startsWith('#')) continue;
|
||||
|
||||
const title = normalize(a.textContent);
|
||||
if (!title || title.length < 6 || title.length > 180) continue;
|
||||
if (blockedTitles.has(title)) continue;
|
||||
|
||||
let absoluteUrl = '';
|
||||
try {
|
||||
absoluteUrl = new URL(href, location.href).href;
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
|
||||
const container = a.closest('tr,li,article,section,div,p,dd,dt') || a.parentElement;
|
||||
const context = normalize(container ? container.textContent : title);
|
||||
|
||||
const dateMatch = context.match(/(20\\d{2}[-/.年]\\d{1,2}[-/.月]\\d{1,2}日?)|(\\d{1,2}[-/.月]\\d{1,2}日?)/);
|
||||
const amountMatch = context.match(/([0-9][0-9,.\\s]*(?:亿元|万元|万|元))/);
|
||||
|
||||
const key = (title + '@@' + absoluteUrl).toLowerCase();
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
|
||||
rows.push({
|
||||
title,
|
||||
url: absoluteUrl,
|
||||
date: dateMatch ? dateMatch[0] : '',
|
||||
amount: amountMatch ? amountMatch[0].replace(/\\s+/g, '') : null,
|
||||
context,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
pageUrl: location.href,
|
||||
items: rows.slice(0, 300),
|
||||
};
|
||||
});
|
||||
|
||||
console.log('${PAYLOAD_MARKER}' + JSON.stringify(payload));
|
||||
JSON.stringify(payload);
|
||||
`;
|
||||
}
|
||||
|
||||
export async function runScraperWithBrowser(firecrawl, scraper, options = {}) {
|
||||
const prefix = options.logPrefix || '[Browser]';
|
||||
if (!scraper?.url) throw new Error('抓取 URL 不能为空');
|
||||
|
||||
const prompt = scraper.prompt || DEFAULT_SCRAPER_PROMPT;
|
||||
const targetDate = parseTargetDate(prompt);
|
||||
|
||||
const ttl = toFiniteNumber(scraper.browserTtl, 180);
|
||||
const activityTtl = toFiniteNumber(scraper.browserActivityTtl, 90);
|
||||
|
||||
const session = await firecrawl.browser({ ttl, activityTtl });
|
||||
if (!session?.success || !session.id) {
|
||||
throw new Error(session?.error || '创建 Browser 会话失败');
|
||||
}
|
||||
|
||||
let executeResult;
|
||||
try {
|
||||
executeResult = await firecrawl.browserExecute(session.id, {
|
||||
code: buildBrowserScript(scraper.url),
|
||||
language: 'node',
|
||||
});
|
||||
} finally {
|
||||
try {
|
||||
await firecrawl.deleteBrowser(session.id);
|
||||
} catch (closeError) {
|
||||
console.warn(`${prefix} 会话关闭失败: ${closeError.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (!executeResult?.success) {
|
||||
throw new Error(executeResult?.error || executeResult?.stderr || 'Browser 执行失败');
|
||||
}
|
||||
|
||||
const payload = parseBrowserExecutePayload(executeResult);
|
||||
const rawItems = Array.isArray(payload.items) ? payload.items : [];
|
||||
const items = normalizeItems(rawItems, targetDate, scraper.type);
|
||||
|
||||
console.log(`${prefix} URL=${scraper.url} raw=${rawItems.length} normalized=${items.length}${targetDate ? ` targetDate=${targetDate}` : ''}`);
|
||||
|
||||
return {
|
||||
items,
|
||||
targetDate,
|
||||
pageUrl: payload.pageUrl || scraper.url,
|
||||
};
|
||||
}
|
||||
416
src/scheduler.js
416
src/scheduler.js
@@ -1,216 +1,76 @@
|
||||
import 'dotenv/config';
|
||||
import cron from 'node-cron';
|
||||
import { readFileSync } from 'fs';
|
||||
import { readFileSync, writeFileSync, existsSync } from 'fs';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { dirname, join } from 'path';
|
||||
import axios from 'axios';
|
||||
import * as cheerio from 'cheerio';
|
||||
import iconv from 'iconv-lite';
|
||||
import { sendReportEmail } from './emailService.js';
|
||||
import Firecrawl from '@mendable/firecrawl-js';
|
||||
import { sendScraperResultsEmail } from './emailService.js';
|
||||
import { runScraperWithBrowser } from './firecrawlBrowserScraper.js';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
|
||||
// 初始化 Firecrawl 客户端
|
||||
const firecrawl = new Firecrawl({ apiKey: process.env.FIRECRAWL_API_KEY });
|
||||
|
||||
const RESULTS_PATH = join(__dirname, '..', 'results.json');
|
||||
|
||||
// 加载配置文件
|
||||
function loadConfig() {
|
||||
try {
|
||||
const configPath = join(__dirname, '..', 'config.json');
|
||||
const configContent = readFileSync(configPath, 'utf-8');
|
||||
return JSON.parse(configContent);
|
||||
return JSON.parse(readFileSync(configPath, 'utf-8'));
|
||||
} catch (error) {
|
||||
console.error('加载配置文件失败:', error.message);
|
||||
console.error('请确保 config.json 文件存在并配置正确');
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// 根据时间范围类型获取开始和结束日期
|
||||
function getDateRangeByType(timeRange) {
|
||||
const now = new Date();
|
||||
const year = now.getFullYear();
|
||||
const month = String(now.getMonth() + 1).padStart(2, '0');
|
||||
const day = String(now.getDate()).padStart(2, '0');
|
||||
// ========== 结果存取（与 server.js 保持一致） ==========
|
||||
|
||||
let startDate, endDate;
|
||||
endDate = `${year}-${month}-${day}`; // 结束日期都是今天
|
||||
|
||||
switch (timeRange) {
|
||||
case 'today':
|
||||
// 今日
|
||||
startDate = `${year}-${month}-${day}`;
|
||||
break;
|
||||
|
||||
case 'thisWeek': {
|
||||
// 本周 (从周一开始)
|
||||
const dayOfWeek = now.getDay(); // 0是周日,1是周一
|
||||
const diff = dayOfWeek === 0 ? 6 : dayOfWeek - 1; // 计算到周一的天数差
|
||||
const monday = new Date(now);
|
||||
monday.setDate(now.getDate() - diff);
|
||||
const weekYear = monday.getFullYear();
|
||||
const weekMonth = String(monday.getMonth() + 1).padStart(2, '0');
|
||||
const weekDay = String(monday.getDate()).padStart(2, '0');
|
||||
startDate = `${weekYear}-${weekMonth}-${weekDay}`;
|
||||
break;
|
||||
}
|
||||
|
||||
case 'thisMonth':
|
||||
default:
|
||||
// 本月
|
||||
startDate = `${year}-${month}-01`;
|
||||
break;
|
||||
function readResults() {
|
||||
if (!existsSync(RESULTS_PATH)) return [];
|
||||
try {
|
||||
return JSON.parse(readFileSync(RESULTS_PATH, 'utf-8'));
|
||||
} catch (e) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return { startDate, endDate };
|
||||
}
|
||||
|
||||
// 南京市公共资源交易平台 - 房建市政招标公告
|
||||
const BASE_URL = 'https://njggzy.nanjing.gov.cn/njweb/fjsz/068001/068001002/';
|
||||
|
||||
const http = axios.create({
|
||||
responseType: 'arraybuffer',
|
||||
timeout: 15000,
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
},
|
||||
});
|
||||
|
||||
function pickEncoding(contentType = '') {
|
||||
const match = /charset=([^;]+)/i.exec(contentType);
|
||||
if (!match) return 'utf-8';
|
||||
const charset = match[1].trim().toLowerCase();
|
||||
if (charset.includes('gb')) return 'gbk';
|
||||
return charset;
|
||||
function saveResults(results) {
|
||||
writeFileSync(RESULTS_PATH, JSON.stringify(results, null, 2), 'utf-8');
|
||||
}
|
||||
|
||||
async function fetchHtml(url) {
|
||||
const res = await http.get(url);
|
||||
const encoding = pickEncoding(res.headers['content-type']);
|
||||
const html = iconv.decode(res.data, encoding || 'utf-8');
|
||||
return html;
|
||||
function appendResult(result) {
|
||||
const results = readResults();
|
||||
results.unshift({ ...result, id: `result-${Date.now()}-${Math.random().toString(36).slice(2, 7)}` });
|
||||
if (results.length > 500) results.splice(500);
|
||||
saveResults(results);
|
||||
}
|
||||
|
||||
function getPageUrl(pageIndex) {
|
||||
if (pageIndex === 1) {
|
||||
return `${BASE_URL}moreinfo.html`;
|
||||
}
|
||||
return `${BASE_URL}${pageIndex}.html`;
|
||||
// ========== 抓取执行（复用 server.js 中 runScraper 的逻辑） ==========
|
||||
|
||||
async function runScraper(scraper) {
|
||||
console.log(`[定时任务][Browser] ${scraper.city} - ${scraper.section} ${scraper.subsection} - ${scraper.type}：${scraper.url}`);
|
||||
const { items } = await runScraperWithBrowser(firecrawl, scraper, { logPrefix: '[Browser][Scheduler]' });
|
||||
console.log(`[定时任务][Browser] 提取到 ${items.length} 条公告`);
|
||||
|
||||
const record = {
|
||||
scraperId: scraper.id,
|
||||
city: scraper.city,
|
||||
section: scraper.section,
|
||||
subsection: scraper.subsection,
|
||||
type: scraper.type,
|
||||
url: scraper.url,
|
||||
scrapedAt: new Date().toISOString(),
|
||||
data: { result: items, total: items.length },
|
||||
};
|
||||
appendResult(record);
|
||||
return record;
|
||||
}
|
||||
|
||||
// 解析列表页HTML，提取招标信息
|
||||
function parseList(html) {
|
||||
const $ = cheerio.load(html);
|
||||
const items = [];
|
||||
// ========== 定时任务执行函数 ==========
|
||||
|
||||
$('li.ewb-info-item2').each((_, row) => {
|
||||
const $row = $(row);
|
||||
const cells = $row.find('div.ewb-info-num2');
|
||||
|
||||
if (cells.length >= 5) {
|
||||
const bidNo = $(cells[0]).find('p').attr('title') || $(cells[0]).find('p').text().trim();
|
||||
const projectName = $(cells[1]).find('p').attr('title') || $(cells[1]).find('p').text().trim();
|
||||
const bidName = $(cells[2]).find('p').attr('title') || $(cells[2]).find('p').text().trim();
|
||||
const estimatedPrice = $(cells[3]).find('p').text().trim();
|
||||
const publishDate = $(cells[4]).find('p').text().trim();
|
||||
|
||||
const onclick = $row.attr('onclick') || '';
|
||||
const hrefMatch = onclick.match(/window\.open\(['"]([^'"]+)['"]\)/);
|
||||
let href = '';
|
||||
if (hrefMatch) {
|
||||
href = hrefMatch[1];
|
||||
if (href.startsWith('/')) {
|
||||
href = `https://njggzy.nanjing.gov.cn${href}`;
|
||||
}
|
||||
}
|
||||
|
||||
if (!/^\d{4}-\d{2}-\d{2}$/.test(publishDate)) return;
|
||||
|
||||
const price = parseFloat(estimatedPrice);
|
||||
if (isNaN(price)) return;
|
||||
|
||||
items.push({
|
||||
bidNo,
|
||||
title: projectName,
|
||||
bidName,
|
||||
budget: {
|
||||
amount: price,
|
||||
unit: '万元'
|
||||
},
|
||||
date: publishDate,
|
||||
href
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
function isDateInRange(dateStr, startDate, endDate) {
|
||||
if (!dateStr) return false;
|
||||
const date = new Date(dateStr);
|
||||
if (isNaN(date.getTime())) return false;
|
||||
|
||||
if (startDate && date < new Date(startDate)) return false;
|
||||
if (endDate && date > new Date(endDate)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
async function fetchListByDateRange(startDate, endDate, maxPages = 50) {
|
||||
const allItems = [];
|
||||
let shouldContinue = true;
|
||||
let pageIndex = 1;
|
||||
|
||||
console.log(`开始按时间范围采集: ${startDate || '不限'} 至 ${endDate || '不限'}`);
|
||||
|
||||
while (shouldContinue && pageIndex <= maxPages) {
|
||||
const pageUrl = getPageUrl(pageIndex);
|
||||
console.log(`正在采集第 ${pageIndex} 页: ${pageUrl}`);
|
||||
|
||||
try {
|
||||
const html = await fetchHtml(pageUrl);
|
||||
const items = parseList(html);
|
||||
|
||||
if (items.length === 0) {
|
||||
console.log(`第 ${pageIndex} 页没有数据，停止采集`);
|
||||
break;
|
||||
}
|
||||
|
||||
let hasItemsInRange = false;
|
||||
let allItemsBeforeRange = true;
|
||||
|
||||
for (const item of items) {
|
||||
if (isDateInRange(item.date, startDate, endDate)) {
|
||||
allItems.push(item);
|
||||
hasItemsInRange = true;
|
||||
allItemsBeforeRange = false;
|
||||
} else if (startDate && new Date(item.date) < new Date(startDate)) {
|
||||
allItemsBeforeRange = allItemsBeforeRange && true;
|
||||
} else {
|
||||
allItemsBeforeRange = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (allItemsBeforeRange && startDate) {
|
||||
console.log(`第 ${pageIndex} 页所有项目都早于起始日期，停止采集`);
|
||||
shouldContinue = false;
|
||||
}
|
||||
|
||||
console.log(`第 ${pageIndex} 页找到 ${items.length} 条，符合条件 ${hasItemsInRange ? '有' : '无'}`);
|
||||
|
||||
pageIndex++;
|
||||
|
||||
if (shouldContinue && pageIndex <= maxPages) {
|
||||
await new Promise(resolve => setTimeout(resolve, 500));
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`采集第 ${pageIndex} 页失败: ${err.message}`);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`总共采集了 ${pageIndex - 1} 页，找到 ${allItems.length} 条符合条件的公告`);
|
||||
return allItems;
|
||||
}
|
||||
|
||||
// 定时任务执行函数
|
||||
async function executeScheduledTask(config) {
|
||||
try {
|
||||
console.log('========================================');
|
||||
@@ -218,78 +78,66 @@ async function executeScheduledTask(config) {
|
||||
console.log('执行时间:', new Date().toLocaleString('zh-CN'));
|
||||
console.log('========================================');
|
||||
|
||||
const timeRange = config.scheduler.timeRange || 'thisMonth';
|
||||
const { startDate, endDate } = getDateRangeByType(timeRange);
|
||||
const threshold = config.scheduler.threshold || 10000; // 默认1亿(10000万元)
|
||||
// 获取所有已启用的抓取来源
|
||||
const scrapers = (config.scrapers || []).filter(s => s.enabled);
|
||||
|
||||
const timeRangeNames = {
|
||||
'today': '今日',
|
||||
'thisWeek': '本周',
|
||||
'thisMonth': '本月'
|
||||
};
|
||||
console.log(`采集时间段: ${timeRangeNames[timeRange] || '本月'}`);
|
||||
console.log(`采集时间范围: ${startDate} 至 ${endDate}`);
|
||||
console.log(`金额阈值: ${threshold}万元 (${(threshold / 10000).toFixed(2)}亿元)`);
|
||||
|
||||
// 采集列表（直接包含合同估算价）
|
||||
const items = await fetchListByDateRange(startDate, endDate, 50);
|
||||
|
||||
if (items.length === 0) {
|
||||
console.log('暂无公告数据');
|
||||
if (scrapers.length === 0) {
|
||||
console.log('没有已启用的抓取来源，跳过');
|
||||
return;
|
||||
}
|
||||
|
||||
// 筛选大于阈值的项目
|
||||
const filtered = items.filter((item) => {
|
||||
return item.budget && item.budget.amount > threshold;
|
||||
});
|
||||
console.log(`共 ${scrapers.length} 个已启用的抓取来源`);
|
||||
|
||||
console.log('========================================');
|
||||
console.log(`筛选结果: 找到 ${filtered.length} 个大于 ${threshold}万元 的项目`);
|
||||
// 逐个运行抓取任务
|
||||
const results = [];
|
||||
for (const scraper of scrapers) {
|
||||
try {
|
||||
console.log(`\n---------- 抓取: ${scraper.city} - ${scraper.section} ${scraper.type} ----------`);
|
||||
const r = await runScraper(scraper);
|
||||
results.push(r);
|
||||
console.log(`✓ 抓取成功`);
|
||||
} catch (err) {
|
||||
console.error(`✗ 抓取失败: ${err.message}`);
|
||||
const errRecord = {
|
||||
scraperId: scraper.id,
|
||||
city: scraper.city,
|
||||
section: scraper.section,
|
||||
subsection: scraper.subsection,
|
||||
type: scraper.type,
|
||||
url: scraper.url,
|
||||
scrapedAt: new Date().toISOString(),
|
||||
error: err.message,
|
||||
data: null,
|
||||
};
|
||||
appendResult(errRecord);
|
||||
results.push(errRecord);
|
||||
}
|
||||
}
|
||||
|
||||
if (filtered.length === 0) {
|
||||
console.log('暂无符合条件的大额项目');
|
||||
const successCount = results.filter(r => !r.error).length;
|
||||
const failCount = results.filter(r => r.error).length;
|
||||
console.log(`\n========== 抓取完成 ==========`);
|
||||
console.log(`成功: ${successCount} 条，失败: ${failCount} 条`);
|
||||
|
||||
// 检查是否需要发送邮件
|
||||
if (successCount === 0) {
|
||||
console.log('没有成功的抓取结果，不发送邮件');
|
||||
return;
|
||||
}
|
||||
|
||||
// 计算总金额
|
||||
const total = filtered.reduce(
|
||||
(sum, item) => sum + (item.budget?.amount || 0),
|
||||
0
|
||||
);
|
||||
// 发送邮件报告
|
||||
if (config.email?.smtpHost && config.email?.smtpUser) {
|
||||
console.log('\n正在发送抓取结果邮件...');
|
||||
try {
|
||||
const emailResult = await sendScraperResultsEmail(config.email, results);
|
||||
console.log('邮件发送成功! MessageId:', emailResult.messageId);
|
||||
} catch (emailErr) {
|
||||
console.error('邮件发送失败:', emailErr.message);
|
||||
}
|
||||
} else {
|
||||
console.log('邮件配置不完整，跳过邮件发送');
|
||||
}
|
||||
|
||||
// 生成报告
|
||||
const report = {
|
||||
summary: {
|
||||
total_count: items.length,
|
||||
filtered_count: filtered.length,
|
||||
threshold: `${threshold}万元`,
|
||||
total_amount: `${total.toFixed(2)}万元`,
|
||||
generated_at: new Date().toISOString(),
|
||||
date_range: { startDate, endDate },
|
||||
},
|
||||
projects: filtered.map((item) => ({
|
||||
bidNo: item.bidNo,
|
||||
title: item.title,
|
||||
bidName: item.bidName,
|
||||
date: item.date,
|
||||
budget: item.budget,
|
||||
url: item.href,
|
||||
})),
|
||||
};
|
||||
|
||||
// 发送邮件
|
||||
console.log('========================================');
|
||||
console.log('正在发送邮件报告...');
|
||||
const emailConfig = config.email;
|
||||
|
||||
const result = await sendReportEmail(emailConfig, report);
|
||||
|
||||
console.log('邮件发送成功!');
|
||||
console.log('收件人:', emailConfig.recipients);
|
||||
console.log('MessageId:', result.messageId);
|
||||
console.log('========================================');
|
||||
console.log('定时任务执行完成');
|
||||
console.log('========================================');
|
||||
|
||||
} catch (error) {
|
||||
@@ -303,94 +151,60 @@ async function executeScheduledTask(config) {
|
||||
// 存储当前的定时任务
|
||||
let currentScheduledTask = null;
|
||||
|
||||
// 初始化定时任务
|
||||
export function initScheduler() {
|
||||
const config = loadConfig();
|
||||
|
||||
if (!config) {
|
||||
console.error('无法启动定时任务: 配置文件加载失败');
|
||||
return;
|
||||
}
|
||||
|
||||
if (!config.scheduler || !config.scheduler.enabled) {
|
||||
console.log('定时任务已禁用');
|
||||
return;
|
||||
}
|
||||
|
||||
if (!config.email || !config.email.smtpHost || !config.email.smtpUser) {
|
||||
console.error('无法启动定时任务: 邮件配置不完整');
|
||||
console.error('请在 config.json 中配置邮件信息');
|
||||
return;
|
||||
}
|
||||
if (!config) { console.error('无法启动定时任务: 配置文件加载失败'); return; }
|
||||
if (!config.scheduler?.enabled) { console.log('定时任务已禁用'); return; }
|
||||
|
||||
const cronTime = config.scheduler.cronTime || '0 9 * * *';
|
||||
|
||||
const enabledCount = (config.scrapers || []).filter(s => s.enabled).length;
|
||||
console.log('========================================');
|
||||
console.log('定时任务已启动');
|
||||
console.log('执行计划:', cronTime);
|
||||
console.log('金额阈值:', config.scheduler.threshold, '万元');
|
||||
console.log('收件人:', config.email.recipients);
|
||||
console.log('定时任务已启动，执行计划:', cronTime);
|
||||
console.log(`已启用的抓取来源: ${enabledCount} 个`);
|
||||
if (config.email?.recipients) console.log('收件人:', config.email.recipients);
|
||||
console.log('========================================');
|
||||
|
||||
// 如果已有任务在运行,先停止
|
||||
if (currentScheduledTask) {
|
||||
currentScheduledTask.stop();
|
||||
console.log('已停止旧的定时任务');
|
||||
}
|
||||
if (currentScheduledTask) { currentScheduledTask.stop(); }
|
||||
|
||||
// 创建定时任务
|
||||
currentScheduledTask = cron.schedule(cronTime, () => {
|
||||
executeScheduledTask(config);
|
||||
}, {
|
||||
timezone: 'Asia/Shanghai'
|
||||
});
|
||||
// 每次执行时重新加载配置，确保使用最新的 scrapers
|
||||
const latestConfig = loadConfig();
|
||||
if (latestConfig) {
|
||||
executeScheduledTask(latestConfig);
|
||||
}
|
||||
}, { timezone: 'Asia/Shanghai' });
|
||||
}
|
||||
|
||||
// 重新加载配置并重启定时任务
|
||||
export function reloadScheduler() {
|
||||
console.log('重新加载定时任务配置...');
|
||||
|
||||
// 停止当前任务
|
||||
if (currentScheduledTask) {
|
||||
currentScheduledTask.stop();
|
||||
currentScheduledTask = null;
|
||||
console.log('已停止当前定时任务');
|
||||
}
|
||||
|
||||
// 重新初始化
|
||||
if (currentScheduledTask) { currentScheduledTask.stop(); currentScheduledTask = null; }
|
||||
initScheduler();
|
||||
}
|
||||
|
||||
// 停止定时任务
|
||||
export function stopScheduler() {
|
||||
if (currentScheduledTask) {
|
||||
currentScheduledTask.stop();
|
||||
currentScheduledTask = null;
|
||||
console.log('定时任务已停止');
|
||||
return true;
|
||||
currentScheduledTask.stop(); currentScheduledTask = null;
|
||||
console.log('定时任务已停止'); return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// 获取定时任务状态
|
||||
export function getSchedulerStatus() {
|
||||
const config = loadConfig();
|
||||
const enabledScrapers = (config?.scrapers || []).filter(s => s.enabled).length;
|
||||
return {
|
||||
isRunning: currentScheduledTask !== null,
|
||||
enabledScrapers,
|
||||
config: config ? {
|
||||
enabled: config.scheduler?.enabled || false,
|
||||
cronTime: config.scheduler?.cronTime || '0 9 * * *',
|
||||
threshold: config.scheduler?.threshold || 10000,
|
||||
timeRange: config.scheduler?.timeRange || 'thisMonth',
|
||||
description: config.scheduler?.description || '',
|
||||
} : null,
|
||||
};
|
||||
}
|
||||
|
||||
// 手动执行任务(用于测试)
|
||||
export async function runTaskNow() {
|
||||
const config = loadConfig();
|
||||
if (!config) {
|
||||
throw new Error('配置文件加载失败');
|
||||
}
|
||||
if (!config) throw new Error('配置文件加载失败');
|
||||
await executeScheduledTask(config);
|
||||
}
|
||||
|
||||
668
src/server.js
668
src/server.js
@@ -1,11 +1,13 @@
|
||||
import 'dotenv/config';
|
||||
import express from 'express';
|
||||
import cors from 'cors';
|
||||
import axios from 'axios';
|
||||
import * as cheerio from 'cheerio';
|
||||
import iconv from 'iconv-lite';
|
||||
import { sendReportEmail } from './emailService.js';
|
||||
import Firecrawl from '@mendable/firecrawl-js';
|
||||
import { readFileSync, writeFileSync, existsSync } from 'fs';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { dirname, join } from 'path';
|
||||
import { sendCombinedReportEmail } from './emailService.js';
|
||||
import { initScheduler, runTaskNow, reloadScheduler, getSchedulerStatus } from './scheduler.js';
|
||||
import { runScraperWithBrowser } from './firecrawlBrowserScraper.js';
|
||||
|
||||
const app = express();
|
||||
const PORT = process.env.PORT || 5000;
|
||||
@@ -14,461 +16,307 @@ app.use(cors());
|
||||
app.use(express.json());
|
||||
app.use(express.static('public'));
|
||||
|
||||
// 南京市公共资源交易平台 - 房建市政招标公告
|
||||
const BASE_URL = 'https://njggzy.nanjing.gov.cn/njweb/fjsz/068001/068001002/';
|
||||
const firecrawl = new Firecrawl({ apiKey: process.env.FIRECRAWL_API_KEY });
|
||||
|
||||
// 获取分页URL
|
||||
function getPageUrl(pageIndex) {
|
||||
if (pageIndex === 1) {
|
||||
return `${BASE_URL}moreinfo.html`;
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
const CONFIG_PATH = join(__dirname, '..', 'config.json');
|
||||
const RESULTS_PATH = join(__dirname, '..', 'results.json');
|
||||
|
||||
function readConfig() {
|
||||
return JSON.parse(readFileSync(CONFIG_PATH, 'utf-8'));
|
||||
}
|
||||
|
||||
function saveConfig(cfg) {
|
||||
writeFileSync(CONFIG_PATH, JSON.stringify(cfg, null, 2), 'utf-8');
|
||||
}
|
||||
|
||||
// ========== 抓取结果存取 ==========
|
||||
|
||||
function readResults() {
|
||||
if (!existsSync(RESULTS_PATH)) return [];
|
||||
try {
|
||||
return JSON.parse(readFileSync(RESULTS_PATH, 'utf-8'));
|
||||
} catch (e) {
|
||||
return [];
|
||||
}
|
||||
return `${BASE_URL}${pageIndex}.html`;
|
||||
}
|
||||
|
||||
// 检查日期是否在范围内
|
||||
function isDateInRange(dateStr, startDate, endDate) {
|
||||
if (!dateStr) return false;
|
||||
const date = new Date(dateStr);
|
||||
if (isNaN(date.getTime())) return false;
|
||||
|
||||
if (startDate && date < new Date(startDate)) return false;
|
||||
if (endDate && date > new Date(endDate)) return false;
|
||||
return true;
|
||||
function saveResults(results) {
|
||||
writeFileSync(RESULTS_PATH, JSON.stringify(results, null, 2), 'utf-8');
|
||||
}
|
||||
|
||||
// 按时间范围采集多页列表
|
||||
async function fetchListByDateRange(startDate, endDate, maxPages = 50) {
|
||||
const allItems = [];
|
||||
let shouldContinue = true;
|
||||
let pageIndex = 1;
|
||||
function appendResult(result) {
|
||||
const results = readResults();
|
||||
results.unshift({ ...result, id: `result-${Date.now()}-${Math.random().toString(36).slice(2, 7)}` });
|
||||
// 最多保留 500 条
|
||||
if (results.length > 500) results.splice(500);
|
||||
saveResults(results);
|
||||
}
|
||||
|
||||
console.log(`开始按时间范围采集: ${startDate || '不限'} 至 ${endDate || '不限'}`);
|
||||
// 查询结果（支持分页与筛选）
|
||||
app.get('/api/results', (req, res) => {
|
||||
try {
|
||||
const { city, type, section, page = 1, pageSize = 20, scraperId } = req.query;
|
||||
let results = readResults();
|
||||
if (city) results = results.filter(r => r.city === city);
|
||||
if (type) results = results.filter(r => r.type === type);
|
||||
if (section) results = results.filter(r => r.section === section);
|
||||
if (scraperId) results = results.filter(r => r.scraperId === scraperId);
|
||||
const total = results.length;
|
||||
const start = (parseInt(page) - 1) * parseInt(pageSize);
|
||||
const data = results.slice(start, start + parseInt(pageSize));
|
||||
res.json({ success: true, total, page: parseInt(page), pageSize: parseInt(pageSize), data });
|
||||
} catch (e) {
|
||||
res.status(500).json({ success: false, error: e.message });
|
||||
}
|
||||
});
|
||||
|
||||
while (shouldContinue && pageIndex <= maxPages) {
|
||||
const pageUrl = getPageUrl(pageIndex);
|
||||
console.log(`正在采集第 ${pageIndex} 页: ${pageUrl}`);
|
||||
// 删除单条结果
|
||||
app.delete('/api/results/:id', (req, res) => {
|
||||
try {
|
||||
const results = readResults();
|
||||
const before = results.length;
|
||||
const updated = results.filter(r => r.id !== req.params.id);
|
||||
if (updated.length === before) return res.status(404).json({ success: false, error: '未找到' });
|
||||
saveResults(updated);
|
||||
res.json({ success: true });
|
||||
} catch (e) {
|
||||
res.status(500).json({ success: false, error: e.message });
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
const html = await fetchHtml(pageUrl);
|
||||
const items = parseList(html);
|
||||
// 清空所有结果
|
||||
app.delete('/api/results', (req, res) => {
|
||||
try {
|
||||
saveResults([]);
|
||||
res.json({ success: true, message: '已清空所有结果' });
|
||||
} catch (e) {
|
||||
res.status(500).json({ success: false, error: e.message });
|
||||
}
|
||||
});
|
||||
|
||||
if (items.length === 0) {
|
||||
console.log(`第 ${pageIndex} 页没有数据，停止采集`);
|
||||
break;
|
||||
}
|
||||
// 获取结果的筛选选项（城市/板块/类型下拉枚举）
|
||||
app.get('/api/results/filters', (req, res) => {
|
||||
try {
|
||||
const results = readResults();
|
||||
const cities = [...new Set(results.map(r => r.city).filter(Boolean))];
|
||||
const sections = [...new Set(results.map(r => r.section).filter(Boolean))];
|
||||
const types = [...new Set(results.map(r => r.type).filter(Boolean))];
|
||||
res.json({ success: true, data: { cities, sections, types } });
|
||||
} catch (e) {
|
||||
res.status(500).json({ success: false, error: e.message });
|
||||
}
|
||||
});
|
||||
|
||||
let hasItemsInRange = false;
|
||||
let allItemsBeforeRange = true;
|
||||
// ========== 抓取来源 CRUD ==========
|
||||
|
||||
for (const item of items) {
|
||||
if (isDateInRange(item.date, startDate, endDate)) {
|
||||
allItems.push(item);
|
||||
hasItemsInRange = true;
|
||||
allItemsBeforeRange = false;
|
||||
} else if (startDate && new Date(item.date) < new Date(startDate)) {
|
||||
allItemsBeforeRange = allItemsBeforeRange && true;
|
||||
} else {
|
||||
allItemsBeforeRange = false;
|
||||
}
|
||||
}
|
||||
app.get('/api/scrapers', (req, res) => {
|
||||
try {
|
||||
const cfg = readConfig();
|
||||
res.json({ success: true, data: cfg.scrapers || [] });
|
||||
} catch (e) {
|
||||
res.status(500).json({ success: false, error: e.message });
|
||||
}
|
||||
});
|
||||
|
||||
if (allItemsBeforeRange && startDate) {
|
||||
console.log(`第 ${pageIndex} 页所有项目都早于起始日期，停止采集`);
|
||||
shouldContinue = false;
|
||||
}
|
||||
app.post('/api/scrapers', (req, res) => {
|
||||
try {
|
||||
const cfg = readConfig();
|
||||
if (!cfg.scrapers) cfg.scrapers = [];
|
||||
const item = {
|
||||
id: `scraper-${Date.now()}`,
|
||||
city: req.body.city || '',
|
||||
url: req.body.url || '',
|
||||
section: req.body.section || '',
|
||||
subsection: req.body.subsection || '',
|
||||
type: req.body.type || '招标公告',
|
||||
prompt: req.body.prompt || '',
|
||||
enabled: req.body.enabled !== false,
|
||||
model: req.body.model || 'spark-1-mini',
|
||||
};
|
||||
cfg.scrapers.push(item);
|
||||
saveConfig(cfg);
|
||||
res.json({ success: true, data: item });
|
||||
} catch (e) {
|
||||
res.status(500).json({ success: false, error: e.message });
|
||||
}
|
||||
});
|
||||
|
||||
console.log(`第 ${pageIndex} 页找到 ${items.length} 条，符合条件 ${hasItemsInRange ? '有' : '无'}`);
|
||||
app.put('/api/scrapers/:id', (req, res) => {
|
||||
try {
|
||||
const cfg = readConfig();
|
||||
const idx = (cfg.scrapers || []).findIndex(s => s.id === req.params.id);
|
||||
if (idx === -1) return res.status(404).json({ success: false, error: '未找到该配置' });
|
||||
cfg.scrapers[idx] = { ...cfg.scrapers[idx], ...req.body, id: req.params.id };
|
||||
saveConfig(cfg);
|
||||
res.json({ success: true, data: cfg.scrapers[idx] });
|
||||
} catch (e) {
|
||||
res.status(500).json({ success: false, error: e.message });
|
||||
}
|
||||
});
|
||||
|
||||
pageIndex++;
|
||||
app.delete('/api/scrapers/:id', (req, res) => {
|
||||
try {
|
||||
const cfg = readConfig();
|
||||
const before = (cfg.scrapers || []).length;
|
||||
cfg.scrapers = (cfg.scrapers || []).filter(s => s.id !== req.params.id);
|
||||
if (cfg.scrapers.length === before) return res.status(404).json({ success: false, error: '未找到' });
|
||||
saveConfig(cfg);
|
||||
res.json({ success: true });
|
||||
} catch (e) {
|
||||
res.status(500).json({ success: false, error: e.message });
|
||||
}
|
||||
});
|
||||
|
||||
if (shouldContinue && pageIndex <= maxPages) {
|
||||
await new Promise(resolve => setTimeout(resolve, 500));
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`采集第 ${pageIndex} 页失败: ${err.message}`);
|
||||
break;
|
||||
// ========== 统一抓取执行 ==========
|
||||
|
||||
// 执行单个抓取来源并保存结果
|
||||
async function runScraper(scraper) {
|
||||
console.log(`[Browser] ${scraper.city} - ${scraper.section} ${scraper.subsection} - ${scraper.type}：${scraper.url}`);
|
||||
const { items } = await runScraperWithBrowser(firecrawl, scraper, { logPrefix: '[Browser][API]' });
|
||||
console.log(`[Browser] 提取到 ${items.length} 条公告`);
|
||||
|
||||
const record = {
|
||||
scraperId: scraper.id,
|
||||
city: scraper.city,
|
||||
section: scraper.section,
|
||||
subsection: scraper.subsection,
|
||||
type: scraper.type,
|
||||
url: scraper.url,
|
||||
scrapedAt: new Date().toISOString(),
|
||||
data: { result: items, total: items.length }, // 统一为 result 字段
|
||||
};
|
||||
appendResult(record);
|
||||
return record;
|
||||
}
|
||||
|
||||
// 运行指定 ID 的抓取来源（单条测试）
|
||||
app.post('/api/scrapers/:id/run', async (req, res) => {
|
||||
try {
|
||||
const cfg = readConfig();
|
||||
const scraper = (cfg.scrapers || []).find(s => s.id === req.params.id);
|
||||
if (!scraper) return res.status(404).json({ success: false, error: '未找到该配置' });
|
||||
const result = await runScraper(scraper);
|
||||
res.json({ success: true, data: result });
|
||||
} catch (e) {
|
||||
console.error('测试抓取失败:', e.message);
|
||||
res.status(500).json({ success: false, error: e.message });
|
||||
}
|
||||
});
|
||||
|
||||
// 批量运行多个抓取来源
|
||||
// body: { ids: ['id1','id2',...] } 不传则运行所有已启用的
|
||||
app.post('/api/scrape/run', async (req, res) => {
|
||||
try {
|
||||
const cfg = readConfig();
|
||||
let scrapers = cfg.scrapers || [];
|
||||
|
||||
if (req.body.ids && req.body.ids.length > 0) {
|
||||
scrapers = scrapers.filter(s => req.body.ids.includes(s.id));
|
||||
} else {
|
||||
scrapers = scrapers.filter(s => s.enabled);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`总共采集了 ${pageIndex - 1} 页，找到 ${allItems.length} 条符合条件的公告`);
|
||||
return allItems;
|
||||
}
|
||||
|
||||
const http = axios.create({
|
||||
responseType: 'arraybuffer',
|
||||
timeout: 15000,
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
},
|
||||
});
|
||||
|
||||
function pickEncoding(contentType = '') {
|
||||
const match = /charset=([^;]+)/i.exec(contentType);
|
||||
if (!match) return 'utf-8';
|
||||
const charset = match[1].trim().toLowerCase();
|
||||
if (charset.includes('gb')) return 'gbk';
|
||||
return charset;
|
||||
}
|
||||
|
||||
async function fetchHtml(url) {
|
||||
const res = await http.get(url);
|
||||
const encoding = pickEncoding(res.headers['content-type']);
|
||||
const html = iconv.decode(res.data, encoding || 'utf-8');
|
||||
return html;
|
||||
}
|
||||
|
||||
// 解析列表页HTML，提取招标信息
|
||||
function parseList(html) {
|
||||
const $ = cheerio.load(html);
|
||||
const items = [];
|
||||
|
||||
// 解析南京公共资源交易平台的列表结构
|
||||
// <li class="ewb-info-item2 clearfix" onclick="window.open('详情URL');">
|
||||
$('li.ewb-info-item2').each((_, row) => {
|
||||
const $row = $(row);
|
||||
const cells = $row.find('div.ewb-info-num2');
|
||||
|
||||
if (cells.length >= 5) {
|
||||
// 获取各字段
|
||||
const bidNo = $(cells[0]).find('p').attr('title') || $(cells[0]).find('p').text().trim();
|
||||
const projectName = $(cells[1]).find('p').attr('title') || $(cells[1]).find('p').text().trim();
|
||||
const bidName = $(cells[2]).find('p').attr('title') || $(cells[2]).find('p').text().trim();
|
||||
const estimatedPrice = $(cells[3]).find('p').text().trim();
|
||||
const publishDate = $(cells[4]).find('p').text().trim();
|
||||
|
||||
// 从onclick提取详情链接
|
||||
const onclick = $row.attr('onclick') || '';
|
||||
const hrefMatch = onclick.match(/window\.open\(['"]([^'"]+)['"]\)/);
|
||||
let href = '';
|
||||
if (hrefMatch) {
|
||||
href = hrefMatch[1];
|
||||
// 转换为绝对URL
|
||||
if (href.startsWith('/')) {
|
||||
href = `https://njggzy.nanjing.gov.cn${href}`;
|
||||
}
|
||||
}
|
||||
|
||||
// 验证日期格式 (YYYY-MM-DD)
|
||||
if (!/^\d{4}-\d{2}-\d{2}$/.test(publishDate)) return;
|
||||
|
||||
// 解析合同估算价
|
||||
const price = parseFloat(estimatedPrice);
|
||||
if (isNaN(price)) return;
|
||||
|
||||
items.push({
|
||||
bidNo, // 标段编号
|
||||
title: projectName, // 项目名称
|
||||
bidName, // 标段名称
|
||||
budget: {
|
||||
amount: price,
|
||||
unit: '万元'
|
||||
},
|
||||
date: publishDate,
|
||||
href
|
||||
});
|
||||
if (scrapers.length === 0) {
|
||||
return res.json({ success: true, data: [], message: '没有可运行的抓取来源' });
|
||||
}
|
||||
});
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
// API 路由
|
||||
|
||||
// 获取列表
|
||||
app.get('/api/list', async (req, res) => {
|
||||
try {
|
||||
const page = parseInt(req.query.page) || 1;
|
||||
const pageUrl = getPageUrl(page);
|
||||
|
||||
const html = await fetchHtml(pageUrl);
|
||||
const items = parseList(html);
|
||||
res.json({ success: true, data: items, page });
|
||||
} catch (error) {
|
||||
res.status(500).json({ success: false, error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// 按时间范围获取列表
|
||||
app.post('/api/list-daterange', async (req, res) => {
|
||||
try {
|
||||
const { startDate, endDate, maxPages = 50 } = req.body;
|
||||
const items = await fetchListByDateRange(startDate, endDate, maxPages);
|
||||
res.json({ success: true, data: items });
|
||||
} catch (error) {
|
||||
res.status(500).json({ success: false, error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// 生成报告
|
||||
app.post('/api/report', async (req, res) => {
|
||||
try {
|
||||
const { limit = 50, threshold = 50 } = req.body;
|
||||
|
||||
// 采集列表
|
||||
const items = [];
|
||||
let pageIndex = 1;
|
||||
const maxPagesToFetch = Math.ceil(limit / 10) + 1;
|
||||
|
||||
while (items.length < limit && pageIndex <= maxPagesToFetch) {
|
||||
const pageUrl = getPageUrl(pageIndex);
|
||||
console.log(`正在采集第 ${pageIndex} 页: ${pageUrl}`);
|
||||
|
||||
const results = [];
|
||||
for (const scraper of scrapers) {
|
||||
try {
|
||||
const html = await fetchHtml(pageUrl);
|
||||
const pageItems = parseList(html);
|
||||
|
||||
if (pageItems.length === 0) {
|
||||
console.log(`第 ${pageIndex} 页没有数据，停止采集`);
|
||||
break;
|
||||
}
|
||||
|
||||
items.push(...pageItems);
|
||||
pageIndex++;
|
||||
|
||||
if (items.length < limit && pageIndex <= maxPagesToFetch) {
|
||||
await new Promise(resolve => setTimeout(resolve, 500));
|
||||
}
|
||||
const r = await runScraper(scraper);
|
||||
results.push(r);
|
||||
} catch (err) {
|
||||
console.error(`采集第 ${pageIndex} 页失败: ${err.message}`);
|
||||
break;
|
||||
const errRecord = {
|
||||
scraperId: scraper.id,
|
||||
city: scraper.city,
|
||||
section: scraper.section,
|
||||
subsection: scraper.subsection,
|
||||
type: scraper.type,
|
||||
url: scraper.url,
|
||||
scrapedAt: new Date().toISOString(),
|
||||
error: err.message,
|
||||
data: null,
|
||||
};
|
||||
appendResult(errRecord);
|
||||
results.push(errRecord);
|
||||
}
|
||||
}
|
||||
|
||||
const results = items.slice(0, limit);
|
||||
|
||||
// 按阈值筛选
|
||||
const filtered = results.filter((item) => {
|
||||
return item.budget && item.budget.amount > threshold;
|
||||
});
|
||||
|
||||
const total = filtered.reduce(
|
||||
(sum, item) => sum + (item.budget?.amount || 0),
|
||||
0
|
||||
);
|
||||
|
||||
const report = {
|
||||
summary: {
|
||||
total_count: results.length,
|
||||
filtered_count: filtered.length,
|
||||
threshold: `${threshold}万元`,
|
||||
total_amount: `${total.toFixed(2)}万元`,
|
||||
generated_at: new Date().toISOString(),
|
||||
},
|
||||
projects: filtered.map((item) => ({
|
||||
bidNo: item.bidNo,
|
||||
title: item.title,
|
||||
bidName: item.bidName,
|
||||
date: item.date,
|
||||
budget: item.budget,
|
||||
url: item.href,
|
||||
})),
|
||||
};
|
||||
|
||||
res.json({ success: true, data: report });
|
||||
} catch (error) {
|
||||
res.status(500).json({ success: false, error: error.message });
|
||||
res.json({ success: true, data: results });
|
||||
} catch (e) {
|
||||
res.status(500).json({ success: false, error: e.message });
|
||||
}
|
||||
});
|
||||
|
||||
// 按时间范围生成报告
|
||||
app.post('/api/report-daterange', async (req, res) => {
|
||||
// ========== 配置管理 ==========
|
||||
|
||||
app.get('/api/config', (req, res) => {
|
||||
try {
|
||||
const { startDate, endDate, threshold = 50, maxPages = 50 } = req.body;
|
||||
|
||||
// 按时间范围采集列表
|
||||
const items = await fetchListByDateRange(startDate, endDate, maxPages);
|
||||
|
||||
if (items.length === 0) {
|
||||
return res.json({
|
||||
success: true,
|
||||
data: {
|
||||
summary: {
|
||||
total_count: 0,
|
||||
filtered_count: 0,
|
||||
threshold: `${threshold}万元`,
|
||||
total_amount: '0.00万元',
|
||||
generated_at: new Date().toISOString(),
|
||||
date_range: { startDate, endDate },
|
||||
},
|
||||
projects: [],
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// 按阈值筛选
|
||||
const filtered = items.filter((item) => {
|
||||
return item.budget && item.budget.amount > threshold;
|
||||
});
|
||||
|
||||
const total = filtered.reduce(
|
||||
(sum, item) => sum + (item.budget?.amount || 0),
|
||||
0
|
||||
);
|
||||
|
||||
const report = {
|
||||
summary: {
|
||||
total_count: items.length,
|
||||
filtered_count: filtered.length,
|
||||
threshold: `${threshold}万元`,
|
||||
total_amount: `${total.toFixed(2)}万元`,
|
||||
generated_at: new Date().toISOString(),
|
||||
date_range: { startDate, endDate },
|
||||
},
|
||||
projects: filtered.map((item) => ({
|
||||
bidNo: item.bidNo,
|
||||
title: item.title,
|
||||
bidName: item.bidName,
|
||||
date: item.date,
|
||||
budget: item.budget,
|
||||
url: item.href,
|
||||
})),
|
||||
};
|
||||
|
||||
res.json({ success: true, data: report });
|
||||
} catch (error) {
|
||||
res.status(500).json({ success: false, error: error.message });
|
||||
const cfg = readConfig();
|
||||
if (cfg.email?.smtpPass) cfg.email.smtpPass = '***已配置***';
|
||||
res.json({ success: true, data: cfg });
|
||||
} catch (e) {
|
||||
res.status(500).json({ success: false, error: e.message });
|
||||
}
|
||||
});
|
||||
|
||||
// 发送报告邮件
|
||||
app.post('/api/config', (req, res) => {
|
||||
try {
|
||||
const newCfg = req.body;
|
||||
const oldCfg = readConfig();
|
||||
if (newCfg.email?.smtpPass === '***已配置***') {
|
||||
newCfg.email.smtpPass = oldCfg.email?.smtpPass || '';
|
||||
}
|
||||
saveConfig(newCfg);
|
||||
reloadScheduler();
|
||||
res.json({ success: true, message: '配置已保存' });
|
||||
} catch (e) {
|
||||
res.status(500).json({ success: false, error: e.message });
|
||||
}
|
||||
});
|
||||
|
||||
// ========== 邮件 ==========
|
||||
|
||||
app.post('/api/send-email', async (req, res) => {
|
||||
try {
|
||||
const { emailConfig, report } = req.body;
|
||||
if (!emailConfig?.smtpHost || !emailConfig?.smtpUser || !emailConfig?.smtpPass)
|
||||
return res.status(400).json({ success: false, error: '邮件配置不完整' });
|
||||
if (!emailConfig.recipients?.trim())
|
||||
return res.status(400).json({ success: false, error: '请指定收件人' });
|
||||
if (!report)
|
||||
return res.status(400).json({ success: false, error: '没有报告数据' });
|
||||
|
||||
// 验证必需的配置参数
|
||||
if (!emailConfig || !emailConfig.smtpHost || !emailConfig.smtpUser || !emailConfig.smtpPass) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: '邮件配置不完整,请填写SMTP服务器、用户名和密码',
|
||||
});
|
||||
}
|
||||
|
||||
if (!emailConfig.recipients || emailConfig.recipients.trim() === '') {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: '请至少指定一个收件人',
|
||||
});
|
||||
}
|
||||
|
||||
if (!report) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: '没有可发送的报告数据',
|
||||
});
|
||||
}
|
||||
|
||||
// 发送邮件
|
||||
const { sendReportEmail } = await import('./emailService.js');
|
||||
const result = await sendReportEmail(emailConfig, report);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
message: '邮件发送成功',
|
||||
messageId: result.messageId,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('发送邮件API错误:', error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: error.message,
|
||||
});
|
||||
res.json({ success: true, message: '邮件发送成功', messageId: result.messageId });
|
||||
} catch (e) {
|
||||
res.status(500).json({ success: false, error: e.message });
|
||||
}
|
||||
});
|
||||
|
||||
// 获取配置
|
||||
app.get('/api/config', async (req, res) => {
|
||||
// ========== 定时任务 ==========
|
||||
|
||||
app.get('/api/scheduler/status', (req, res) => {
|
||||
try {
|
||||
const { readFileSync } = await import('fs');
|
||||
const { join } = await import('path');
|
||||
const { fileURLToPath } = await import('url');
|
||||
const { dirname } = await import('path');
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
const configPath = join(__dirname, '..', 'config.json');
|
||||
|
||||
const configContent = readFileSync(configPath, 'utf-8');
|
||||
const config = JSON.parse(configContent);
|
||||
|
||||
// 不返回敏感信息(密码)
|
||||
if (config.email && config.email.smtpPass) {
|
||||
config.email.smtpPass = '***已配置***';
|
||||
}
|
||||
|
||||
res.json({ success: true, data: config });
|
||||
} catch (error) {
|
||||
res.status(500).json({ success: false, error: error.message });
|
||||
res.json({ success: true, data: getSchedulerStatus() });
|
||||
} catch (e) {
|
||||
res.status(500).json({ success: false, error: e.message });
|
||||
}
|
||||
});
|
||||
|
||||
// 更新配置
|
||||
app.post('/api/config', async (req, res) => {
|
||||
app.post('/api/run-scheduled-task', (req, res) => {
|
||||
try {
|
||||
const { writeFileSync, readFileSync } = await import('fs');
|
||||
const { join } = await import('path');
|
||||
const { fileURLToPath } = await import('url');
|
||||
const { dirname } = await import('path');
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
const configPath = join(__dirname, '..', 'config.json');
|
||||
|
||||
const newConfig = req.body;
|
||||
|
||||
// 读取旧配置以保留敏感信息
|
||||
const oldConfigContent = readFileSync(configPath, 'utf-8');
|
||||
const oldConfig = JSON.parse(oldConfigContent);
|
||||
|
||||
// 如果密码字段是占位符,保留原密码
|
||||
if (newConfig.email && newConfig.email.smtpPass === '***已配置***') {
|
||||
newConfig.email.smtpPass = oldConfig.email?.smtpPass || '';
|
||||
}
|
||||
|
||||
// 保存配置
|
||||
writeFileSync(configPath, JSON.stringify(newConfig, null, 2), 'utf-8');
|
||||
|
||||
// 重新加载定时任务(如果定时任务配置有变化)
|
||||
reloadScheduler();
|
||||
|
||||
res.json({ success: true, message: '配置已保存并重新加载定时任务' });
|
||||
} catch (error) {
|
||||
res.status(500).json({ success: false, error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// 获取定时任务状态
|
||||
app.get('/api/scheduler/status', async (req, res) => {
|
||||
try {
|
||||
const status = getSchedulerStatus();
|
||||
res.json({ success: true, data: status });
|
||||
} catch (error) {
|
||||
res.status(500).json({ success: false, error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// 手动触发定时任务的API(用于测试)
|
||||
app.post('/api/run-scheduled-task', async (req, res) => {
|
||||
try {
|
||||
console.log('手动触发定时任务...');
|
||||
// 在后台执行任务,不阻塞响应
|
||||
runTaskNow().catch(err => {
|
||||
console.error('定时任务执行失败:', err);
|
||||
});
|
||||
res.json({
|
||||
success: true,
|
||||
message: '定时任务已触发,正在后台执行...'
|
||||
});
|
||||
} catch (error) {
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: error.message
|
||||
});
|
||||
runTaskNow().catch(err => console.error('定时任务执行失败:', err));
|
||||
res.json({ success: true, message: '定时任务已在后台触发' });
|
||||
} catch (e) {
|
||||
res.status(500).json({ success: false, error: e.message });
|
||||
}
|
||||
});
|
||||
|
||||
app.listen(PORT, () => {
|
||||
console.log(`Server running at http://localhost:${PORT}`);
|
||||
|
||||
// 启动定时任务
|
||||
console.log('正在初始化定时任务...');
|
||||
initScheduler();
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user