农产品价格报告爬虫使用说明

# **************************************************************************
# *                                                                          *
# *                      农产品价格报告爬虫                                   *
# *                                                                          *
# *                        作者: xiaohai                                     *
# *                        版本: v1.0.0                                      *
# *                        日期: 2024-12-05                                  *
# *                                                                          *
# *        功能说明:                                                         *
# *            1. 日度报告                                                   *
# *               - 生成今日分析报告                                         *
# *               - 生成指定日期报告                                         *
# *               - 包含价格指数、分品类分析等                                *
# *                                                                          *
# *            2. 周度报告                                                   *
# *               - 生成本周分析报告                                         *
# *               - 生成指定周报告                                           *
# *               - 汇总周内价格变化                                         *
# *                                                                          *
# *            3. 价格走势                                                   *
# *               - 农产品价格200指数走势                                    *
# *               - 猪肉价格全国走势                                         *
# *               - 猪肉价格区域走势                                         *
# *               - 粮油价格指数走势                                         *
# *                                                                          *
# *            4. 数据导出                                                   *
# *               - 支持Excel格式导出                                        *
# *               - 包含多个数据分类                                         *
# *               - 支持时间范围选择                                         *
# *                                                                          *
# *        : 农业农村部市场信息中心                                   *
# *        版权声明: 仅用于学习交流                                          *
# *                                                                          *
# **************************************************************************import os
import json
import logging
import requests
from datetime import datetime, timedelta
import matplotlib.pyplot as plt
import pandas as pd
import warnings
import urllib3
import sys
import subprocess
import pkg_resources
from bs4 import BeautifulSoup
import re
import time# 禁用SSL警告
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
warnings.filterwarnings('ignore')# 配置常量
VERSION = 'v1.0.0'
AUTHOR = 'xiaohai'
DATA_SOURCE = '农业农村部市场信息中心'# API配置
API_BASE_URL = 'https://ncpscxx.moa.gov.cn'
API_ENDPOINTS = {'price_index': '/product/common-price-index/getIndexList','variety_list': '/product/sys-variety/selectList','price_trend': '/product/price-info/getPriceInfoList','market_list': '/product/sys-market/selectList','daily_price': '/product/price-info/getDailyPrice','analysis_report': '/product/analysis-report/pageList'
}# 输出目录配置
OUTPUT_DIRS = {'base': 'reports','daily': 'reports/daily','weekly': 'reports/weekly'
}# 图表样式配置
CHART_STYLE = {'figure': {'figsize': (12, 6),'facecolor': '#f8fcfa'},'grid': {'linestyle': '--','alpha': 0.3,'color': 'gray'},'line': {'marker': 'o','markersize': 4,'linewidth': 2},'colors': {'blue': '#40a9ff','green': '#73d13d','orange': '#ffa940','red': '#ff4d4f','purple': '#9254de','cyan': '#36cfc9'}
}def check_and_install_packages():"""检查并安装所需的包"""required_packages = {'requests': 'requests',      # HTTP请求'pandas': 'pandas',          # 数据处理'matplotlib': 'matplotlib',  # 绘图支持'urllib3': 'urllib3',        # HTTP客户端'openpyxl': 'openpyxl',     # Excel支持'colorama': 'colorama'       # 控制台颜色}print("\n" + "="*50)print("检查并安装依赖包...")print("="*50)try:import coloramacolorama.init()success_mark = colorama.Fore.GREEN + "✓" + colorama.Style.RESET_ALLerror_mark = colorama.Fore.RED + "✗" + colorama.Style.RESET_ALLexcept ImportError:success_mark = "✓"error_mark = "✗"all_success = Truefor package, import_name in required_packages.items():try:pkg_resources.require(package)print(f"{success_mark} {package:15} 已安装")except (pkg_resources.DistributionNotFound, pkg_resources.VersionConflict):print(f"{error_mark} {package:15} 未安装，正在安装...")try:subprocess.check_call([sys.executable, "-m", "pip", "install", "--disable-pip-version-check","--no-cache-dir",package], stdout=subprocess.DEVNULL)print(f"{success_mark} {package:15} 安装成功")except Exception as e:print(f"{error_mark} {package:15} 安装失败: {str(e)}")all_success = Falseprint("\n依赖包检查" + ("全部完成" if all_success else "存在问题"))print("="*50 + "\n")if not all_success:print("某些依赖包安装失败，程序能无法正常运行！")if input("是否继续运行？(y/n): ").lower() != 'y':sys.exit(1)class ReportCrawler:"""农产品价格报告爬虫"""def __init__(self):# 禁用SSL警告warnings.filterwarnings('ignore')# 基础配置self._setup_directories()self._setup_logger()self._setup_api()def _setup_directories(self):"""创建输出目录"""self.output_dir = "reports"self.daily_dir = os.path.join(self.output_dir, "daily")self.weekly_dir = os.path.join(self.output_dir, "weekly")for d in [self.output_dir, self.daily_dir, self.weekly_dir]:if not os.path.exists(d):os.makedirs(d)def _setup_logger(self):"""配置日志"""log_file = os.path.join("logs", f"crawler_{datetime.now().strftime('%Y%m%d')}.log")os.makedirs("logs", exist_ok=True)formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s',datefmt='%Y-%m-%d %H:%M:%S')# 文件处理器file_handler = logging.FileHandler(log_file, encoding='utf-8')file_handler.setFormatter(formatter)# 制台处理器console_handler = logging.StreamHandler()console_handler.setFormatter(formatter)# 配置日志器self.logger = logging.getLogger(__name__)self.logger.setLevel(logging.INFO)self.logger.addHandler(file_handler)self.logger.addHandler(console_handler)def _setup_api(self):"""配置API"""self.headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36','Origin': 'https://ncpscxx.moa.gov.cn','Referer': 'https://ncpscxx.moa.gov.cn/','Accept': 'application/json, text/plain, */*','Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8','Content-Type': 'application/json;charset=UTF-8'}def show_menu(self):"""显示功能菜单"""menu = """
农产品价格报告爬虫系统
====================
1. 成今日分析报告
2. 生成本周分报告
3. 生成指定日期报告
4. 生成指定周报告
5. 生成价格指数走势图
6. 生成猪肉价格走势图
7. 生成区域价格走势图
8. 生成粮油价格走势图
9. 导出Excel数据
0. 退出系统请输入���能编号(0-9): """print("\n" + "="*50)  # 添加分隔线choice = input(menu)print("="*50 + "\n")  # 添加分隔线return choicedef run(self):"""运行系统"""while True:choice = self.show_menu()if choice == "0":print("感谢使用,再见!")breakelif choice == "1":print("正在生成今日分析报告...")self.generate_daily_report(datetime.now())elif choice == "2":print("正在生成本周分析报告...")today = datetime.now()self.generate_weekly_report(today.year, int(today.strftime("%W")))elif choice == "3":date_str = input("请输入日期(格式:YYYY-MM-DD): ")try:date = datetime.strptime(date_str, "%Y-%m-%d")self.generate_daily_report(date)except:print("日期格式错误!")elif choice == "4":year = int(input("请输入年份: "))week = int(input("请输入周数(1-52): "))self.generate_weekly_report(year, week)elif choice == "5":days = int(input("请输入要查看的天数: "))end = datetime.now()start = end - timedelta(days=days)self.plot_index_trend(start, end)elif choice == "6":days = int(input("请输入要查看的天数: "))end = datetime.now()start = end - timedelta(days=days)self.plot_pig_price_trend(start, end)elif choice == "7":days = int(input("请输入要查看的天数: "))end = datetime.now()start = end - timedelta(days=days)self.plot_pig_price_region_trend(start, end)elif choice == "8":days = int(input("请输入要查看的天数: "))end = datetime.now()start = end - timedelta(days=days)self.plot_grain_price_trend(start, end)elif choice == "9":days = int(input("请输入要导天数: "))end = datetime.now()start = end - timedelta(days=days)self.export_data(start, end)else:print("无效的选择,请重试!")input("\n按回车键继续...")def _make_request(self, url, method='get', params=None, data=None):"""发送HTTP请求Args:url: 请求URLmethod: 请求方法，支持 'get'/'post'params: URL参数data: POST数据Returns:Response对象或None(请求失败)"""try:headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'}if method.lower() == 'get':response = requests.get(url,params=params,headers=headers,verify=False,timeout=10)else:response = requests.post(url,params=params,json=data,  # 添加json参数支持headers=headers,verify=False,timeout=10)response.raise_for_status()return responseexcept requests.exceptions.RequestException as e:self.logger.error(f"请求失败: {str(e)}")return Nonedef fetch_daily_report(self, date):"""获取日度价格报告"""try:url = f"{API_BASE_URL}/api/FarmDaily/list"data = {"daylyDate": date.strftime("%Y-%m-%d")}response = self._make_request(url, method='post', data=data)if not response:return Nonedata = response.json()if data.get("code") == 200 and data.get("content",{}).get("list"):# 找到指定日期的报告target_date = date.strftime("%Y-%m-%d")for report in data["content"]["list"]:if report["daylyDate"].startswith(target_date):# 提取所需数据return {"conclusion": report["counclesion"],"indexConclusion": report["indexConclusion"],"animalConclusion": report["animalConclusion"],"aquaticConclusion": report["aquaticConclusion"],"vegetablesConclusion": report["vegetablesConclusion"],"fruitsConclusion": report["fruitsConclusion"],"content": report["countent"],"incOrReduRange": report["incOrReduRange"]}self.logger.warning(f"未找到{target_date}的报告")return Noneself.logger.warning(f"获取数据失败: {data.get('message', '未知错误')}")return Noneexcept Exception as e:self.logger.error(f"获取日度报告出错: {str(e)}")return Nonedef _extract_conclusions(self, report):"""从报告中提取各类结论"""try:return {"index": report.get("indexConclusion", ""),"animal": report.get("animalConclusion", ""),"aquatic": report.get("aquaticConclusion", ""),"vegetables": report.get("vegetablesConclusion", ""),"fruits": report.get("fruitsConclusion", ""),"range": report.get("incOrReduRange", "")}except Exception as e:self.logger.error(f"提取论出错: {str(e)}")return {}def fetch_index_data(self, start_date, end_date):"""获取价格指数数据"""try:url = "https://pfsc.agri.cn/price_portal/pi-info-day/getPortalPiInfoDay"response = requests.post(url, headers=self.headers, verify=False)data = response.json()if data["code"] == 200:result = []for item in data["content"]:pub_date = datetime.strptime(item["publishDate"], "%Y-%m-%d")if start_date <= pub_date <= end_date:result.append({"日期": item["publishDate"],"农产品批发价格200指数": item["agriculture"],"粮油指数": item["grainAndOil"],"篮子数": item["vegetableBasket"]})return resultreturn Noneexcept Exception as e:self.logger.error(f"获取指数数据失败: {str(e)}")return Nonedef fetch_pig_price_data(self, start_date, end_date):"""获取猪肉价格数据"""try:url = f"{API_BASE_URL}{API_ENDPOINTS['variety_list']}"params = {'pid': 'MH'}  # 猪肉品类IDresponse = self._make_request(url, method='post', params=params)if not response:return Nonedata = response.json()if data.get("code") == 200 and data.get("data"):# 转换数据格式result = []for item in data["data"]:if start_date <= datetime.strptime(item["date"], "%Y-%m-%d") <= end_date:result.append({"日期": item["date"],"全国": float(item["national"]),"东北": float(item["northEast"]),"华北": float(item["northChina"]),"华东": float(item["eastChina"]),"华中": float(item["centralChina"]),"华南": float(item["southChina"]),"西南": float(item["southWest"])})return resultself.logger.warning(f"获取数据失败: {data.get('message', '未知错误')}")return Noneexcept Exception as e:self.logger.error(f"获取猪肉价格数据失败: {str(e)}")return Nonedef fetch_grain_price_data(self, start_date, end_date):"""获取粮油价格数据"""try:url = f"{API_BASE_URL}{API_ENDPOINTS['variety_list']}"params = {'pid': 'TL'}  # 粮油品类IDresponse = self._make_request(url, method='post', params=params)if not response:return Nonedata = response.json()if data.get("code") == 200 and data.get("data"):# 转换数据格式result = []for item in data["data"]:if start_date <= datetime.strptime(item["date"], "%Y-%m-%d") <= end_date:result.append({"日期": item["date"],"通义粮价指数": float(item["grainPriceIndex"]),"通义粮市指数": float(item["grainMarketIndex"]),"通义粮市第1号": float(item["grainMarketNo1"]),"通义粮天指数": float(item["grainDayIndex"]),"通义���指": float(item["grainIndex"]),"通义粮天指数(干粮)": float(item["grainDayDryIndex"])})return resultself.logger.warning(f"获取数据失败: {data.get('message', '未知错误')}")return Noneexcept Exception as e:self.logger.error(f"获取粮油价格数据失败: {str(e)}")return Nonedef generate_daily_report(self, date):"""生成每日分析报告"""try:report_data = self.fetch_daily_report(date)if not report_data:self.logger.warning(f"未获取到 {date.strftime('%Y-%m-%d')} 的报告数据")returnreport_file = os.path.join(self.daily_dir,f"{date.strftime('%Y年%m月%d日')}_价格分析报告.md")# 使用更清晰模板格式content = f"""# {date.strftime('%Y年%m月%d日')} 农产品价格分析报告## 一、价格指数变化
{report_data["indexConclusion"]}## 二、分品类分析### 1. 畜禽产品
{report_data["animalConclusion"]}### 2. 水产品
{report_data["aquaticConclusion"]}### 3. 蔬菜
{report_data["vegetablesConclusion"]}### 4. 水果
{report_data["fruitsConclusion"]}## 三、价格波动情况
{report_data["incOrReduRange"]}## 四、数据说明
- 数据来源: {report_data["source"]}
- 生成时间: {datetime.now().strftime('%Y年%m月%d日 %H:%M:%S')}
- 价格单位: 元/斤
- 涨跌幅: 与上一交易日相比---
*注: 本报告由系统自动生成，仅供参考。*
"""with open(report_file, "w", encoding="utf-8") as f:f.write(content)self.logger.info(f"分析报告已生成: {report_file}")except Exception as e:self.logger.error(f"生成分析报告失败: {str(e)}")def generate_weekly_report(self, year, week):"""生成周度汇总报告"""try:start_date = datetime.strptime(f'{year}-W{week:02d}-1', '%Y-W%W-%w')end_date = start_date + timedelta(days=6)print(f"\n正在生成第{week}周报告...")print(f"时间范围: {start_date.strftime('%Y-%m-%d')} 至 {end_date.strftime('%Y-%m-%d')}")print("="*50)# 获周内所有报告reports = []current = start_datetotal_days = (end_date - start_date).days + 1for i in range(total_days):print(f"\r进度: {i+1}/{total_days} ", end="")report = self.fetch_daily_report(current)if report:reports.append(report)current += timedelta(days=1)if not reports:self.logger.warning("本周无可用数据")return# 计算周度汇总数据weekly_summary = self._calculate_weekly_summary(reports)report_file = os.path.join(self.weekly_dir,f"{year}年第{week:02d}周_{start_date.strftime('%m月%d日')}-{end_date.strftime('%m月%d日')}_价格分析报告.md")with open(report_file, "w", encoding="utf-8") as f:f.write(f"""# {year}年第{week:02d}周农产品价格分析报告
({start_date.strftime('%Y年%m月%d日')} 至 {end_date.strftime('%Y年%m月%d日')})## 一、本周价格概况
{weekly_summary['overview']}## 二、价格指数变化
- 周初: {weekly_summary['index_start']}
- 周末: {weekly_summary['index_end']}
- 度变化: {weekly_summary['index_change']}## 三、分品类周度分析
### 1. 畜禽产品
{weekly_summary['animal_summary']}### 2. 水产品
{weekly_summary['aquatic_summary']}### 3. 蔬菜
{weekly_summary['vegetables_summary']}### 4. 水果
{weekly_summary['fruits_summary']}## 四、日度价格详情
""")for report in reports:pub_date = datetime.strptime(report['daylyDate'][:10], '%Y-%m-%d')f.write(f"""### {pub_date.strftime('%Y年%m月%d日')}
1. 价格指数: {report.get('indexConclusion', '暂无数据')}
2. 畜禽产品: {report.get('animalConclusion', '暂无数据')}
3. 水产品: {report.get('aquaticConclusion', '暂无数据')}
4. 蔬菜: {report.get('vegetablesConclusion', '暂无数据')}
5. 水果: {report.get('fruitsConclusion', '暂无数据')}""")f.write(f"""## 五、数据说明
- 数据来源: {reports[0]["source"]}
- 生成时间: {datetime.now().strftime('%Y年%m月%d日 %H:%M:%S')}
- 价格单位: 元/公斤
- 跌幅: 与上期相比---
*注: 本报告由系统自动生成，仅供参考。*""")print("\n报告生成完成!")self.logger.info(f"周度报告已生成: {report_file}")except Exception as e:self.logger.error(f"生成周度报告失败: {str(e)}")def _calculate_weekly_summary(self, reports):"""计算周度汇总数据"""summary = {'overview': '','index_start': reports[0].get('indexConclusion', '暂无数据'),'index_end': reports[-1].get('indexConclusion', '暂无数据'),'index_change': '','animal_summary': '','aquatic_summary': '','vegetables_summary': '','fruits_summary': ''}# 计算价格指数变化try:start_index = float(reports[0]['indexConclusion'].split('为')[1].split('，')[0])end_index = float(reports[-1]['indexConclusion'].split('为')[1].split('，')[0])change = end_index - start_indexsummary['index_change'] = f"{'上升' if change >= 0 else '下降'}{abs(change):.2f}个点"except:summary['index_change'] = '数据异常'# 生成概述summary['overview'] = f"本周农产品批发价格200指数从{summary['index_start']}，到{summary['index_end']}，整体{summary['index_change']}。"# 其他品类汇总...return summarydef plot_index_trend(self, start_date, end_date):"""绘制价格指数走势图"""try:data = self.fetch_index_data(start_date, end_date)if not data:returnplt.figure(figsize=(12, 6), facecolor='#f8fcfa')ax = plt.gca()ax.set_facecolor('#f8fcfa')dates = [item["日期"] for item in data]indices = [("农品批发价格200指数", "#ffa940"),("菜篮子指", "#73d13d"),("粮油指数", "#40a9ff")]for name, color in indices:values = [item[name] for item in data]plt.plot(dates, values, color=color, marker='o',markersize=4, linewidth=2, label=name)plt.title('农业农村部"农产品批发价格200指数"日度走势图',pad=20, fontsize=12, loc='left')plt.grid(True, linestyle='--', alpha=0.3)plt.xticks(rotation=45)plt.legend(loc='upper right', frameon=False)plt.tight_layout()plt.savefig(os.path.join(self.output_dir, "价格指数走势图.png"),dpi=300,bbox_inches='tight')plt.close()self.logger.info("价格指数走势已生成")except Exception as e:self.logger.error(f"生成价格指数走势图失败: {str(e)}")def plot_pig_price_trend(self, start_date, end_date):"""绘制猪肉价格走势图"""try:data = self.fetch_pig_price_data(start_date, end_date)if not data:returnplt.figure(figsize=(12, 6), facecolor='#f8fcfa')ax = plt.gca()ax.set_facecolor('#f8fcfa')dates = [item["日期"] for item in data]values = [item["全国"] for item in data]plt.plot(dates, values, color='#40a9ff', marker='o',markersize=4, linewidth=2)plt.fill_between(dates, values, color='#e6f7ff', alpha=0.5)plt.title('"瘦肉型白条猪肉出厂价格指数"全国走势图',pad=20, fontsize=12, loc='left')plt.grid(True, linestyle='--', alpha=0.3)plt.xticks(rotation=45)plt.tight_layout()plt.savefig(os.path.join(self.output_dir, "猪肉价格走势图.png"),dpi=300,bbox_inches='tight')plt.close()self.logger.info("猪肉价格走势图已生成")except Exception as e:self.logger.error(f"生成猪肉价格走势图失败: {str(e)}")def plot_pig_price_region_trend(self, start_date, end_date):"""绘制猪肉分区域价格走势图"""try:data = self.fetch_pig_price_data(start_date, end_date)if not data:returnplt.figure(figsize=(12, 6), facecolor='#f8fcfa')ax = plt.gca()ax.set_facecolor('#f8fcfa')dates = [item["日期"] for item in data]regions = [("东北", "#40a9ff"),("华南", "#73d13d"),("华", "#ffa940"),("华中", "#ff4d4f"),("华东", "#9254de"),("西南", "#36cfc9")]for region, color in regions:values = [item[region] for item in data]plt.plot(dates, values, color=color, marker='o',markersize=4, linewidth=2, label=region)plt.title('"瘦肉型条猪肉出厂价格指数"区域走势图',pad=20, fontsize=12, loc='left')plt.grid(True, linestyle='--', alpha=0.3)plt.xticks(rotation=45)plt.legend(loc='upper right', frameon=False)plt.tight_layout()plt.savefig(os.path.join(self.output_dir, "猪肉价格区域走势图.png"),dpi=300,bbox_inches='tight')plt.close()self.logger.info("猪肉价格区域走势图已生成")except Exception as e:self.logger.error(f"生成猪肉价格区域走势图失败: {str(e)}")def plot_grain_price_trend(self, start_date, end_date):"""绘制粮油价格走势图"""try:data = self.fetch_grain_price_data(start_date, end_date)if not data:returnplt.figure(figsize=(12, 6), facecolor='#f8fcfa')ax = plt.gca()ax.set_facecolor('#f8fcfa')dates = [item["日期"] for item in data]indices = [("通义粮价指数", "#40a9ff"),("通义粮市指数", "#73d13d"),("通义粮市第1号", "#ffa940"),("通义粮天指数", "#ff4d4f"),("通义粮指", "#9254de"),("通义粮天指数(干粮)", "#36cfc9")]for name, color in indices:values = [item[name] for item in data]plt.plot(dates, values, color=color, marker='o',markersize=4, linewidth=2, label=name)plt.title('中国通义粮油发价格指数走势图',pad=20, fontsize=12, loc='left')plt.grid(True, linestyle='--', alpha=0.3)plt.xticks(rotation=45)plt.legend(loc='upper right', frameon=False)plt.tight_layout()plt.savefig(os.path.join(self.output_dir, "粮油价格指数走势图.png"),dpi=300,bbox_inches='tight')plt.close()self.logger.info("粮油价格指数走势图已生成")except Exception as e:self.logger.error(f"生成粮油价格指数走势失败: {str(e)}")def export_data(self, start_date, end_date, format='excel'):"""导出数据Args:start_date: 开始日期end_date: 结束日期format: 导出格式，支持 'excel'/'csv'/'json'"""try:# 获取数据data = {'index': self.fetch_index_data(start_date, end_date),'pig': self.fetch_pig_price_data(start_date, end_date),'grain': self.fetch_grain_price_data(start_date, end_date)}if not any(data.values()):return# 根据格式导出if format == 'excel':self._export_excel(data, start_date, end_date)elif format == 'csv':self._export_csv(data, start_date, end_date)elif format == 'json':self._export_json(data, start_date, end_date)else:self.logger.error(f"不支持的导出格式: {format}")except Exception as e:self.logger.error(f"导出数据失败: {str(e)}")def _clean_text(self, text):"""清理文本内容"""if not text:return ""# 去除多余空白字符text = ' '.join(text.split())# 修复可能的标点符号问题text = text.replace('。。', '。').replace('，。', '。').replace('；。', '。')# 修复中文编码text = text.encode('utf-8').decode('utf-8', 'ignore')return textdef _validate_report_data(self, report):"""验证报告数据完整性"""required_fields = ["indexConclusion","animalConclusion","aquaticConclusion","vegetablesConclusion","fruitsConclusion"]is_valid = Truefor field in required_fields:if not report.get(field):self.logger.warning(f"报告缺少 {field} 数据")is_valid = Falsereport[field] = "暂无数据"return is_validdef _export_excel(self, data, start_date, end_date):"""导出Excel数据"""try:filename = f"价格数据_{start_date.strftime('%Y%m%d')}_{end_date.strftime('%Y%m%d')}.xlsx"filepath = os.path.join(self.output_dir, filename)with pd.ExcelWriter(filepath) as writer:# 导出价格指数if data.get('index'):df_index = pd.DataFrame(data['index'])df_index.to_excel(writer, sheet_name='价格指数', index=False)# 导出猪肉价格if data.get('pig'):df_pig = pd.DataFrame(data['pig'])df_pig.to_excel(writer, sheet_name='猪肉价格', index=False)# 导出粮油价格if data.get('grain'):df_grain = pd.DataFrame(data['grain'])df_grain.to_excel(writer, sheet_name='粮油价格', index=False)self.logger.info(f"���据已导出至: {filepath}")return Trueexcept Exception as e:self.logger.error(f"导出Excel失败: {str(e)}")return Falsedef fetch_all_categories(self):"""获取所有品类数据"""categories = {'MH': '猪肉','SC': '蔬菜','SG': '水果','TL': '粮油','SC': '水产','DJ': '蛋鸡','NR': '牛肉','YR': '羊肉'}result = {}for code, name in categories.items():try:url = f"{API_BASE_URL}{API_ENDPOINTS['variety_list']}"params = {'pid': code}response = self._make_request(url, method='post', params=params)if response and response.json().get("code") == 200:result[name] = response.json().get("data", [])except Exception as e:self.logger.error(f"获取{name}品类数据失败: {str(e)}")return resultdef fetch_market_prices(self, market_id=None, variety_id=None, start_date=None, end_date=None):"""获取市场价格数据"""try:url = f"{API_BASE_URL}{API_ENDPOINTS['daily_price']}"params = {'marketId': market_id,'varietyId': variety_id,'startDate': start_date.strftime("%Y-%m-%d") if start_date else None,'endDate': end_date.strftime("%Y-%m-%d") if end_date else None}response = self._make_request(url, method='post', params=params)if response and response.json().get("code") == 200:return response.json().get("data", [])return Noneexcept Exception as e:self.logger.error(f"获取市场价格数据失败: {str(e)}")return Nonedef fetch_analysis_reports(self, page=1, page_size=10):"""获取分析报告列表"""try:url = f"{API_BASE_URL}{API_ENDPOINTS['analysis_report']}"params = {'pageNum': page,'pageSize': page_size}response = self._make_request(url, method='post', params=params)if response and response.json().get("code") == 200:return response.json().get("data", {}).get("list", [])return Noneexcept Exception as e:self.logger.error(f"获取分析报告失败: {str(e)}")return Nonedef crawl_all_data(self, start_date, end_date):"""爬取所有数据"""try:# 获取所有品类categories = self.fetch_all_categories()# 获取所有市场markets_response = self._make_request(f"{API_BASE_URL}{API_ENDPOINTS['market_list']}", method='post')markets = markets_response.json().get("data", []) if markets_response else []# 存储结果results = {'categories': categories,'markets': markets,'prices': {},'reports': []}# 获取每个品类的价格数据for category, varieties in categories.items():results['prices'][category] = {}for variety in varieties:variety_id = variety.get('id')if variety_id:prices = self.fetch_market_prices(variety_id=variety_id,start_date=start_date,end_date=end_date)results['prices'][category][variety.get('name')] = prices# 获取分析报告page = 1while True:reports = self.fetch_analysis_reports(page=page)if not reports:breakresults['reports'].extend(reports)page += 1return resultsexcept Exception as e:self.logger.error(f"爬取所有数据失败: {str(e)}")return Nonedef fetch_weekly_report_content(self, report_id=None):"""获取周度报告内容"""try:url = f"{API_BASE_URL}/product/analysis-report/getReportContent"params = {'id': report_id} if report_id else Noneresponse = self._make_request(url, method='post', params=params)if not response:return None# 解析HTML内容soup = BeautifulSoup(response.text, 'html.parser')# 提取报告基本信息title = soup.find('h1', class_='report-title').text.strip()date = soup.find('div', class_='report-date').text.strip()source = soup.find('div', class_='report-source').text.strip()# 提取报告主体内容content = soup.find('div', class_='report-content')# 提取表格数据tables = []for table in content.find_all('table'):df = pd.read_html(str(table))[0]tables.append(df.to_dict('records'))# 提取文本内容paragraphs = []for p in content.find_all('p'):text = p.text.strip()if text:paragraphs.append(text)return {'title': title,'date': date,'source': source,'content': {'text': paragraphs,'tables': tables}}except Exception as e:self.logger.error(f"获取报告内容失败: {str(e)}")return Nonedef crawl_all_reports(self, start_date=None, end_date=None):"""爬取所有报告"""try:reports = []page = 1while True:# 获取报告列表report_list = self.fetch_analysis_reports(page=page)if not report_list:break# 过滤日期范围if start_date or end_date:filtered_reports = []for report in report_list:report_date = datetime.strptime(report['publishDate'], '%Y-%m-%d')if start_date and report_date < start_date:continueif end_date and report_date > end_date:continuefiltered_reports.append(report)report_list = filtered_reports# 获取每个报告的详细内容for report in report_list:report_content = self.fetch_weekly_report_content(report['id'])if report_content:reports.append({'meta': report,'content': report_content})# 添加延时避免请求过快time.sleep(1)page += 1return reportsexcept Exception as e:self.logger.error(f"爬取报告失败: {str(e)}")return Nonedef save_reports(self, reports, output_dir='reports'):"""保存报告到文件"""try:if not os.path.exists(output_dir):os.makedirs(output_dir)for report in reports:# 生成文件名date = datetime.strptime(report['meta']['publishDate'], '%Y-%m-%d')filename = f"{date.strftime('%Y%m%d')}_{report['meta']['id']}.json"filepath = os.path.join(output_dir, filename)# 保存为JSON文件with open(filepath, 'w', encoding='utf-8') as f:json.dump(report, f, ensure_ascii=False, indent=2)return Trueexcept Exception as e:self.logger.error(f"保存报告失败: {str(e)}")return Falseif __name__ == "__main__":try:# 检查并安装依赖包check_and_install_packages()# 运行爬虫crawler = ReportCrawler()crawler.run()except KeyboardInterrupt:print("\n程序已被用中断")sys.exit(0)except Exception as e:print(f"\n程序运行出错: {str(e)}")sys.exit(1)