天气数据爬取

历史气象数据获取

主要的python包
requests
BeautifulSoup
re
pandas
lxml

浏览器访问模拟

根据浏览器Request-Header参数，让request模拟浏览器行为

import requests
from bs4 import BeautifulSoup
import re
import pandas as pdurl = 'https://www.wentian123.com/history/?location=%E5%98%89%E5%B3%AA%E5%85%B3&startdate=2024-01-01&enddate=2024-08-15'
header = {'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Encoding': 'gzip, deflate, br, zstd',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Cache-Control': 'max-age=0',
'Connection': 'keep-alive',
'Cookie': 'Hm_lvt_452d5df9c96fd4e38bdb12c20493de8a=1724145184; HMACCOUNT=7E8A91446E19E40E; Hm_lvt_a1574f7ae5f0b9e15ea9a7c1cd8e90c2=1724145900; Hm_lpvt_a1574f7ae5f0b9e15ea9a7c1cd8e90c2=1724918557; Hm_lpvt_452d5df9c96fd4e38bdb12c20493de8a=1724923348',
'Host': 'www.wentian123.com',
'Referer': 'https://qq.ip138.com/weather/lishi.htm',
'Sec-Fetch-Dest':'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'cross-site',
'Sec-Fetch-User': '?1',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36',
'sec-ch-ua': '"Not)A;Brand";v="99", "Google Chrome";v="127", "Chromium";v="127"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': "Windows",
}
#  创建数据表格
#

header参数获取方式;
访问页面->右键->检查

这里主要是为了避免一些基本的反爬虫操作，若网站没有反爬虫机制，不配置，或者之配置基础的’User-Agent’参数即可。
在这里插入图片描述

# 输入参数根据检索网址进行配置
如：url = 'https://www.XXX.com/history/?location=%E5%98%89%E5%B3%AA%E5%85%B3&startdate=2024-01-01&enddate=2024-08-15'
# 其中的location、startdate、enddate是当你进行网页检索时输入的信息，这里通过params进行传递；中文的话，会进行自动转码
params = {'location': '嘉峪关','startdate': '2024-01-01','enddate': '2024-08-15'
}
# 请求网页链接
response = requests.get(url,params=params,headers=header)
# 解析网页
soup = BeautifulSoup(response.content, 'lxml')# 根据网页检查显示的结果，通过访问树文件提取检索数据
datasoup = soup.find_all('tbody')
datas = datasoup[0].find_all('tr')
result= []
for data in datas:# 使用正则表达式搜索匹配项test_list  = data.text.split('\n')test_list = [item for item in test_list if item.strip() != '']# ['星期四', '2024-08-15', '多云转晴', '15℃ ～ 30℃', '西北风转微风 3-4级转<3级']week = test_list[0]date = test_list[1]weather = test_list[2]temperature = test_list[3]winds = test_list[4].split(' ')[0]winds_level = test_list[4].split(' ')[1]result.append([date, weather, temperature, winds, winds_level])
data = pd.DataFrame(result, columns=['日期', '天气', '温度', '风向','风力'])
data.to_excel(r'XXX\爬虫代码\县区_weather_data.xlsx', index=False)