需求:
爬取 中国天气网指定城市一周的天气,以天津为例
实现:
1,先找到一周的数据位置。
divs = html.xpath("//div[@class='hanml']")
2,再遍历每天。
trs = div.xpath("./div/div[2]/table//tr[position()=1]")
3,获取每天的最高气温 最低气温。
city_data['max_tem'] = int(tr.xpath("./td[@width='92']/text()")[0]) # 得到城市最高气温
city_data['min_tem'] = int(tr.xpath("./td[@width='86']/text()")[0]) # 得到城市最低气温
代码:
import requests
from lxml import etreeBASE_URL = 'http://www.weather.com.cn/textFC/tianjin.shtml' # 爬取一个网页数据
def get_data():headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36'} try:response = requests.get(BASE_URL, headers= headers)response.raise_for_status()response.encoding = 'utf-8' # 在打开文件的时候需要指出编码方式,否则会报错with open('data.txt', 'w', encoding='utf-8') as fp:fp.write(response.text)except:print("爬取数据失败")# 解析网页数据
def get_usefuldata():with open('data.txt', 'r', encoding='utf-8') as fp:text = fp.read()html = etree.HTML(text)#得到所有天气divs = html.xpath("//div[@class='hanml']") for div in divs:# 获取第1个tr标签trs = div.xpath("./div/div[2]/table//tr[position()=1]") for tr in trs:city_data = {}city_data['max_tem'] = int(tr.xpath("./td[@width='92']/text()")[0]) # 得到城市最高气温city_data['min_tem'] = int(tr.xpath("./td[@width='86']/text()")[0]) # 得到城市最低气温print(city_data)def main():get_data()get_usefuldata()if __name__ == '__main__':main()
效果: