要下载的图片网站
1、总共多少页，得到每页的 url 列表
2、每页的图片详情的 ulr 列表（因为该高清大图在图片详情页，因此需要去图片详情页拿图片的url)
3、进入图片详情页，获取到图片url 然后下载。
完整代码如下：
import aiofiles
import aiohttp
import asyncio
import requests
from lxml import etree# 下载单个图片
async def download_one(url):print("开始下载", url)name = url[0].split("/")[-1][:-4]head = {'Host': 'file.jiutuvip.com','User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/114.0','Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8','Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2','Accept-Encoding': 'gzip, deflate, br','Connection': 'keep-alive','Upgrade-Insecure-Requests': '1','Sec-Fetch-Dest': 'document','Sec-Fetch-Mode': 'navigate','Sec-Fetch-Site': 'none','Sec-Fetch-User': '?1','TE': 'trailers'}# 发送网络请求async with aiohttp.ClientSession() as session:async with session.get(url=url[0], headers=head) as resp: # 相当于 requests.get(url=url[0], headers=head)# await resp.text() => resp.textcontent = await resp.content.read() # => resp.content# 写入文件async with aiofiles.open('./img/' + name + '.webp', "wb") as f:await f.write(content)print("下载完毕")# 获取图片的url
async def download(href_list):for href in href_list:async with aiohttp.ClientSession() as session:async with session.get(url=href) as child_res:html = await child_res.text()child_tree = etree.HTML(html)src = child_tree.xpath("//div[@class='img_box']/a/img/@src") # 选手图片地址 url 列表await download_one(src)# 获取图片详情url
async def get_img_url(html_url):async with aiohttp.ClientSession() as session:async with session.get(url=html_url) as resp:html = await resp.text()tree = etree.HTML(html)href_list = tree.xpath("//div[@class='list-box-p']/ul/li/a/@href") # 选手详情页 url 列表print(href_list)await download(href_list)
# 页面总页数
def get_html_url(url):page = 2response = requests.get(url=url)response.encoding = "utf-8"tree = etree.HTML(response.text)total_page = tree.xpath("//*[@id='pages']/a[12]/text()") # 页面总页数print(total_page)html_url_list = []while page <= 4: # int(total_page[0]) 页数太多，本例只取第 2、3、4 页next_url = f"https://www.yeitu.com/meinv/xinggan/{page}.html"html_url_list.append(next_url)page += 1return html_url_listasync def main():# 拿到每页url列表url = "https://www.yeitu.com/meinv/xinggan/"html_url_list = get_html_url(url=url) # 588个页面的url列表tasks = []for html_url in html_url_list:t = asyncio.create_task(get_img_url(html_url)) # 创建任务tasks.append(t)await asyncio.wait(tasks)if __name__ == '__main__':event_loop = asyncio.get_event_loop()event_loop.run_until_complete(main())
结果如图所示：
