1. 导入爬虫需要使用的包

 import urllib.request

 import urllib.parse

 2.创建请求函数

 def create_request(page):

# 定义不变的url部分

base_url = 'https://movie.douban.com/j/chart/top_list?type=5&interval_id=100%3A90&action=&'

# 根据规律定义data拼接url

data = { 'start':(page-1)*20, 'limit':20, }

#将字典数据编码为字符串

data = urllib.parse.urlencode(data)

url = base_url + data

headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" }

# 请求对象定制

request = urllib.request.Request(url=url,headers=headers)

return request

3.创建获取内容函数

def get_content(request):

# 向服务器发送请求,接收获取响应

response = urllib.request.urlopen(request)

# 将字节形式的内容转码

content = response.read().decode('utf-8') return content 

4.创建下载函数 

 def download(page,content):

fp = open('movie_'+ str(page) + '.json','w',encoding='utf-8')

fp.write(content)

5. 程序入口

 if __name__ == '__main__':

        start_page = int(input('请输入起始页码'))

        end_page = int(input('请输入结束页码'))

        for page in range(start_page,end_page+1):

        request = create_request(page)

        content = get_content(request)

        download(page,content)

 6.展示

 7.源码

import urllib.request
import  urllib.parse# 创建请求函数
def create_request(page):# 源网址# 第一页https: // movie.douban.com / j / chart / top_list?type = 5 & interval_id = 100 % 3A90 & action = & start = 0 & limit = 20# 第二页https: // movie.douban.com / j / chart / top_list?type = 5 & interval_id = 100 % 3A90 & action = & start = 20 & limit = 20# 第三页https: // movie.douban.com / j / chart / top_list?type = 5 & interval_id = 100 % 3A90 & action = & start = 40 & limit = 20# 定义不变的url部分base_url = 'https://movie.douban.com/j/chart/top_list?type=5&interval_id=100%3A90&action=&'# 根据规律定义data拼接urldata = {'start':(page-1)*20,'limit':20,}#将字典数据编码为字符串data = urllib.parse.urlencode(data)url = base_url + dataheaders = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"}# 请求对象定制request = urllib.request.Request(url=url,headers=headers)return  request# 创建获取内容函数
def get_content(request):# 向服务器发送请求,接收获取响应response = urllib.request.urlopen(request)# 将字节形式的内容转码content = response.read().decode('utf-8')return content# 创建下载函数
def download(page,content):fp = open('movie_'+ str(page) + '.json','w',encoding='utf-8')fp.write(content)
# 主程序入口
if __name__ == '__main__':# 手动输入要爬取的起始页码start_page = int(input('请输入起始页码'))end_page = int(input('请输入结束页码'))# 循环遍历每一页数据for page in range(start_page,end_page+1):# 调用接收函数request = create_request(page)content = get_content(request)download(page,content)