python下载大文件

1. wget

def download_big_file_with_wget(url, target_file_name):"""使用wget下载大文件Note: 需要系统安装wget"""import osimport subprocessdownload_process = subprocess.Popen(["wget", "-c", "-O", target_file_name, "'{}'".format(url)])download_process.wait()if not os.path.exists(target_file_name):raise Exception("fail to download file from {}".format(url))

 

2. python核心库

def download_big_file(url, target_file_name):"""使用python核心库下载大文件ref: https://stackoverflow.com/questions/1517616/stream-large-binary-files-with-urllib2-to-file"""import sysif sys.version_info > (2, 7):# Python 3from urllib.request import urlopenelse:# Python 2from urllib2 import urlopenresponse = urlopen(url)chunk = 16 * 1024with open(target_file_name, 'wb') as f:while True:chunk = response.read(chunk)if not chunk:breakf.write(chunk)

 

源码见blog.