爬虫学习心得
- 下面给新学者分享一些爬虫的技巧
- 1 xpath的学习
- a xpath的基本用法
- b xpath案例
- 2.正则表达式学习
- a 基本使用
- b 12306提取省份成json格式
- 3.Bs的学习
- 3.1基础
- 3.3基础案例
- 3.5基本案例
- 4.拿取数据技巧
- 4.1通过标签获取
- 4.2通过lamda表达式获取数据
下面给新学者分享一些爬虫的技巧
1 xpath的学习
xpath为用的最广，也是最好用的，不过比较吃性能。
a xpath的基本用法
from lxml import etree
from lxml import html
etree=html. etree
#需要加载准备解析的数据
f=open("test.html",modle="r",encoding='utf-8')
pageSource = f.read()
#加载数据，返回element对象
et=etree.HTML(pageSource)
#xpath的语法
result =et.xpath('/html')#/html表示根节
result = et.xpath('/html/body')#表达式中间的/表示一层html节点
result= et.xpath('/html/body/span')
result =et.xpath('/html/body/span/text()')#获取span中的文字
#text()表示提取标签中的文本信息
result = et.xpath('/html/body/*/l/a/text()')
#*任意的.通配符
result =et.xpath('/htmi/body/*/li/a/@href')#@表示属性
result =et.xpath("//li/a/@href")#/表示任意位置
result=et.xpath('/div[@class=job]/text()') #[@xx=xxxx]属性的限定
print(result)
#带循环的
result= et.xpath('/html/body/ul/li')
for item in result:href= item.xpath("./a/@href")[0]#./表示当前这个元素text =item.xpath("./a/text()")[0]#./表示当前这个元素print(text, href)
b xpath案例
import requests
import re
import os
import json
from lxml import etree
from lxml import etree
url = "xxx"
headers = {"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36"
}
try:response = requests.get(url=url,headers=headers)# 设置编码为和父类一样response.encoding =response.apparent_encodingprint(response.text)et = etree.HTML(response.text)#xpath匹配results = et.xpath(f'//*[@id="J-lemma-main-wrapper"]/div[2]/div/div[1]/div/*/*/*/dd/span/text()')print(results)
except:print("errrro")
2.正则表达式学习
a 基本使用
import re
result=re.search(r'\d+','今天我有100块，买了两个蛋糕')#返回match对象
print(result.group())result=re.finditer(r'\d+','今天我有100块，买了两个蛋糕')#返回的是迭代器
print(result)
for item in result:print(item.group())
#预加载
obj=re.compile(r'\d+')
result=obj.findall('今天我有100块，买了两个蛋糕')
print(result)match="divorce 分歧;diverse 各色各样;diversify 使多样化;divert 转入;divest 脱去;divorce 分开; divulge 泄露"
pattern=".*? (.*?);"
match_1=re.search(pattern,match)
print(match_1.group())
b 12306提取省份成json格式
import requests
import re
url = "https://kyfw.12306.cn/otn/resources/js/framework/favorite_name.js"
headers = {"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36"
}
response = requests.get(url=url,headers=headers)
response.encoding =response.apparent_encoding
parr = re.compile(r"@.*?\|(.*?)\|(.*?)\|")
images = re.findall(parr,response.text)
len1=len(images)
print(response.text)
data={}
json_file = open('data.json', 'w+', encoding='UTF-8',newline='')
json_file.write("{")
json_file.write('\n')
for image in images:# print(image[0],end='')# print(image[1])name=image[0]id=image[1]json_file.write(f"\"{name}\":\"{id}\"")if name!=images[len1-1][0]:json_file.write(",")json_file.write("\n")
json_file.write('\n')
json_file.write("}")
3.Bs的学习
3.1基础
from urllib.request import urlopen
from bs4 import BeautifulSoup
html=urlopen('xxx.html')
bs=BeautifulSoup(html, 'html.parser')
# data=bs.select('li a[href="news/index.html"]')
# data=bs.select('title')
# print(data)
#1.通过标签名查找
data=bs.select('title')[0].get_text()
print('通过标签名查找:'+data)
print('----------'*30)
#2.通过类名查找
data1=bs.select('.infoPath a')[0].get_text()
print('通过类名查找:'+data1)
print('----------'*30)
#3.通过id名查找
data2=bs.select('#header_login_user a')[0].get_text()
print('通过id名查找:'+data2)
print('----------'*30)
#4.组合查找
data3=bs.select('.readAreaBox.content h1')[0].get_text()
print('通过组合查找查找:'+data3)
print('----------'*30)
#5.属性查找
data3=bs.select('div a[href="/man"]')[0].get_text()
print('通过属性查找查找:'+data3)
###3.2基础案例
import requests
from bs4 import BeautifulSoup
class Content:def __init__(self, url, title, body):self.url = urlself.title = titleself.body = body
def getPage(url):html = requests.get(url)return BeautifulSoup(html.content, 'html.parser')
def scrapeNew(url):#小说bs =getPage(url)title = bs.find('h1').textbody = bs.select('#readArea > div.readAreaBox.content > div.p')[0].textreturn Content(url, title, body)
def scrapeNew1(url):#故事大全bs=getPage(url)title = bs.find('h1').text# print(title)# body=bs.select('#newsnr p')body=bs.find('div',class_='article_content').text# body=''.join(body)return Content(url,title,body)def scrapeNew2(url):#新浪新闻bs=getPage(url)title = bs.find('h1').textprint(title)body=bs.find('div',class_='article').textreturn Content(url,title,body)
url= 'xxxx'content = scrapeNew(url)
print('Title:{}'.format(content.title))
print('URL:{}'.format(content.url))
print('body:{}'.format(content.body))
print("-" * 130)
url='xxxx'content=scrapeNew1(url)
print('Title:{}'.format(content.title))
print('URL:{}' .format(content.url))
print('body:{}'.format(content.body))
print("-" * 130)
url='xxxx'content=scrapeNew2(url)
print('Title:{}'.format(content.title))
print('URL:{}' .format(content.url))
print('body:{}'.format(content.body))3.3基础案例
import requests
from bs4 import BeautifulSoup
class Content:def __init__(self, topic, url, title, body):self.topic=topicself.url=urlself.title=titleself.body=bodydef print(self):print("找到的相关关键字的文章:{}".format(self.topic))print("网站:{}".format (self.url))print("标题:{}".format (self.title))print("内容:{}".format (self.body))
class Website:def __init__(self, name, url, searchUrl, resultListing, resultUrl, absoluteUrl, titleTag, bodyTag):self.name=name#网站名self.urL=url#网站网址self.searchUrl=searchUrl#关键词的总界面self.resultListing=resultListing#存放每个信息的盒子，即每个结果链接的上层标签，例如diself.resultUrl=resultUrl#更细致的定位准确的URL，如div.result h3 aself.absoluteUrl=absoluteUrl#一个布尔值，判断绝对相对链接self.titleTag=titleTag#具体链接文章的标题标签self.bodyTag=bodyTag
headers={"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 Edg/118.0.2088.61"}
class Crawler:def getPage (self, url):# print('测试'+url)try:req=requests.get (url,headers=headers)except requests.exceptions.RequestException:return Nonereturn BeautifulSoup(req.content,'html.parser')def safeGet (self, pageObj, selector):childObj=pageObj.select(selector)if len(childObj) > 0:return childObj[0].get_text()return ""def search(self, topic, site):bs=self.getPage(site.searchUrl+topic+'&type=web')# print(bs)searchResults=bs.select(site.resultListing)# print(searchResults)for result in searchResults:# print(result)url=result.select(site.resultUrl)[0].attrs['lanmu1']print(url)if (site.absoluteUrl):bs=self.getPage(url)else:bs=self.getPage(site.url+url)if bs is None:print("网页有错")returntitle=self.safeGet(bs,site.titleTag)# print(title)body=self.safeGet(bs,site.bodyTag)if title !="" and body !="":content=Content(topic,url,title,body)content.print()
# #news_list > table:nth-child(5) > tbody > tr:nth-child(1) > td:nth-child(2) > ul > li.news_title > a
crawler=Crawler()
sitedata=[['中国新闻网','https://news.cctv.com','https://search.cctv.com/search.php?qtext=','div .tright','h3.tit span',True,'h1','#content_area']]
sites=[]
for row in sitedata:site_obj=Website(row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7])sites.append(site_obj)topics=['双十一']for topic in topics:print ('GETTING INFO ABOUT:'+topic)for targetSite in sites:crawler.search(topic,targetSite)
3.4基础案例
import requests
from bs4 import BeautifulSoup
import re
class Website:def __init__(self, name, url, targetPattern, absoluteUrl, titleTag, bodyTag) :self. name = nameself.url = urlself. targetPattern = targetPatternself.absoluteUrl=absoluteUrlself. titleTag = titleTagself.bodyTag = bodyTag
class Content:def __init__(self, url, title, body):self.url = urlself.title=titleself.body = bodydef print(self):print("URL: {}".format(self.url))print("TITLE:{}".format(self.title))print("BODY:{]".format(self.body))
class Crawler:def __init__(self, site):self.site = siteself.visited=[]def getPage(self, url):try:req = requests.get(url)except requests.exceptions.RequestException:return Nonereturn BeautifulSoup(req.content,'html.parser')def safeGet (self, pageObj, selector):selectedElems = pageObj.select(selector)if selectedElems is not None and len(selectedElems) > 0:return '\n'.join([elem.get_text() for elem in selectedElems])return ''def parser(self, url):bs = self.getPage('http://zs.lywhxy.com'+url)print(bs)if bs is not None:title = self.safeGet(bs, self.site.titleTag)print(title)body = self.safeGet(bs, self.site.bodyTag)if title != '' and body != '':content = Content(url, title, body)content.print()def crawl(self): # 获取网站士贝的页面链投bs = self.getPage(self.site.url)targetPages = bs.findAll('a',href=re.compile(self.site.targetPattern))# print(targetPages)for targetPage in targetPages:targetPage = targetPage.attrs['href']# print(targetPage)if targetPage not in self.visited:self.visited.append(targetPage)if not self.site.absoluteUrl:# targetPage = '{}{}'.format(self.site.url, targetPage)# print(targetPage)self.parser(targetPage)#reuters = Website('brook.double', http://www.lywhy.com', '^(/Life/)', False, 'h4 span', 'dd.fr')
# reuters=Website('学院概況','http://ww.lywhxy.com','^（/about/）',False, 'h4 span','h1']
reuters = Website('Reuters','http://zs.lywhxy.com', '/.*?/', False,'h1', 'p')
crawler = Crawler(reuters)
crawler.crawl()3.5基本案例
import requests
from bs4 import BeautifulSoup
import re
class Content:#所有文意人网页的共同基类def __init__(self,topic,url,title,body):self.topic=topicself.url=urlself.title=titleself.body=bodydef print(self):#打印函数，控制結果输出，友便烹服print("找到的相关关键字的文章：{}".format(self.topic))print("网址:{}".format(self.url))print("标题:{}".format(self.title))print("内容:{}".format(self.body))
class Website:def __init__(self, name, url, titleTag, bodyTag):self.name = nameself.url = urlself.titleTag = titleTagself.bodyTag = bodyTag
class Crawler:def getPage(self,url):try:html = requests.get(url)# html.encoding="UTF-8"except requests.exceptions.RequestException:return Nonereturn BeautifulSoup(html.content,'html.parser')#return BeautifulSoup(html.content, 'html.parser')def safeGet(self, pageObj, selector):#pageObj指bs对象selectedElems = pageObj.select(selector)if selectedElems is not None and len(selectedElems) > 0:return '\n'.join([elem.get_text() for elem in selectedElems])return ''def parse(self, site_obj, url):"""调用 getPage()获取包含目标数据的 bs对象，使用 safeGet()解析 bs对象的 title和 body,非空时存储到 content里"""bs = self.getPage(url)if bs is not None:title = self.safeGet(bs, site_obj.titleTag)print(title)body = self.safeGet(bs, site_obj.bodyTag)# print(body)if title != '' and body != '':content = Content(site_obj.name,url, title, body)content.print() # 调用封装后的 print()
# 将要爬取的目标网页的 name,url,tag,cssselector等信息存储在嵌套列表里：
siteData = [# ['丽江文化旅游学院','https://lywhxy.com','h1','p']#readArea > div.readAreaBox.content > div.p #readArea > div.readAreaBox.content body > div.wrap_q > div.main_r.floatleft_q > div.right_r.floatleft_q > div['央视新闻', 'https://news.cctv.com', '.title_area h1', '.content_area p'],['免费小说网','https://www.17k.com/chapter/3328785/44207503.html','div.readAreaBox.content h1','div.readAreaBox.content > div.p'],['故事大全','https://www.qigushi.com/tonghuagushi/1067.html','h1','.article_content'],['新浪新闻','https://news.sina.com.cn/gov/xlxw/2023-10-25/doc-imzshqvs2406187.shtml','h1','.article'],['青年文摘','https://blog.csdn.net/csdnnews/article/details/134025189?spm=1000.2115.3001.5927','h1','.wznr_r'],['领导留言板','','title','body > div.main > div.layout.rm_txt.cf > div.col.col-1.fl > div.rm_txt_con.cf']
]
#print(type(siteData))
# 将上述信息实例化成 website对象：
websites=[]
# print(type(websites))
for site in siteData:# site_obj=Website(site[0],site[1], site[2],site[3])site_obj = Website(site[0], site[1], site[2], site[3])websites.append(site_obj)
crawler = Crawler()
crawler.parse(websites[0],'https://news.cctv.com/2023/10/24/ARTIFCTydl9njIwEibuO0C2j231024.shtml?spm=C94212.P4YnMod9m2uD.ENPMkWvfnaiV.4')
crawler.parse(websites[1],'https://www.17k.com/chapter/3328785/44207503.html')
crawler.parse(websites[2],'https://www.qigushi.com/tonghuagushi/1067.html')
crawler.parse(websites[3],'https://news.sina.com.cn/gov/xlxw/2023-10-25/doc-imzshqvs2406187.shtml')
crawler.parse(websites[4],'http://www.cyp.com.cn/?viewnews-4481.html')
crawler.parse(websites[5],'http://henan.people.com.cn/n2/2023/1020/c351638-40610144.html')
4.拿取数据技巧
4.1通过标签获取
import time
import requests
import random
from bs4 import BeautifulSoup
import re
url='https://www.baidu.com/'
head={
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
}
pages=set()
random.seed(time.time())
html=requests.get(url)
html.encoding='utf-8'
bs=BeautifulSoup(html.text,"html.parser")
#获取页面中所有内链的列表
includeUrl='baidu.com'
links=bs.find_all('a',href=re.compile('^(.*'+includeUrl+')'))
# print(resp.json())
for link in links:print(link.attrs['href'])
4.2通过lamda表达式获取数据
html = httpx.get(url, headers=header, verify=False)
bs = BeautifulSoup(html.text, 'html.parser')
# try:
# bodys = bs.select('.detail_content p')
# for body in bodys:
# body = body.get_text()
# print(body)
# except:
# pass
#
#
# biaoge=bs.select('.wenben>table>tbody > tr')
# for bg in biaoge:
# bg=bg.get_text()
# bg=bg.replace('\n', '**')
# print(bg)
#
#
# neirong=bs.select('#mycontent>div>span')
# for nr in neirong:
# print(nr.text)# txt=bs.find_all(lambda tag: tag.name=='p')
# for i in txt:
# print(i.text)
#
# txt1=bs.find_all(lambda tag: tag.name=='tbody')
# for i in txt1:
# print(i.text)# txt2=bs.find_all(lambda tag: len(tag.attrs)==1 and tag.attrs[0]=='title')
# for i in txt2:
# print(i.text)# txt2=bs.find_all(lambda tag: tag.name == 'span' and tag.has_attr('class') and tag['class'][0] == 'title')
# print(len(txt2))
# for i in txt2:
# print(i.text)
#
# txt4=bs.find_all(lambda tag: tag.name == 'span' and tag.has_attr('class')==False)
# print(len(txt4))
# for i in txt4:
# print(i.text)txt5=bs.find_all(lambda tag: tag.name == 'span')
print(len(txt5))
for i in txt5:print(i.text)
biaoge=bs.find_all(lambda tag: tag.name == 'tr')
for bg in biaoge:bg=bg.get_text()bg=bg.replace('\n', '**')print(bg)
![[笔记] 使用 Jenkins 实现 CI/CD ：从 GitLab 拉取 Java 项目并部署至 Windows Server](https://i-blog.csdnimg.cn/direct/80690f955e024be69ee1d43474992041.png)
