经过了几天的摸索，照猫画虎的把爬虫的部分做完了。
但是很多原理性的东西都不是很理解，就是照着抄的，还需要继续学习。
看这个目录结构，只看.py的文件，.pyc的文件是运行的时候生成的不管它。
items.py:定义想要导出的数据
Pipelines.py:用于将数据导出
settings.py:告诉程序数据传输需要的文件
init.py:没用到过还
（以上是我自己暂时的理解）
我最终抓取的是智联招聘的信息，注意点都在注释里
获取职位连接的关键代码，linkspider，其他文件都是生成时默认的
#encoding:utf-8 from scrapy.spider import BaseSpider from scrapy.http import FormRequest, Request from scrapy.selector import HtmlXPathSelector import os import sys import datetime import reclass ZhaoPinSpider(BaseSpider):name = "zhaopin"allowed_domains = ["zhaopin.com"]#url加上pd=1就只显示今天新添加的职位 城市后面都固定为全国zlzp_urlpatten = "http://sou.zhaopin.com/jobs/searchresult.ashx?pd=1&jl={CITY}&kw={KEYWORD}&p={CURR_PAGE}"def __init__(self):self.headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0',\'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',\'Accept-Language':'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',\'Connection':'keep-alive'} self.start_urls = self.set_url()#set_url方法动态设定要抓取的链接列表def set_url(self):url_list = []#从配置文件中取出所有的关键字列表，逐个检索keys = '大数据,hadoop,hive,hbase,spark,storm,sqoop,pig'for keyword in keys.split(','):url = self.zlzp_urlpattenurl = url.format(CITY='全国', KEYWORD=keyword, CURR_PAGE=1)url_list.append(url)return url_listdef start_requests(self): #该方法必须返回一个可迭代对象(iterable)。该对象包含了spider用于爬取的第一个Request。for url in self.start_urls:yield FormRequest(url,\headers = self.headers,\callback = self.parse)#使用回调函数def parse(self, response):hxs = HtmlXPathSelector(response)keyword = hxs.select('//div[@class="search"]//input[@name="KeyWord"]/@value').extract()[0]keyword = keyword.encode('utf-8')url = self.zlzp_urlpatten#找总页数pageInfo = hxs.select('//div[@class="pagesDown"]//button/@onclick').extract()if pageInfo: #注意 只有一页时找不到pageInfopageInfo = pageInfo[0]pattern = re.compile('.*?value,(.*?),.*', re.S)findPageNum = re.search(pattern, pageInfo)pageNum = int(findPageNum.group(1))else:pageNum = 1for curPage in range(1,pageNum + 1):each_url = url.format(CITY='全国', KEYWORD=keyword, CURR_PAGE=curPage)yield Request(each_url,callback=self.get_joburls_bypage)def get_joburls_bypage(self, response):hxs = HtmlXPathSelector(response)links = hxs.select('//td[@class="zwmc"]//a/@href').extract()# 获得的信息都是当天的 直接入库for link in links:if(link != 'http://e.zhaopin.com/products/1/detail.do'): #有的地方会冒出这个链接 去掉open('../output/link_output/link.txt', 'ab').write(link+'\n')
获取具体职位信息的page相关代码：
settings
# Scrapy settings for zhaopin_page project # # For simplicity, this file contains only the most important settings by # default. All the other settings are documented here: # # http://doc.scrapy.org/topics/settings.html # BOT_NAME = 'zhaopin_page' BOT_VERSION = '1.0'SPIDER_MODULES = ['zhaopin_page.spiders'] NEWSPIDER_MODULE = 'zhaopin_page.spiders' USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION) ITEM_PIPELINES = {'zhaopin_page.FilePipelines.PagePipeline':5}
FilePipelines
# encoding: utf-8 import traceback import datetime import sys reload(sys) sys.setdefaultencoding( "utf-8" ) sys.path.append("../../../")class PagePipeline(object):#把解析后的内容放入文件中def process_item(self, item, spider):fname = '../output/page_output/' + item['file_id'] + '.txt'try:outfile = open(fname, 'wb')outfile.write(item['web_id']+self.getJobFieldSpt()+item['job_url']+self.getJobFieldSpt()+item['job_name']+self.getJobFieldSpt()+item['job_location']+self.getJobFieldSpt()+item['job_desc']+self.getJobFieldSpt()+item['edu']+self.getJobFieldSpt()+item['gender']+self.getJobFieldSpt()+item['language']+self.getJobFieldSpt()+item['major']+self.getJobFieldSpt()+item['work_years']+self.getJobFieldSpt()+item['salary']+self.getJobFieldSpt()+item['company_name']+self.getJobFieldSpt()+item['company_desc']+self.getJobFieldSpt()+item['company_address']+self.getJobFieldSpt()+item['company_worktype']+self.getJobFieldSpt()+item['company_scale']+self.getJobFieldSpt()+item['company_prop']+self.getJobFieldSpt()+item['company_website']+self.getJobFieldSpt()+self.getCurrentTimestamp())except Exception as e:print "ERROR GEN FILE!! >>> " + fnameprint traceback.format_exc()def getCurrentTimestamp(self):# 得到时间戳return datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')def getJobFieldSpt(self):#得到生成的职位文件字段间的分隔符。使用ascii码1，和hive中默认的分隔符相同 return chr(1)
items
# encoding: utf-8from scrapy.item import Item, Field#定义存放帖子内容的类 class PageItem(Item):#网站标识web_id = Field()#生成的文件名file_id = Field()#职位来源网址job_url = Field()#工作名称job_name = Field()#工作地点 job_location = Field()#职位描述 job_desc = Field()#学历要求 edu = Field()#性别要求 gender = Field()#语言要求 language = Field()#专业要求 major = Field()#工作年限 work_years = Field()#薪水范围 salary = Field()#职位发布时间job_datetime = Field()#公司名称 company_name = Field()#企业介绍company_desc = Field()#公司地址company_address = Field()#行业company_worktype = Field()#规模company_scale = Field()#性质company_prop = Field()#网址company_website = Field()
spider
# encoding: utf-8from scrapy.spider import BaseSpider from scrapy.http import FormRequest, Request from scrapy.selector import HtmlXPathSelector from zhaopin_page import items import traceback import sys import datetime import re#定义要抓取页面的爬虫类 class ZhaoPinPageSpider(BaseSpider):name = "page" start_urls = []def __init__(self): self.start_urls = self.set_url()#从jobs_task表中读出要抓取的链接列表，放入数组中def set_url(self):url_list = []link_file = open('../output/link_output/link.txt', 'r')loops = 0for each_link in link_file:each_link = each_link.replace('\r','')each_link = each_link.replace('\n','')url_list.append(each_link)loops+=1if (loops == 100):breaklink_file.close()return url_listdef parse(self, response):try:#url中后面的数字串file_id = response.url.split("/")[-1].split(".")[0]hxs = HtmlXPathSelector(response)#获取最上面一栏的内容title = '' #职位名companyName = '' #公司名basicInfo = hxs.select('//div[@class="fixed-inner-box"]').extract()[0] #会有两个 后面有个clone的pattern = re.compile('.*?<h1>(.*?)</h1>.*?<a.*?>(.*?)</a>.*?', re.S)findBasicInfo = re.search(pattern, basicInfo)if findBasicInfo:title = findBasicInfo.group(1).strip() #职位名companyName = findBasicInfo.group(2).strip() #公司名#获取左侧基本公司信息 不能用正则表达式，因为有信息不全的情况 如http://jobs.zhaopin.com/297851037250005.htmcompanySize = '' #公司规模companyType = '' #公司性质 companyLine = '' #公司行业companyHost = '' #公司主页companyAddress = '' #公司地companyInfo = hxs.select('//div[@class="company-box"]').extract()[0].encode('utf-8') #尽管只有一个，但是是列表形式，还是需要取出来if(companyInfo.find('公司规模：')>-1):companySize = companyInfo.split('公司规模：</span>')[1]companySize = companySize.split('<strong>')[1]companySize = companySize.split('</strong>')[0].strip()if(companyInfo.find('公司性质：')>-1):companyType = companyInfo.split('公司性质：</span>')[1]companyType = companyType.split('<strong>')[1]companyType = companyType.split('</strong>')[0].strip()if(companyInfo.find('公司行业：')>-1):companyLine = companyInfo.split('公司行业：</span>')[1]companyLine = companyLine.split('<strong>')[1]companyLine = companyLine.split('</a>')[0]companyLine = companyLine.split('>')[1].strip()if(companyInfo.find('公司主页：')>-1):companyHost = companyInfo.split('公司主页：</span>')[1]companyHost = companyHost.split('<strong>')[1]companyHost = companyHost.split('</a>')[0]companyHost = companyHost.split('>')[1].strip()if(companyInfo.find('公司地址：')>-1):companyAddress = companyInfo.split('公司地址：</span>')[1]companyAddress = companyAddress.split('<strong>')[1]companyAddress = companyAddress.split('</strong>')[0].strip()#获取中部工作要求信息salary = '' #职位月薪 必须先声明变量 否则会出错address = '' #工作地点jobDateTime = '' #发布日期jobCategory = '' #工作性质experience = '' #工作经验education = '' #最低学历numberInNeed = '' #招聘人数jobType = '' #职位类别jobRequirementInfo = hxs.select('/html/body/div[4]/div[1]/ul').extract()[0]pattern = re.compile('.*?<strong>(.*?)</strong>\ .*?<strong>.*?<a.*?>(.*?)</a>\ .*?<strong>.*?<span.*?>(.*?)</span>\ .*?<strong>(.*?)</strong>\ .*?<strong>(.*?)</strong>\ .*?<strong>(.*?)</strong>\ .*?<strong>(.*?)</strong>\ .*?<strong>.*?target.*?>(.*?)</a>',re.S) #前面不能有空格或者TAB 否则匹配不上findJobRequirementInfo = re.search(pattern, jobRequirementInfo)if findJobRequirementInfo:salary = findJobRequirementInfo.group(1).strip() #职位月薪address = findJobRequirementInfo.group(2).strip() #工作地点jobDateTime = findJobRequirementInfo.group(3).strip() #发布日期jobCategory = findJobRequirementInfo.group(4).strip() #工作性质experience = findJobRequirementInfo.group(5).strip() #工作经验education = findJobRequirementInfo.group(6).strip() #最低学历numberInNeed = findJobRequirementInfo.group(7).strip() #招聘人数jobType = findJobRequirementInfo.group(8).strip() #职位类别#获取描述信息detailInfo = hxs.select('//div[@class="tab-inner-cont"]').extract()jobDescribe = detailInfo[0]companyDescribe = detailInfo[1]pattern = re.compile('<.*?>| ',re.S) #删除无用的信息jobDescribe = re.sub(pattern,'',jobDescribe).strip() #职位描述companyDescribe = re.sub(pattern,'',companyDescribe).strip() #公司介绍companySize = re.sub(pattern,'',companySize).strip()companyType = re.sub(pattern,'',companyType).strip()companyLine = re.sub(pattern,'',companyLine).strip()companyHost = re.sub(pattern,'',companyHost).strip()companyAddress = re.sub(pattern,'',companyAddress).strip() salary = re.sub(pattern,'',salary).strip()address = re.sub(pattern,'',address).strip()jobDateTime = re.sub(pattern,'',jobDateTime).strip()jobCategory = re.sub(pattern,'',jobCategory).strip()experience = re.sub(pattern,'',experience).strip()education = re.sub(pattern,'',education).strip()numberInNeed = re.sub(pattern,'',numberInNeed).strip()jobType = re.sub(pattern,'',jobType).strip()title = re.sub(pattern,'',title).strip()companyName = re.sub(pattern,'',companyName).strip()data = items.PageItem()data['web_id'] = "zhaopin"data['file_id'] = file_iddata['job_url'] = response.urldata['job_name'] = titledata['job_desc'] = jobDescribedata['gender'] = ""data['major'] = ""data['company_name'] = companyNamedata['job_datetime'] = jobDateTimedata['job_location'] = addressdata['work_years'] = experiencedata['edu'] = educationdata['salary'] = salarydata['company_desc'] = companyDescribedata['company_address'] = companyAddressdata['company_website'] = companyHostdata['language'] = ""data['company_worktype'] = companyLinedata['company_prop'] = companyTypedata['company_scale'] = companySize#更新任务表中抓取状态#self.jobsTool.updateCrulInfo(ConfigPropObj.liepin_webid, response.url, 1, "")return dataexcept Exception as e:print "ERROR PARSE"print response.urlprint traceback.format_exc()
