PHP爬取历史天气

PHP作为宇宙第一语言，爬虫也是非常方便，这里爬取的是从天气网获得中国城市历史天气统计结果。

程序架构

main.php

<?phpinclude_once("./parser.php");include_once("./storer.php");#解析器和存储器见下文$parser = new parser();$storer = new storer();#获得url列表$urlList = $parser->getCityList("http://lishi.tianqi.com/");#依次解析新的URL网站内容，并存到数据库中foreach($urlList as $url){$data = $parser->getData($url);$storer->store($data);}

解析器

解析器提供两个接口，一个是解析主页，获得url列表；另一个是解析每座城市的数据，获得该城市的历史天气数据。

这里使用到的解析库是phpquery，使用JQuery的查询方式，简单高效。

<?php#借助JQuery库解析include_once("./phpQuery-onefile.php");
class parser
{//获取城市url列表function getCityList($url){//直接在线流下载phpQuery::newDocumentFile($url);//第一次选择$links = pq(".bcity *");$urlList = [];foreach ($links as $link) {#第二次选择$tmp = pq($link)->find('a')->attr('href');#过滤组标签if ($tmp!="#" and $tmp!="") {#检查urlif(strpos($tmp,"-")==false and filter_var($tmp, FILTER_VALIDATE_URL))$urlList[] = $tmp; #添加URL列表}}return $urlList;}//获取某个城市的历史气候function getData($url){//直接在线流下载phpQuery::newDocumentFile($url);//第一次选择$text = pq("div .tqtongji p")->text();#匹配城市$city = $this->match("/，(.+)共出现/",$text);#匹配天气$rainy = $this->match("/雨(\d+)天/",$text);$cloudy = $this->match("/多云(\d+)天/",$text);$sunny = $this->match("/晴(\d+)天/",$text);$overcast = $this->match("/阴(\d+)天/",$text); #为了跟cloudy区分$snowy = $this->match("/雪(\d+)天/",$text);#匹配拼音$pinYin = $this->match("/http:\/\/lishi\.tianqi\.com\/(.*?)\/index\.html/",$url);$result["url"] = $url;$result["city"] = $city;$result["pinYin"] =  $pinYin;$result["rainy"] = $rainy;$result["cloudy"] = $cloudy;$result["sunny"] = $sunny;$result["overcast"] = $overcast;$result["snowy"] = $snowy;return $result;}#正则解析function match($rule,$text){preg_match_all($rule, $text, $result);#有些地区不是所有天气都有if(count($result[1])==0)return "0";return $result[1][0];}
}

存储器

使用MySQLi接口即可，代码如下：

<?phpclass storer{public $mysqli;function __construct(){$this->mysqli = new mysqli('localhost', '***', '******', 'phpWeather');$this->mysqli->query("SET NAMES UTF8");}function store($data){$url = $data["url"];$city = $data["city"];$pinYin = $data["pinYin"];$rainy = $data["rainy"];$cloudy = $data["cloudy"];$sunny = $data["sunny"];$overcast = $data["overcast"];$snowy = $data["snowy"];#字符串在插入时要添加''来区分$insertData = "VALUES('$city','$pinYin',$rainy,$cloudy,$sunny,$overcast,$snowy,'$url');";#sql分开写更加清楚$sql = "INSERT INTO record(city,pinYin,rainy,cloudy,sunny,overcast,snowy,url)".$insertData;$isok = $this->mysqli->query($sql);if($isok){echo "$city 数据添加成功\n";}else{echo $sql . "\n";echo "$city 数据添加失败\n";}}function __destruct(){$this->mysqli->close();}}
?>