还是小菜，第一次分享代码哈，这是自己以前写的一个php的采集类，自己一直在用，自我感觉很简单很强大，只要懂一点点选择器的知识就可以采集任何页面了，也支持https页面，做简单的采集足够用了。 array(选择器,类型),.......) * 【类型】说明：值 text ,html ,属性 *【块选择器】：指 先按照规则 选出 几个大块 ，然后再分别再在块里面 进行相关的选择 *************************************************/ function querylist($pageurl,$regarr=array(),$regrange='') { $this->pageurl = $pageurl; //为了能获取https:// $ch = curl_init(); curl_setopt($ch, curlopt_url,$this->pageurl); curl_setopt($ch, curlopt_ssl_verifypeer, false); curl_setopt($ch, curlopt_ssl_verifyhost, false); curl_setopt($ch,curlopt_returntransfer,1); $this->html = curl_exec($ch); curl_close($ch); if(!empty($regarr)) { $this->regarr = $regarr; $this->regrange = $regrange; $this->getlist(); } } function setquery($regarr,$regrange='') { $this->jsonarr=array(); $this->regarr = $regarr; $this->regrange = $regrange; $this->getlist(); } private function getlist() { $hobj = phpquery::newdocumenthtml($this->html); if(!empty($this->regrange)) { $robj = pq($hobj)->find($this->regrange); $i=0; foreach($robj as $item) { while(list($key,$reg_value)=each($this->regarr)) { $iobj = pq($item)->find($reg_value[0]); switch($reg_value[1]) { case 'text': $this->jsonarr[$i][$key] = trim(pq($iobj)->text()); break; case 'html': $this->jsonarr[$i][$key] = trim(pq($iobj)->html()); break; default: $this->jsonarr[$i][$key] = pq($iobj)->attr($reg_value[1]); break; } } //重置数组指针 reset($this->regarr); $i++; } } else { while(list($key,$reg_value)=each($this->regarr)) { $lobj = pq($hobj)->find($reg_value[0]); $i=0; foreach($lobj as $item) { switch($reg_value[1]) { case 'text': $this->jsonarr[$i++][$key] = trim(pq($item)->text()); break; case 'html': $this->jsonarr[$i++][$key] = trim(pq($item)->html()); break; default: $this->jsonarr[$i++][$key] = pq($item)->attr($reg_value[1]); break; } } } } } function getjson() { return json_encode($this->jsonarr); } }
复制代码
array(.code_title a:eq(0),text),url=>array(.code_title a:eq(0),href),author=>array(img,title));$rang = .code_list li;$hj = new querylist($url,$reg,$rang);$arr = $hj->jsonarr;print_r($arr);//如果还想采当前页面右边的 top40活跃贡献者 图像，得到json数据,可以这样写$reg = array(portrait=>array(.hot_top img,src));$hj->setquery($reg);$json = $hj->getjson();echo $json . ;//采osc内容页内容$url = http://www.oschina.net/code/snippet_186288_23816;$reg = array(title=>array(.qtitle h1,text),con=>array(.content,html));$hj = new querylist($url,$reg);$arr = $hj->jsonarr;print_r($arr);//就举这么多例子吧，是不是用来做采集很方便
复制代码
regarr = array(title=>array(h3.t a,#ting_singlesong_box a,text),tcon=>array(div.c-abstract,font:slice(0,2),div#weibo,table tr:eq(0),div.c-abstract-size p:eq(0),div.vd_sitcom_new_tinfo,text),url=>array(h3.t a,#ting_singlesong_box a,href)); $this->regrange = 'table.result,table.result-op'; $this->regznum=array(znum=>array(span.nums,text)); } else if($searcher=='google') { $this->regarr = array(title=>array(h3.r a,text),tcon=>array(span.st,text),url=>array(h3.r a,href)); $this->regrange = 'li.g'; $this->regznum=array(znum=>array(div#resultstats,text)); } $this->searcher = $searcher; $this->key = $key; $this->num = $num; $this->page = $page-1; $this->getlist(); } private function getlist() { $s = urlencode($this->key); $num = $this->num; $start = $this->num*$this->page; if($this->searcher=='baidu') { $url = http://www.baidu.com/s?pn=$start&rn=$num&wd=$s; $reg_znum='/[\d,]+/'; } else if($this->searcher=='google') { $url=https://www.google.com.hk/search?filter=0&lr=&newwindow=1&safe=images&hl=en&as_qdr=all&num=$num&start=$start&q=$s; $reg_znum='/([\d,]+) result(s)?/'; } $searcherobj = new querylist($url,$this->regarr,$this->regrange); for($i=0;$ijsonarr);$i++) { if($this->searcher=='baidu') { $searcherobj->jsonarr[$i]['url'] = $this->getbaidurealurl($searcherobj->jsonarr[$i]['url']); } else if($this->searcher=='google') { $searcherobj->jsonarr[$i]['url'] = $this->getgooglerealurl($searcherobj->jsonarr[$i]['url']); } } $this->jsonarr = $searcherobj->jsonarr ; //获取总共结果条数 $searcherobj->setquery($this->regznum); $znum = $searcherobj->jsonarr[0]['znum']; preg_match($reg_znum,$znum,$arr)?$znum=$arr[0]:$znum=0; $znum = (int)str_replace(',','',$znum); //计算总页数 $zpage = ceil($znum/$this->num); $this->jsonarr=array('num'=>$this->num,'page'=>((int)$this->page+1),'znum'=>$znum,'zpage'=>$zpage,s=>$this->key,'other'=>array('author'=>'jae','qq'=>'734708094','blog'=>'http://blog.jaekj.com'),'data'=>$this->jsonarr); } function getjson() { return json_encode($this->jsonarr); } private function getbaidurealurl($url) { //得到百度跳转的真正地址 $header = get_headers($url,1); if (strpos($header[0],'301') || strpos($header[0],'302')) { if(is_array($header['location'])) { //return $header['location'][count($header['location'])-1]; return $header['location'][0]; } else { return $header['location']; } } else { return $url; } } private function getgooglerealurl($url) { $reg_url = '/q=(.+)&/u'; return preg_match($reg_url,$url,$arr)?urldecode($arr[1]):$url; } }// $hj = new searcher('google','oschina',20,2); // print_r( $hj->jsonarr);//效果演示地址//http://blog.jaekj.com//jae/demo/searcher/searcher_class.php?searcher=baidu&s=jaekj&num=20&page=1
复制代码