1.composer安装一个curl封装好的类库
composer require php-curl-class/php-curl-class
2.创建数据库
CREATE TABLE `shixin` (
`id` varchar(255) DEFAULT NULL,
`SiteId` varchar(255) DEFAULT NULL,
`StdStg` varchar(255) DEFAULT NULL,
`StdStl` varchar(255) DEFAULT NULL,
`age` varchar(255) DEFAULT NULL,
`areaName` varchar(255) DEFAULT NULL,
`businessEntity` varchar(255) DEFAULT NULL,
`cambrian_appid` varchar(255) DEFAULT NULL,
`cardNum` varchar(255) DEFAULT NULL,
`caseCode` varchar(255) DEFAULT NULL,
`changefreq` varchar(255) DEFAULT NULL,
`courtName` varchar(255) DEFAULT NULL,
`disruptTypeName` varchar(255) DEFAULT NULL,
`duty` longtext,
`focusNumber` varchar(255) DEFAULT NULL,
`gistId` varchar(255) DEFAULT NULL,
`gistUnit` varchar(255) DEFAULT NULL,
`iname` varchar(255) DEFAULT NULL,
`lastmod` varchar(255) DEFAULT NULL,
`loc` varchar(255) DEFAULT NULL,
`partyTypeName` varchar(255) DEFAULT NULL,
`performance` varchar(255) DEFAULT NULL,
`performedPart` varchar(255) DEFAULT NULL,
`priority` varchar(255) DEFAULT NULL,
`publishDate` varchar(255) DEFAULT NULL,
`publishDateStamp` varchar(255) DEFAULT NULL,
`regDate` varchar(255) DEFAULT NULL,
`sexy` varchar(255) DEFAULT NULL,
`sitelink` varchar(255) DEFAULT NULL,
`type` varchar(255) DEFAULT NULL,
`unperformPart` varchar(255) DEFAULT NULL,
`_select_time` varchar(255) DEFAULT NULL,
`_update_time` varchar(255) DEFAULT NULL,
`_version` varchar(255) DEFAULT NULL,
KEY `id` (`id`) USING BTREE,
KEY `iname` (`iname`) USING BTREE,
KEY `areaName` (`areaName`) USING BTREE,
KEY `cardNum` (`cardNum`) USING BTREE
)
3.代码
<?php
namespace Home\Controller;
use Curl\Curl;
use Think\Controller;
class ShixinController extends Controller
{
public function __construct()
{
require "./vendor/autoload.php";
}
/**
* 根据姓名获取数据
* @param string $iname
* @param int $pn
* @return mixed
* @throws \ErrorException
*/
public function getData($iname="测试",$pn=0)
{
$curl = new Curl();
$url = 'https://sp0.baidu.com/8aQDcjqpAAV3otqbppnN2DJv/api.php?';
$userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36';
$referer = "https://www.baidu.com/s?wd=%E5%A4%B1%E4%BF%A1%E4%BA%BA&rsv_spt=1&rsv_iqid=0xd609bc53000007a2&issp=1&f=8&rsv_bp=1&rsv_idx=2&ie=utf-8&rqlang=cn&tn=baiduhome_pg&rsv_enter=1&oq=%25E4%25BA%25BA%25E4%25BA%25BA%25E7%25BD%2591&inputT=4103&rsv_t=284b4GKK9AtCD7tiOTYP1zvlPucgiWINKuuwdhMhPwufPfGhspZJaAnNGqS4bjC9fL2e&rsv_sug3=29&rsv_sug1=27&rsv_sug7=100&rsv_pq=dadd35d50000105a&rsv_sug2=0&rsv_sug4=4817";
$params = array(
'pn'=> strval($pn*10),
'resource_id'=> '6899',
'query'=> '失信被执行人名单',
'iname'=> $iname,
'rn'=> '10',
'ie'=>'utf-8',
'oe'=> 'utf-8',
'format'=> 'json',
'from_mid'=>1,
'_'=>time().rand(100,999)
);
$curl->setUserAgent($userAgent);
$curl->setReferer($referer);
$curl->get($url,$params);
if ($curl->error) {
echo 'Error: ' . $curl->errorCode . ': ' . $curl->errorMessage . "\n";
} else {
// echo 'Response:' . "\n";
// print_r($curl->response->data[0]->disp_data);
$response = $curl->response->data[0]->disp_data; //返回的结果
// $dispNum = $curl->response->data[0]->dispNum; //这个name的总数
$response =json_decode( json_encode( $response),true);
// print_r($response);
return $response;
}
}
public function shixin()
{
$iname = "测试";
$num = $this->getnum($iname);
echo '共' .$num. '条数据<br>';
$pn = ceil($num/10);
echo '共' .$pn. '页<br>';
$result = array();
for ($i=0;$i<$pn;$i++)
{
echo '爬取第' .($i +1). '页<br>';
$result = $this->getData($iname,$i);
try {
$this->addData($result);
}catch (\Exception $e){
echo $e->getMessage();
}
}
}
/**
* 获取总数
* @param $iname
* @param int $pn
* @return bool
* @throws \ErrorException
*/
public function getnum($iname,$pn=0)
{
$url = 'https://sp0.baidu.com/8aQDcjqpAAV3otqbppnN2DJv/api.php?';
$referer = 'https://www.baidu.com/s?wd=%E5%A4%B1%E4%BF%A1%E4%BA%BA&rsv_spt=1&rsv_iqid=0xd609bc53000007a2&issp=1&f=8&rsv_bp=1&rsv_idx=2&ie=utf-8&rqlang=cn&tn=baiduhome_pg&rsv_enter=1&oq=%25E4%25BA%25BA%25E4%25BA%25BA%25E7%25BD%2591&inputT=4103&rsv_t=284b4GKK9AtCD7tiOTYP1zvlPucgiWINKuuwdhMhPwufPfGhspZJaAnNGqS4bjC9fL2e&rsv_sug3=29&rsv_sug1=27&rsv_sug7=100&rsv_pq=dadd35d50000105a&rsv_sug2=0&rsv_sug4=4817';
$userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36';
$params = array(
'pn' => $pn * 10,
'resource_id' => '6899',
'query' => '失信被执行人名单',
'iname' => $iname,
'rn' => '10',
'ie' => 'utf-8',
'oe' => 'utf-8',
'format' => 'json'
);
$curl = new Curl();
$curl->setUserAgent($userAgent);
$curl->setReferer($referer);
$curl->get($url,$params);
$response = $curl->response;
if ($curl->error) {
echo 'Error: ' . $curl->errorCode . ': ' . $curl->errorMessage . "\n";
} else {
if ($curl->response->data[0]){
$dispNum = $curl->response->data[0]->dispNum; //这个name的总数
return $dispNum;
}else{
// echo "没有数据";
return false;
}
}
}
/**
* 数据库添加数据
* @param $response
*/
public function addData($response)
{
foreach ($response as $value) {
$locid = explode("=", $value['loc'])[1];
$record = M('shixin')->where(array('id'=>$locid))->find();
if ($record)
{
echo "数据已经存在";
}
$data = array(
'id' => $locid,
'loc_id' => $locid,
'SiteId' => $value['SiteId'],
'StdStg' => $value['StdStg'],
'StdStl' => $value['StdStl'],
'age' => $value['age'],
'areaName' => $value['areaName'],
'businessEntity' => $value['businessEntity'],
'cambrian_appid' => $value['cambrian_appid'],
'cardNum' => $value['cardNum'],
'caseCode' => $value['caseCode'],
'changefreq' => $value['changefreq'],
'courtName' => $value['courtName'],
'disruptTypeName' => $value['disruptTypeName'],
'duty' => $value['duty'],
'focusNumber' => $value['focusNumber'],
'gistId' => $value['gistId'],
'gistUnit' => $value['gistUnit'],
'iname' => $value['iname'],
'lastmod' => $value['lastmod'],
'loc' => $value['loc'],
'partyTypeName' => $value['partyTypeName'],
'performance' => $value['performance'],
'performedPart' => $value['performedPart'],
'priority' => $value['priority'],
'publishDate' => $value['publishDate'],
'publishDateStamp' => $value['publishDateStamp'],
'regDate' => $value['regDate'],
'sexy' => $value['sexy'],
'sitelink' => $value['sitelink'],
'type' => $value['type'],
'unperformPart' => $value['unperformPart'],
'_select_time' => $value['_select_time'],
'_update_time' => $value['_update_time'],
'_version' => $value['_version']
);
M('shixin')->add($data);
}
}
}
4.数据
