QueryList采集器開發(fā)手冊
/ Multi 多線程插件
Multi 多線程插件
Multi 多線程插件
Multi擴(kuò)展，可以實(shí)現(xiàn)多線程采集。
安裝:
composer require jaeger/querylist-ext-multi
GIT地址:
https://github.com/jae-jae/QueryList-Ext-Multi.git
依賴（通過Composer安裝的請忽略）
Multi擴(kuò)展依賴CurlMulti類,Git地址為:https://github.com/jae-jae/CurlMulti
用法一
<?php
/**
* 下面實(shí)現(xiàn)多線程采集文章信息
*/
use QL\QueryList;
//多線程擴(kuò)展
QueryList::run('Multi',[
//待采集鏈接集合
'list' => [
'http://cms.querylist.cc/news/it/547.html',
'http://cms.querylist.cc/news/it/545.html',
'http://cms.querylist.cc/news/it/543.html'
//更多的采集鏈接....
],
'curl' => [
'opt' => array(
//這里根據(jù)自身需求設(shè)置curl參數(shù)
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_SSL_VERIFYHOST => false,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_AUTOREFERER => true,
//........
),
//設(shè)置線程數(shù)
'maxThread' => 100,
//設(shè)置最大嘗試數(shù)
'maxTry' => 3
],
'success' => function($a){
//采集規(guī)則
$reg = array(
//采集文章標(biāo)題
'title' => array('h1','text'),
//采集文章正文內(nèi)容,利用過濾功能去掉文章中的超鏈接，但保留超鏈接的文字，并去掉版權(quán)、JS代碼等無用信息
'content' => array('.post_content','html','a -.content_copyright -script' )
);
$rang = '.content';
$ql = QueryList::Query($a['content'],$reg,$rang);
$data = $ql->getData();
//打印結(jié)果，實(shí)際操作中這里應(yīng)該做入數(shù)據(jù)庫操作
print_r($data);
}
]);用法二
<?php
require 'QueryList/vendor/autoload.php';
use QL\QueryList;
//多線程擴(kuò)展
$cm = QueryList::run('Multi',[
//待采集鏈接集合
'list' => [
'http://cms.querylist.cc/news/it/547.html',
'http://cms.querylist.cc/news/it/545.html',
'http://cms.querylist.cc/news/it/543.html'
//更多的采集鏈接....
],
'curl' => [
'opt' => array(
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_SSL_VERIFYHOST => false,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_AUTOREFERER => true,
),
//設(shè)置線程數(shù)
'maxThread' => 100,
//設(shè)置最大嘗試數(shù)
'maxTry' => 3
],
//不自動開始線程，默認(rèn)自動開始
'start' => false,
'success' => function($html,$info){
//采集操作....
},
'error' => function(){
//出錯處理
}
]);
//再額外添加一些采集鏈接
$cm->add([
'http://cms.querylist.cc/news/it/532.html',
'http://cms.querylist.cc/news/it/528.html',
'http://cms.querylist.cc/news/other/530.html'
],function($html,$info){
//sucess
//可選的，不同的采集操作....
},
function(){
//error
//可選的，不同的出錯處理
});
//開始采集
$cm->start();用法三
<?php
require 'QueryList/vendor/autoload.php';
use QL\QueryList;
$url = 'http://www.phpddt.com/category/php/1/';
$curl = QueryList::getInstance('QL\Ext\Lib\CurlMulti');
//100個線程
$curl->maxThread = 100;
$data = QueryList::run('Request',array(
'http' =>array(
'target' => $url,
'referrer'=>$url,
'user_agent'=>'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Ubuntu/11.10 Chromium/27.0.1453.93 Chrome/27.0.1453.93 Safari/537.36',
'cookiePath' => './cookie.txt'
),
'callback' => function($html){
return preg_replace('/<head.+?>.+<\/head>/is','<head></head>',$html);
}
))->setQuery(array('title'=>['h2 a','text'],'link'=>['h2 a','href']))->getData(function($item) use($curl){
// if(!StudyModel::exist($item['title'])){
$curl->add(['url' => $item['link']],function($a){
$html = preg_replace('/<head.+?>.+<\/head>/is','<head></head>',$a['content']);
$data = QueryList::Query($html,array('title'=>['.entry_title','text'],'content'=>['.post','html','-#headline -script -h3.post_tags -.copyright -.wumii-hook a']))->getData();
// echo StudyModel::insert($data[0]['title'],$data[0]['content'],$a['info']['url']);
print_r($data);
});
// }
});
$curl->start();