PHP implements recursive crawling of web page class instances_PHP tutorial

WBOY
Release: 2016-07-13 09:58:13
Original
974 people have browsed it

php implements recursive crawling of web page class instances

The details are as follows:

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

class crawler{

private $_depth=5;

private $_urls=array();

function extract_links($url)

{

if(!$this->_started){

$this->_started=1;

$curr_depth=0;

}else{

$curr_depth ;

}

if($curr_depth<$this->_depth)

{

$data=file_get_contents($url);

if(preg_match_all('/((?:http|https)://(?:www.)*(?:[a-zA-Z0-9_-]{1,15}. [a-zA-Z0-9_]{1,}){1,}(?:[a-zA-Z0-9_/.-?&:%,!;]*))/',$data,$urls12))

{

foreach($urls12[0] as $k=>$v){

$check=get_headers($v,1);

if(strstr($v,$url) && $check[0]=='HTTP/1.1 200 OK' && !array_search($v,$this->_urls) && $curr_depth<$this->_depth){

$this->_urls[]=$v;

$this->extract_links($v);

}

}

}

}

return $this->_urls;

}

}

?>

1 2 3

4

6 7 8 9 10
11 12
13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
<🎜>class crawler{<🎜> <🎜>private $_depth=5;<🎜> <🎜>private $_urls=array();<🎜> <🎜>function extract_links($url)<🎜> <🎜>{<🎜> <🎜>if(!$this->_started){ $this->_started=1; $curr_depth=0; }else{ $curr_depth ; } if($curr_depth<$this->_depth) { $data=file_get_contents($url); if(preg_match_all('/((?:http|https)://(?:www.)*(?:[a-zA-Z0-9_-]{1,15}. [a-zA -Z0-9_]{1,}){1,}(?:[a-zA-Z0-9_/.-?&:%,!;]*))/',$data,$urls12)) { foreach($urls12[0] as $k=>$v){ $check=get_headers($v,1); if(strstr($v,$url) && $check[0]=='HTTP/1.1 200 OK' && !array_search($v,$this->_urls) && $curr_depth<$this-> ;_depth){<🎜> <🎜>$this->_urls[]=$v; $this->extract_links($v); } } } } return $this->_urls; } } ?>
http://www.bkjia.com/PHPjc/978265.htmlwww.bkjia.comtruehttp: //www.bkjia.com/PHPjc/978265.htmlTechArticlePHP implementation of recursive crawling web page class examples are as follows: 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 ?php class crawler{ private $_depth=5; private...
source:php.cn
Statement of this Website
The content of this article is voluntarily contributed by netizens, and the copyright belongs to the original author. This site does not assume corresponding legal responsibility. If you find any content suspected of plagiarism or infringement, please contact admin@php.cn
Popular Tutorials
More>
Latest Downloads
More>
Web Effects
Website Source Code
Website Materials
Front End Template
About us Disclaimer Sitemap
php.cn:Public welfare online PHP training,Help PHP learners grow quickly!