PHP implements recursive crawling of web page class instances_PHP tutorial

php implements recursive crawling of web page class instances

The details are as follows:

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30

class crawler{ private $_depth=5; private $_urls=array(); function extract_links($url) { if(!$this->_started){ $this->_started=1; $curr_depth=0; }else{ $curr_depth ; } if($curr_depth<$this->_depth) { $data=file_get_contents($url); if(preg_match_all('/((?:http|https)://(?:www.)*(?:[a-zA-Z0-9_-]{1,15}. [a-zA-Z0-9_]{1,}){1,}(?:[a-zA-Z0-9_/.-?&:%,!;]*))/',$data,$urls12)) { foreach($urls12[0] as $k=>$v){ $check=get_headers($v,1); if(strstr($v,$url) && $check[0]=='HTTP/1.1 200 OK' && !array_search($v,$this->_urls) && $curr_depth<$this->_depth){ $this->_urls[]=$v; $this->extract_links($v); } } } } return $this->_urls; } } ?>

1 2 3

4

5 6 7 8 9 10

11 12

13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30

<🎜>class crawler{<🎜> <🎜>private $_depth=5;<🎜> <🎜>private $_urls=array();<🎜> <🎜>function extract_links($url)<🎜> <🎜>{<🎜> <🎜>if(!$this->_started){ $this->_started=1; $curr_depth=0; }else{ $curr_depth ; } if($curr_depth<$this->_depth) { $data=file_get_contents($url); if(preg_match_all('/((?:http|https)://(?:www.)*(?:[a-zA-Z0-9_-]{1,15}. [a-zA -Z0-9_]{1,}){1,}(?:[a-zA-Z0-9_/.-?&:%,!;]*))/',$data,$urls12)) { foreach($urls12[0] as $k=>$v){ $check=get_headers($v,1); if(strstr($v,$url) && $check[0]=='HTTP/1.1 200 OK' && !array_search($v,$this->_urls) && $curr_depth<$this-> ;_depth){<🎜> <🎜>$this->_urls[]=$v; $this->extract_links($v); } } } } return $this->_urls; } } ?>

http://www.bkjia.com/PHPjc/978265.htmlwww.bkjia.comtruehttp: //www.bkjia.com/PHPjc/978265.htmlTechArticlePHP implementation of recursive crawling web page class examples are as follows: 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 ?php class crawler{ private $_depth=5; private...