Home > Backend Development > PHP Tutorial > Use curl and regular expressions to crawl web data

Use curl and regular expressions to crawl web data

WBOY
Release: 2016-07-25 08:48:28
Original
1488 people have browsed it
Using curl and regular expressions, we built a novel grabber for non-VIP chapters of the Motie Chinese website. It supports inputting the novel ID to download the novel.
Dependencies: curl
You can take a brief look at it. Curl, regular expressions, ajax and other technologies are used in it, which is suitable for novices.When testing locally, you must ensure that you are connected to the Internet and make sure that PHP turns on curl mode.
  1. session_start();
  2. //Encapsulate into a class to start these automatically crawled articles
  3. #header("Refresh:30;http://www.test.com:8080");
  4. class SpiderTools{
  5. ///////////////////////////////////////////////// ////////////////////////////////////////////////////// //////////
  6. /*Input the article ID and parse out the article title*/
  7. ///////////////////////////// ////////////////////////////////////////////////////// //////////////////////////////
  8. public function getBookNameById($aid){
  9. //Initialize curl
  10. $ch= curl_init() ;
  11. //url
  12. $url='http://www.motie.com/book/'.$aid;
  13. if(is_numeric($aid)){
  14. //Regular expression matching
  15. $ru="/ s*(.*)s*s*/";
  16. }
  17. else {
  18. //The family’s survival in the zombie outbreak_The first chapter of the zombie outbreak is updated for my friend Ai Leer~_Sharpening Iron
  19. $ru="/(.*) /";
  20. }
  21. //Set options, including URL
  22. curl_setopt($ch, CURLOPT_URL, $url);
  23. curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);//Do not automatically output content
  24. curl_setopt( $ch, CURLOPT_HEADER, 0); //Does not return header information
  25. curl_setopt($ch, CURLOPT_CONNECTTIMEOUT_MS, 0);
  26. //Execute curl
  27. $output = curl_exec($ch);
  28. //Error message
  29. if(curl_exec ($ch) === false){
  30. die(curl_error($ch));
  31. }
  32. // Check if an error occurs
  33. if(curl_errno($ch)){
  34. echo 'Curl error: ' . curl_error( $ch);
  35. }
  36. //Release curl handle
  37. curl_close($ch);
  38. $arr=array();
  39. preg_match_all($ru,$output,$arr);
  40. return $arr[1][0] ;
  41. }
  42. //////////////////////////////////////////////// ////////////////////////////////////////////////////// //////////
  43. /*Input the article ID to parse the article content*/
  44. ///////////////////////////// ////////////////////////////////////////////////////// ///////////////////////////////
  45. public function getBookContextById($aid){
  46. //Start parsing the article
  47. $ids=array( );
  48. $ids=explode("_",$aid);
  49. $titleId=trim($ids[0]);
  50. $aticleId=trim($ids[1]);
  51. $ch= curl_init();
  52. $ru="/
    [sS]*
    [sS]*(.*)/ui"; 
  53. $url='http://www.motie.com/book/'.$aid;
  54. //Regular expression matching
  55. //Set options, including URL
  56. curl_setopt($ch, CURLOPT_URL, $url);
  57. curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);//Do not automatically output content
  58. curl_setopt($ch, CURLOPT_HEADER, 0);//Do not return header information
  59. curl_setopt($ch, CURLOPT_CONNECTTIMEOUT_MS, 0 );
  60. //Execute curl
  61. $output = curl_exec($ch);
  62. //Error message
  63. if(curl_exec($ch) === false){
  64. die(curl_error($ch));
  65. }
  66. / / Check if an error occurred
  67. if(curl_errno($ch)){
  68. echo 'Curl error: ' . curl_error($ch);
  69. }
  70. $arr=array();
  71. $arr2=array();
  72. preg_match_all ($ru,$output,$arr);
  73. curl_close($ch);
  74. #var_dump($arr);
  75. $s=$arr[0][0];
  76. $s=substr($s,180) ;
  77. $arr2=explode(" return trim($arr2[0]);
  78. }
  79. ////////////////// ////////////////////////////////////////////////////// /////////////////////////////////////////
  80. /*Static method@generated novel file can be called directly */
  81. ////////////////////////////////////////////////// ////////////////////////////////////////////////////// /////////
  82. public static function createBookById($id){
  83. if(!is_numeric($id)){
  84. echo "
    INIT BEGIN START WRITE!";
  85. $ st=new self();
  86. $cons=$st->getBookContextById($id);
  87. $title=$st->getBookNameById($id);
  88. $cons=trim($cons);
  89. $t =explode(" ",$title);
  90. //Construct directory
  91. $dir=array();
  92. $dir=explode("_",$t[0]);
  93. $wzdir=$dir[0]; //Book name as directory name
  94. $wzchapter=$dir[1]; //Chapter
  95. //Create directory
  96. $wzdir2=iconv("UTF-8", "GBK", $wzdir); //Directory When encoding, please note that the reference to the $wzdir string is retained here, which is used to construct the file name. It cannot be used here to prevent secondary encoding
  97. if(!file_exists($wzdir2)){
  98. mkdir($wzdir2); //Create a directory
  99. }
  100. //Construct the file name
  101. $wztitle="./".$wzdir."/"."$t[0]".".txt";
  102. //Ensure that the saved file name is not garbled
  103. $wztitle=iconv ("UTF-8", "GBK", $wztitle);
  104. $f=fopen($wztitle,"w+");
  105. fwrite($f,$cons);
  106. echo "$wzdir ".$wzchapter."Write successfully";
  107. fclose($f);
  108. }
  109. else{
  110. $ ids=self::getBookIdsById($id);
  111. //The server may be offline here, so it is best to use session recording loop
  112. #for($i=$_SESSION["$id"."_fid"];$ i<=count($ids);$_SESSION["$id"."_fid"]++,$i++){
  113. #self::createBookById($id."_".$ids[$_SESSION[" $id"."_fid"]++]);//Construct id
  114. #}
  115. for($i=$_SESSION["$id"."_fid"];$i<=count($ids); $_SESSION["$id"."_fid"]++,$i++){
  116. self::createBookById($id."_".$ids[$i]);//Construct id
  117. }
  118. # echo "


    The writing work is completed

    ";
  119. #echo $id."_".$ids[0 ]."
    ";
  120. #var_dump($ids);
  121. }
  122. }
  123. /*
  124. Get all the IDs of the novel
  125. @param $id article ID
  126. @return array;
  127. */
  128. public static function getBookIdsById($aid){
  129. $ch= curl_init();
  130. $url='http://www.motie.com/book/'.$aid."/chapter";
  131. //Note here ?The minimum matching items can be obtained
  132. $ru='/[sS]*?
  133. [sS]*?.*?. *?/u';//Regular expression matching
  134. //Set options, including URL
  135. curl_setopt($ch, CURLOPT_URL, $url);
  136. curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);//Do not automatically output content
  137. curl_setopt($ch, CURLOPT_HEADER, 0); //Does not return header information
  138. curl_setopt($ch, CURLOPT_CONNECTTIMEOUT_MS, 0);
  139. //Execute curl
  140. $output = curl_exec($ch);
  141. // Check if there are any errors Occurs
  142. if(curl_errno($ch)){
  143. echo 'Curl error: ' . curl_error($ch);
  144. }
  145. //Release curl handle
  146. curl_close($ch);
  147. $arr=array();
  148. preg_match_all ($ru,$output,$arr,PREG_PATTERN_ORDER);
  149. return $arr[1];
  150. }
  151. }
  152. ?>
Copy code
  1. session_start();
  2. require_once("SpiderTools.class.php");
  3. if($_REQUEST["bid"]){
  4. if(is_numeric($_REQUEST["bid"]) ){
  5. SpiderTools::createBookById(trim($_REQUEST["bid"]));
  6. }
  7. else{
  8. echo "
    Please enter the correct article ID
    ";
  9. }
  10. }
  11. ?>
Copy code
  1. Download the novel
  2. Enter the ID number of the novel you want to see on the Motie Chinese website to download the novel

  3. < /form>
  • Copy code


    source:php.cn
    Statement of this Website
    The content of this article is voluntarily contributed by netizens, and the copyright belongs to the original author. This site does not assume corresponding legal responsibility. If you find any content suspected of plagiarism or infringement, please contact admin@php.cn
    Popular Tutorials
    More>
    Latest Downloads
    More>
    Web Effects
    Website Source Code
    Website Materials
    Front End Template