PHP로 구현된 간단한 크롤러-PHP 튜토리얼-php.cn

PHP로 구현된 간단한 크롤러

WBOY

풀어 주다： 2016-07-29 08:58:30

원래의

980명이 탐색했습니다.

이 소형 크롤러의 기능은 대상 웹페이지의 URL을 크롤링하고 재귀 크롤링을 구현하는 것입니다. 이 작은 데모는 네티즌들의 코드를 기반으로 하고 직접 수정한 것입니다. 온라인 버전이 너무 많기 때문에 원본 작성자는 @하지 않겠습니다. (진짜 작성자가 누구인지는 모르겠습니다.)

코드는 다음과 같습니다.

<code><span><?php</span><span>//爬虫类</span><span><span>class</span><span>Crawler</span>{</span><span>private</span><span>$url</span>;
    <span>public</span><span><span>function</span><span>__construct</span><span>(<span>$url</span>)</span>{</span><span>if</span>(!preg_match(<span>"/^(http)s?/"</span>, <span>$url</span>)){
            <span>$url</span> = <span>"http://"</span>.<span>$url</span>;
        }
        <span>$this</span>->url = <span>$url</span>;
    }
    <span>//从给定的url中获取html内容</span><span>protected</span><span><span>function</span><span>_getUrlContent</span><span>(<span>$url</span>)</span>{</span>
        @<span>$handle</span> = fopen(<span>$url</span>, <span>"r"</span>);
        <span>if</span>(error_get_last()){<span>//捕获异常（不一定是错误）</span><span>$err</span> = <span>new</span><span>Exception</span>(<span>"你的URL好像不对！要不换一个？"</span>);
            <span>echo</span><span>$err</span>->getMessage();
            <span>return</span>;
        }
        <span>if</span>(<span>$handle</span>){
            <span>$content</span> = stream_get_contents(<span>$handle</span>,<span>1024</span>*<span>1024</span>);<span>//将资源流读入字符串</span><span>return</span><span>$content</span>;
        }<span>else</span>{
            <span>return</span><span>false</span>;
        }   
    }
    <span>//从html内容中筛选链接</span><span>protected</span><span><span>function</span><span>_filterUrl</span><span>(<span>$web_content</span>)</span>{</span><span>$reg_tag_a</span> = <span>'/<[a|A].*?href=[\'\"]{0,1}([^>\'\"\ ]*).*?>/'</span>;
        <span>$result</span> = preg_match_all(<span>$reg_tag_a</span>,<span>$web_content</span>,<span>$match_result</span>);
        <span>if</span>(<span>$result</span>){
            <span>return</span><span>$match_result</span>[<span>1</span>];
        }
    }
    <span>//判断是否是完整的url</span><span>protected</span><span><span>function</span><span>_judgeURL</span><span>(<span>$url</span>)</span>{</span><span>$url_info</span> = parse_url(<span>$url</span>);
        <span>if</span>(<span>isset</span>(<span>$url_info</span>[<span>'scheme'</span>])||<span>isset</span>(<span>$url_info</span>[<span>'host'</span>])){
            <span>return</span><span>true</span>;
        }
        <span>return</span><span>false</span>;
    }
    <span>//修正相对路径</span><span>protected</span><span><span>function</span><span>_reviseUrl</span><span>(<span>$base_url</span>,<span>$url_list</span>)</span>{</span><span>$url_info</span> = parse_url(<span>$base_url</span>);<span>//分解url中的各个部分</span><span>unset</span>(<span>$base_url</span>);
        <span>$base_url</span> = <span>isset</span>(<span>$url_info</span>[<span>"scheme"</span>])?<span>$url_info</span>[<span>"scheme"</span>].<span>'://'</span>:<span>""</span>;<span>//$url_info["scheme"]为http、ftp等</span><span>if</span>(<span>isset</span>(<span>$url_info</span>[<span>"user"</span>]) && <span>isset</span>(<span>$url_info</span>[<span>"pass"</span>])){<span>//记录用户名及密码的url</span><span>$base_url</span> .= <span>$url_info</span>[<span>"user"</span>].<span>":"</span>.<span>$url_info</span>[<span>"pass"</span>].<span>"@"</span>;
        }
        <span>$base_url</span> .= <span>isset</span>(<span>$url_info</span>[<span>"host"</span>])?<span>$url_info</span>[<span>"host"</span>]:<span>""</span>;<span>//$url_info["host"]域名</span><span>if</span>(<span>isset</span>(<span>$url_info</span>[<span>"port"</span>])){<span>//$url_info["port"]端口，8080等</span><span>$base_url</span> .= <span>":"</span>.<span>$url_info</span>[<span>"port"</span>];
        }
        <span>$base_url</span> .= <span>isset</span>(<span>$url_info</span>[<span>"path"</span>])?<span>$url_info</span>[<span>"path"</span>]:<span>""</span>;<span>//$url_info["path"]路径</span><span>//目前为止，绝对路径前面已经组装完</span><span>if</span>(is_array(<span>$url_list</span>)){
            <span>foreach</span> (<span>$url_list</span><span>as</span><span>$url_item</span>) {
                <span>// if(preg_match('/^(http)s?/',$url_item)){</span><span>if</span>(<span>$this</span>->_judgeURL(<span>$url_item</span>)){
                    <span>//已经是完整的url</span><span>$result</span>[] = <span>$url_item</span>;
                }<span>else</span> {
                    <span>//不完整的url</span><span>$real_url</span> = <span>$base_url</span>.<span>$url_item</span>;
                    <span>$result</span>[] = <span>$real_url</span>;
                }
            }
            <span>return</span><span>$result</span>;
        }<span>else</span> {
            <span>return</span>;
        }
    }
    <span>//爬虫</span><span>public</span><span><span>function</span><span>crawler</span><span>()</span>{</span><span>$content</span> = <span>$this</span>->_getUrlContent(<span>$this</span>->url);
        <span>if</span>(<span>$content</span>){
            <span>$url_list</span> = <span>$this</span>->_reviseUrl(<span>$this</span>->url,<span>$this</span>->_filterUrl(<span>$content</span>));
            <span>if</span>(<span>$url_list</span>){
                <span>return</span><span>$url_list</span>;
            }<span>else</span> {
                <span>return</span> ;
            }
        }<span>else</span>{
            <span>return</span> ;
        }
    }
}


<span>$fp_puts</span> = fopen(<span>"url.txt"</span>,<span>"ab"</span>);<span>//记录url列表</span><span>$fp_gets</span> = fopen(<span>"url.txt"</span>,<span>"r"</span>);<span>//保存url列表</span><span>$current_url</span> = <span>"www.baidu.com"</span>;
<span>do</span>{
    <span>$Crawler</span> = <span>new</span> Crawler(<span>$current_url</span>);
    <span>$url_arr</span> = <span>$Crawler</span>->crawler();
    <span>if</span>(<span>$url_arr</span>){
        <span>foreach</span> (<span>$url_arr</span><span>as</span><span>$url</span>) {
            fputs(<span>$fp_puts</span>,<span>$url</span>.<span>"\n"</span>);
        }
    }
}<span>while</span> (<span>$current_url</span> = fgets(<span>$fp_gets</span>,<span>1024</span>));<span>//不断获得url</span><span>// echo "<pre class="brush:php;toolbar:false">";</span><span>// var_dump($url_arr);</span><span>// echo "<pre/>";</span><span>?></span></span></code>

로그인 후 복사

루프 중에 새로운 개체가 많을 수 있으므로 당시에는 너무 많은 메모리 오버헤드를 피하기 위해 싱글톤 모드를 사용하는 것이 옳다고 생각했습니다. 나중에 발견했습니다. 귀찮아서 그냥 놔뒀어요. . . .

').addClass('사전 번호 매기기').hide(); $(this).addClass('has-numbering').parent().append($numbering); for (i = 1; i ').text(i)); }; $numbering.fadeIn(1700); }); });

위 내용은 PHP로 구현된 간단한 크롤러를 다양한 측면을 포함하여 소개하고 있으며, PHP 튜토리얼에 관심이 있는 친구들에게 도움이 되기를 바랍니다.