URL抓取工具

有需要csdn免積分下載、pudn免積分下載、51cto免積分，請到http://www.itziy.com/
命令列下執行，直接php呼叫將顯示使用方式
功能說明
1.支援代理
2.支援設定遞歸檢查次數
3.支援輸出類型控制、檢查內容控制

作用：
主要代替肉眼盡量多的抓取可能的請求包及url位址等，方便滲透測試

error_reporting(E_ERROR | E_WARNING | E_PARSE);
ini_set('memory_limit','1024M');ini_set('memory_limit','1024M');
define('CHECK_A_TAG', false);
define('CHECK_JS_TAG', true);
define('CHECK_URL', true);
define('SAVE_ERROR', );
$checkArr = array(
 '$.load',
 '.ajax',
 '$.post',
 '$.get',
 '.getJSON'
);

if ($argc  die(showerror('sorry, parameter error', array('example: php debug.php url num filename header proxy', 'detail information:', 'url: target url address which you want to check it', 'num: The number of pages of recursive,default 3', 'filename: output filename default name ret.txt', 'header: The request header file default null', 'proxy: if you want to use proxy set it here default no use proxy')));

if (!check_extension())
 die(showerror('extension!check_extension())
 die(showerror('extension | support', 'please open php curl extension support'));

//global variable
$url = trim($argv[1]);
if (stripos($url, ' http') === false)
 $url = 'http://'.$url;
$num = isset($argv[2]) ? intval($argv[2]) : 3;
$output = isset($argv[3]) ? trim(str_replace("\", '/', $argv[3])) : str_replace("\", '/', dirname(__FILE__)) .'/ret.txt';
$header = null;
$proxy = null;
$host = null;

if (isset($argv[4]))
{
 $header = trim(str_replace("\", '/', $argv[4]));
 if (file_exists($header))
 $header = array_filter(explode( "n", str_replace("r", '', file_get_contents($header)))));
 else
 {
 $file = str_replace("\", '/', dirname(__FILE__) ).'/'.$header;
 if (file_exists($file))
 $header = array_filter(explode("n", str_replace("r", '', file_get_contents($file))) );
 else
 $header = null;
 }
}

if (isset($argv[5]))
 $proxy = trim($argv[ 5]);

if (!is_array($header) || empty($header))
 $header = null;
$result = check_valid_url($url);
$$ outputArr = array();

if (!empty($result))
{
 $result = str_replace("r", '', $result);
 $result = str_replace("n", '', $result);
 $tmpArr = parse_url($url);
 if (!isset($tmpArr['host']))
 die(showerror(' parse url error', 'can not get host form url: '.$url));
 $host = $tmpArr['host'];
 if (stripos($host, 'http') == = false)
 $host = 'http://'.$host;
 unset($tmpArr);

 //check for current page
 if (!isset($outputArr [md5($url)]))
 {
 $outputArr[md5($url)] = $url;
 file_put_contents($output, $url."n", FILE_APPEND);
 echo 'url: ',$url,' find ajax require so save it',PHP_EOL;
 }
 work($result);
}
echo 'run finish',PHP_EOL;
}
echo 'run finish',PHP_EOL;

function work($result, $reverse = false)
{
 global $num, $host, $outputArr, $checkArr, $output;

 if (!$result)
 return;
 $result = str_replace("r", '', $result);
 $result = str_replace("n", '', $result);

 while ($num > 0)
 {
 echo 'remain: ',$num,' now start to check for url address',PHP_EOL,PHP_EOL;
 preg_match_all('//i', $result, $match);
 if (CHECK_A_TAG && isset($match[2 ]) && !empty($match[2]))
 {
 foreach ($match[2] as $mc)
 {
 $mc = trim($mc);
 if ($mc == '#')
 continue;
 if (stripos($mc, 'http') === false)
 $mc = $host.$mc;

 if (($ret = check_valid_url($mc)))
 {
 if (!isset($outputArr[md5($mc)]))
 {
 $outputArr[md5( $mc)] = $mc;
 file_put_contents($output, $mc."n", FILE_APPEND);
 echo 'url: ',$mc,' find ajax require so save it',PHP_EOL;
 }
 }
 }
 }

 //check for page url
 echo 'remain: ',$num,' now start to check for page url',PHP_EOL ,PHP_EOL; preg_match_all('/(https?|ftp|mms)://([A-z0-9] [_-]?[A-z0-9] .)*[A-z0-9 ] -?[A-z0-9] .[A-z]{2,}(/.*)*/?/i', $result, $match);
 if (CHECK_URL && isset($match[2]) && !empty($match[2]))
 {
 foreach ($match[2] as $mc)
 {
 $mc = trim($mc);
 if ($mc == '#')
 continue;
 if (stripos($mc, 'http') === false)
 $mc = $host.$mc;

 if (($ret = check_valid_url($mc)))
 {
 if (!isset($outputArr[md5($mc)]))
 {
 $outputArr[md5($mc)] = $mc;
 file_put_contents($output, $mc ) ) ."n", FILE_APPEND);
 echo 'url: ',$mc,' 找 ajax 需要，保存因此',PHP_EOL;
 }
 }
 }
 }

 //檢查javascript ajax require
 echo 'remain: ',$num,' 現在開始檢查javascript ajax require',PHP_EOL,PHP_EOL;
 preg_match_all('//i', $result, $match);
 if (CHECK_JS_TAG && isset($ match[2 ]) && !empty($match[2]))
 {
 foreach ($match[2] as $mc)
 {
 $mc = trim($mc);
 if ($mc == '#')
 continue;
 if (stripos($mc, 'http') === false)
 $mc = $host.$mc;

 if (($ret = check_valid_url($mc)))
 {
 //檢查目前頁
 foreach ($checkArr as $ck)
 {
 if ( !isset($outputArr[md5($mc)]) && strpos($ret, $ck) !== false)
 {
 $outputArr[md5($mc)] = $mc;
 file_put_contents($output, $mc."n", FILE_APPEND);
 echo 'url: ',$mc,' 找到ajax require 所以保存它',PHP_EOL;
 break;
 }
 }
 }
 }
 }

 if ($reverse)
 return;

 //檢查下一頁
 preg_match_all('//i', $結果, $match);
 if ( isset( $match[2]) && !empty($match[2]))
 {
 echo '檢查下一頁，剩餘頁數：',$num,PHP_EOL;
 foreach ($ match[ 2] as $mc)
 {
 $mc = trim($mc);
 if ($mc == '#')
 continue;
 if (stripos($ mc, 'http') === false)
 $mc = $host.$mc;
 echo '檢查下一頁：',$mc,PHP_EOL;
 work(check_valid_url($mc) , true );
 }
 }
 $num--;
 sleep(3);
 }
}

函數check_valid_url($url)
{
 if (stripos($url, 'http') === false)
 $url = 'http://'.$url;
 $ch = curl_init();
curl_setopt( $ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_HEADER, true);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_togffchUR CURS,$
curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (相容;Google 機器人/2.1;http://www.google.com/bot.html)');
 if (!is_null($header) )
curl_setopt($ch, CURLOPT_HTTPHEADER, $header);
 if (!is_null ($proxy))
curl_setopt($ch, CURLOPT_PROXY, $proxy);
 $pret. ch);
 $errinfo = curl_error($ch);
curl_close($ch) );
 unset($ch);
 if (!empty($errinfo) || ((strpos ) ($ret, '200 OK') === false) && (strpos($ret, '302)移動') ===已已)) || strpos($ret, '114so.cn') !== false)
 {
 showerror('檢查 url: '.$url.' 發現一些錯誤' , array($errinfo, $ret));
 if (SAVE_ERROR)
 file_put_contents(dirname(__FILE__).'/error.txt', $url."n", FILE_APPEND);
 return false ;
 }
 }
 return false ;
 }
 }
 return $ret ;
}

function check_extension()
{
 if (!function_exists('curl_init') || !extension_loaded('curl' )))
 返回false>回傳true;
}

function showerror($t, $c)
{
 $str = "####### ############ ########################################################### #######n";
 $str .= "# ".$t."n";
 if (is_string($c))
 $str .= "# "。 $c;
 elseif (is_array($c) && !empty($c))
 {
 foreach ($c as $c1)
 $ str .= "# ".$c1. "n";
 }
 $str .= "n################################ # ##########################################n";

複製程式碼