To extract part of the data from Google search, I found that Google blocks the software from capturing its data. In the past, I could forge USER-AGENT to capture the data, but now it doesn’t work. Using packet capture data, we found that Google has determined cookies. When you do not have cookies, it will directly return 302 jumps, and there are dozens of 302 jumps in a row, and no data can be captured at all.
Therefore, when sending a search command, you need to extract and save the cookies first, and then use the saved cookie to send the search command again to capture the data normally. This is actually the same as the simulated login of the forum. You need to POST to log in first, get the cookies and save them, and then use the cookies to access.
$cookie_jar = dirname(__FILE__)."/pic.cookie";
$url = "http://1.2.3.4/"; $ch = curl_init(); curl_setopt($ch, CURLOPT_URL, $url); curl_setopt($ch, CURLOPT_HEADER, 0); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); curl_setopt($ch, CURLOPT_COOKIEJAR, $cookie_jar); $content = curl_exec($ch); curl_close($ch);
$ch = curl_init(); curl_setopt($ch, CURLOPT_URL, 'http://1.2.3.4/getCheckpic.action?rand=6836.185874812305'); curl_setopt($ch, CURLOPT_COOKIEFILE, $cookie_jar); curl_setopt($ch, CURLOPT_HEADER, 0); curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); $ret = curl_exec($ch); curl_close($ch);
$post = "name=2&userType=1&passwd=asdf&loginType=1&rand=6836&imageField.x=25&imageField.y=7"; $ch = curl_init(); curl_setopt($ch, CURLOPT_URL, "http://1.2.3.4/loginstudent.action"); curl_setopt($ch, CURLOPT_HEADER, false); curl_setopt($ch, CURLOPT_RETURNTRANSFER,1); curl_setopt($ch, CURLOPT_POSTFIELDS, $post); curl_setopt($ch, CURLOPT_COOKIEFILE, $cookie_jar); $result=curl_exec($ch); curl_close($ch);
$ch = curl_init(); curl_setopt($ch, CURLOPT_URL, "http://1.2.3.4/accountcardUser.action"); curl_setopt($ch, CURLOPT_HEADER, false); curl_setopt($ch, CURLOPT_HEADER, 0); curl_setopt($ch, CURLOPT_RETURNTRANSFER,0); curl_setopt($ch, CURLOPT_COOKIEFILE, $cookie_jar); $html=curl_exec($ch); // var_dump($html); curl_close($ch);