Capture the 'China IC Network' supplier program_PHP tutorial-PHP Tutorial-php.cn

/**
* Grab the "China IC Network (http://www.ic37.com)" supplier main program
* author Lee.
* Last modify $Date: 2012-2-9 9:32:21 $
* Note: This program is executed according to the encoding GB2312, because the "China IC Network" website is encoded in GB2312, and the database must also be consistent
*/
class ic37 {
    private $key; // 型号
    private $pageNum; // 页码

    /**
* Entry program
*/
    public function go($key) {
        $this->key = $key;
        $this->pageNum = $this->getPageNum();
        $this->getInfo();
    }

    /**
* Get the supplier url link array
* @return ArrayObject
*/
    private function getInfo() {
        if ($this->pageNum==1) { # 处理只有一页的情况
            $arr = $this->shopAddContact($this->shopUrlMatchReArr($this->getContent()));
            $this->isAddSuccess($arr);
        } elseif ($this->pageNum>1) { # 多页
            for ($i=1; $i<=$this->pageNum; $i++) {
                $arr = $this->shopAddContact($this->shopUrlMatchReArr($this->getContent($i)));
                $this->isAddSuccess($arr);
            }
        }
    }

    /**
* Print whether the addition was successful
* @param ArrayObject $arr
* @return string
*/
    private function isAddSuccess($arr) {
        foreach ($arr as $k=>$v) {
            if ($this->execAdd($this->getInfoByShopUrl($v))) {
                echo 'Add Success!!';
            } else {
                echo 'Add Faild!!';
            }
        }
    }

    /**
* Execute adding to database
* @param ArrayObject $infoArr
* @return Number The number of affected rows
*/
    private function execAdd($infoArr) {
        $mysqli = $this->getDb();
        if (!emptyempty($infoArr['company'])) {
            if (!$this->isExists($mysqli, $infoArr)) {
                $num = $mysqli->query("INSERT INTO ic37(company,person,phone,mobile,qq,msn,fax,email,address,country,region,zip,web,shopUrl) VALUES ('{$infoArr['company']}','{$infoArr['person']}','{$infoArr['phone']}','{$infoArr['mobile']}','{$infoArr['qq']}','{$infoArr['msn']}','{$infoArr['fax']}','{$infoArr['email']}','{$infoArr['address']}','{$infoArr['country']}','{$infoArr['region']}','{$infoArr['zip']}','{$infoArr['web']}','{$infoArr['shopUrl']}')");
                                                       return $num;                                                             Return false; # Represents that the data already exists
                                                                                                                                               } else {
             return false;
         }
}

Private function formatStr($str) {
          $str = trim($str);
          $str = str_replace(' ', '', $str);
           $str = str_replace('==Contact us', '', $str);
          return $str;
}

/**
* Connect to database
*/
Private function getDb() {
          $mysqli = new mysqli('localhost', 'root', '1715544', 'weiku');
$mysqli->query('SET NAMES GB2312');
         return $mysqli;
}

/**
* Check if the company already exists
* @param Resource $mysqli
* @param ArrayObject $infoArr
* @return bool
*/
Private function isExists($mysqli, $infoArr) {
          $mysqli->query("SELECT company FROM weiku WHERE company = '{$infoArr['company']}'");
If ($mysqli->affected_rows) {
             return true;
         } else {
             return false;
         }
}

/**
* Grab information
* @param $url
* @return ArrayObject
*/
Private function getInfoByShopUrl($url) {
$re = preg_replace('/(.*)/', '1', str_replace('', '', str_replace('', '', $this->getUrlInfo($url))));
Preg_match_all('/(.*)/Usi', $re, $companyArr);
preg_match_all('/Contact:s*(.*)/Usi', $re, $personArr);
preg_match_all('/Phone:s*(.*)/Usi', $re, $phoneArr);
preg_match_all('/Mobile:s*(.*)/Usi', $re, $mobileArr);
preg_match_all('/QQ：s*(.*)/Usi', $re, $qqArr);
preg_match_all('/MSN：s*(.*)/Usi', $re, $msnArr);
preg_match_all('/Fax:s*(.*)/Usi', $re, $faxArr);
        preg_match_all('/EMail：s*(.*)/Usi', $re, $emailArr);
        preg_match_all('/司地址[：]*[]*[]*[：]*[]*s*(.*)/Usi', $re, $addressArr);
        preg_match_all('/国家[：]*[]*[：]*[]*s*(.*)/Usi', $re, $countryArr);
        preg_match_all('/地区：s*(.*)/Usi', $re, $regionArr);
        preg_match_all('/邮政编码：s*(.*)/Usi', $re, $zipArr);
        preg_match_all('/s*网址[1]*：s*(.*)/Usi', $re, $webArr);
        $infoArr = array(
            'company'=>$this->formatStr($companyArr[1][0]),
            'person'=>$this->formatStr($personArr[1][0]),
            'phone'=>$this->formatStr($phoneArr[1][0]),
            'mobile'=>$this->formatStr($mobileArr[1][0]),
            'qq'=>$this->formatStr($qqArr[1][0]),
            'msn'=>$this->formatStr($msnArr[1][0]),
            'fax'=>$this->formatStr($faxArr[1][0]),
            'email'=>$this->formatStr($emailArr[1][0]),
            'address'=>$this->formatStr($addressArr[1][0]),
            'country'=>$this->formatStr($countryArr[1][0]),
            'region'=>$this->formatStr($regionArr[1][0]),
            'zip'=>$this->formatStr($zipArr[1][0]),
            'web'=>$this->formatStr($webArr[1][0]),
            'shopUrl'=>$url
        );
        return $infoArr;
    }

    /**
* Get the supplier url array based on the page
* @param string $re
* @return ArrayObject
*/
    private function shopUrlMatchReArr($re) {
        preg_match_all('/

[]*.*[]*s*

/Usi', $re, $arr);
        $arr = $this->formatUrlArr(array_unique($arr[1]));
        return $arr;
    }

    /**
* * Format array
* @param Array $arr
* @return ArrayObject
*/
    private function formatUrlArr($arr) {
        $newArr = array();
        foreach ($arr as $key=>$value) {
            if ($this->isExistsHttp($value)) {
                $newArr[$key] = $value;
            }
        }
        return $newArr;
    }

    /**
* Format QQ
* @param string $str
* @return string
*/
    private function formatQqMsn($str, $e='QQ') {
        if (emptyempty($str)) return '';
        preg_match_all('/alt="'.$e.':(.+)"/Usi', $str, $arr);
        if (count($arr[1])==1) return $arr[1][0];
        $newStr = null;
        foreach ($arr[1] as $value) {
            $newStr .= $value . ' ';
        }
        return rtrim($newStr, ' ');
    }

    /**
*Add supplier store link contact.asp
* @param array $arr
* @return string
*/
    private function shopAddContact($arr) {
        foreach ($arr as $k=>$v) {
            if (stristr($v, 'contact.asp')===FALSE)
                $newArr[$k] = $this->addContact($v);
            else
                $newArr[$k] = $v;
        }
        return $newArr;
    }

    /**
* Add link contact.asp
* @param string $str
* @return string
*/
    private function addContact($str) {
        return $str . '/contact.asp';
    }

    /**
* Remove the A tag from the URL
* @param string $site
* @return string
*/
    private function stripATags($site) {
        $site = preg_replace('/(.+)/', '1', $site);
        return $site;
    }

    /**
* Check if the url has http
* @param string $url
* @return bool
*/
    private function isExistsHttp($url) {
        if (stristr($url, 'http://')) {
            return true;
        } else {
            return false;
        }
    }

    /**
* Get page content
* @param Number $page
* @return string
*/
    private function getContent($page=1) {
          $re = file_get_contents($this->getUrl($this->key, $page));
         return $re;
}

/**
* Get page number
* @return Number
*/
Private function getPageNum() {
Preg_match_all('/A total of .* records are displayed in (.*) pages/Usi', $this->getContent(), $arr);
         return $arr[1][0];
}

/**
* Get URL link
* @param string $str
* @param int $page page number
* @return string
*/
Private function getUrl($str, $page=1) {
Return "http://www.ic37.com/sell/search.asp?keyword={$str}&x=86&y=22&page={$page}";
}

/**
* Get page content
* @param string $url
* @return string
*/
Private function getUrlInfo($url) {
          $re = file_get_contents($url);
         return $re;
}
}

/*
Program running idea: According to the IC search function of "China IC Network", enter the model number to search, and then grab the supplier information

Database structure
CREATE TABLE `ic37` (
`id` mediumint(8) unsigned NOT NULL auto_increment,
`company` varchar(500) default NULL,
`person` varchar(500) default NULL,
`phone` varchar(500) default NULL,
`mobile` varchar(500) default NULL,
`qq` varchar(500) default NULL,
`msn` varchar(500) default NULL,
`fax` varchar(500) default NULL,
`email` varchar(500) default NULL,
`address` varchar(1000) default NULL,
`country` varchar(500) default NULL,
`region` varchar(500) default NULL,
`zip` varchar(500) default NULL,
`web` varchar(500) default NULL,
`shopUrl` varchar(500) default NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=gb2312
*/

$k = new ic37();
$arr = array_unique(array('MAX3232', 'AML8613', 'MT6225A', 'OM8373PS/N3/A', 'PT7313', 'MAX8212ESA', 'TL431', 'S3C2440', 'TMS320F2812PGFA', 'PCM1704' 'AN6717', 'CA3162E', 'CA3161E', 'LM393N', 'DS18B20', 'SHT10', 'AML8613', 'AN6717', 'LM393N', 'CA3161E', 'CA3162E', 'PCM1704', ' STK392-040', 'K1667', 'MAX232', 'STM32F103', 'LM358', 'NE555', '78L05', 'LM324', 'TL431', 'PC817', '7805', 'LM339', ' LM317', '46A-3GRI', 'MODEL', '78L05', '93C46-3GRI', '8050', 'DS18B20', 'TDA2030', 'LM393', '74HC595', '6N137', 'SN75176BDR' ));
foreach ($arr as $v) {
$k->go($v);
}
?>
/**
* Grab the "China IC Network (http://www.ic37.com)" supplier main program
* author Lee.
* Last modify $Date: 2012-2-9 9:32:21 $
* Note: This program is executed according to the encoding GB2312, because the "China IC Network" website is encoded in GB2312, and the database must also be consistent
*/
class ic37 {
private $key; //Model
private $pageNum; // Page number

/**
* Entry program
*/
public function go($key) {
$this->key = $key;
$this->pageNum = $this->getPageNum();
$this->getInfo();
}

/**
* Get supplier url link array
* @return ArrayObject
*/
private function getInfo() {
  if ($this->pageNum==1) { # 处理只有一页的情况
   $arr = $this->shopAddContact($this->shopUrlMatchReArr($this->getContent()));
   $this->isAddSuccess($arr);
  } elseif ($this->pageNum>1) { # 多页
   for ($i=1; $i<=$this->pageNum; $i++) {
    $arr = $this->shopAddContact($this->shopUrlMatchReArr($this->getContent($i)));
    $this->isAddSuccess($arr);
   }
  }
}

/**
* Print whether the addition was successful
* @param ArrayObject $arr
* @return string
*/
private function isAddSuccess($arr) {
  foreach ($arr as $k=>$v) {
   if ($this->execAdd($this->getInfoByShopUrl($v))) {
    echo 'Add Success!!';
   } else {
    echo 'Add Faild!!';
   }
  }
}

/**
* Execute adding to database
* @param ArrayObject $infoArr
* @return Number Number of affected rows
*/
private function execAdd($infoArr) {
  $mysqli = $this->getDb();
  if (!empty($infoArr['company'])) {
   if (!$this->isExists($mysqli, $infoArr)) {
    $num = $mysqli->query("INSERT INTO ic37(company,person,phone,mobile,qq,msn,fax,email,address,country,region,zip,web,shopUrl) VALUES ('{$infoArr['company']}','{$infoArr['person']}','{$infoArr['phone']}','{$infoArr['mobile']}','{$infoArr['qq']}','{$infoArr['msn']}','{$infoArr['fax']}','{$infoArr['email']}','{$infoArr['address']}','{$infoArr['country']}','{$infoArr['region']}','{$infoArr['zip']}','{$infoArr['web']}','{$infoArr['shopUrl']}')");
    return $num;
   } else {
    return false; # 表示数据已经存在
   }
  } else {
   return false;
  }
}

private function formatStr($str) {
  $str = trim($str);
  $str = str_replace(' ', '', $str);
  $str = str_replace('==联系我们', '', $str);
  return $str;
}

/**
* Connect to database
*/
private function getDb() {
  $mysqli = new mysqli('localhost', 'root', '1715544', 'weiku');
  $mysqli->query('SET NAMES GB2312');
  return $mysqli;
}

/**
* Check if the company already exists
* @param Resource $mysqli
* @param ArrayObject $infoArr
* @return bool
*/
private function isExists($mysqli, $infoArr) {
  $mysqli->query("SELECT company FROM weiku WHERE company = '{$infoArr['company']}'");
  if ($mysqli->affected_rows) {
   return true;
  } else {
   return false;
  }
}

/**
* Grab information
* @param $url
* @return ArrayObject
*/
private function getInfoByShopUrl($url) {
  $re = preg_replace('/(.*)/', '1', str_replace('', '', str_replace('', '', $this->getUrlInfo($url))));
  preg_match_all('/(.*)/Usi', $re, $companyArr);
  preg_match_all('/联系人：s*(.*)/Usi', $re, $personArr);
  preg_match_all('/电话：s*(.*)/Usi', $re, $phoneArr);
  preg_match_all('/手机：s*(.*)/Usi', $re, $mobileArr);
  preg_match_all('/QQ：s*(.*)/Usi', $re, $qqArr);
  preg_match_all('/MSN：s*(.*)/Usi', $re, $msnArr);
  preg_match_all('/传真：s*(.*)/Usi', $re, $faxArr);
  preg_match_all('/EMail：s*(.*)/Usi', $re, $emailArr);
  preg_match_all('/司地址[：]*[]*[]*[：]*[]*s*(.*)/Usi', $re, $addressArr);
  preg_match_all('/国家[：]*[]*[：]*[]*s*(.*)/Usi', $re, $countryArr);
  preg_match_all('/地区：s*(.*)/Usi', $re, $regionArr);
  preg_match_all('/邮政编码：s*(.*)/Usi', $re, $zipArr);
  preg_match_all('/s*网址[1]*：s*(.*)/Usi', $re, $webArr);
  $infoArr = array(
   'company'=>$this->formatStr($companyArr[1][0]),
   'person'=>$this->formatStr($personArr[1][0]),
   'phone'=>$this->formatStr($phoneArr[1][0]),
   'mobile'=>$this->formatStr($mobileArr[1][0]),
   'qq'=>$this->formatStr($qqArr[1][0]),
   'msn'=>$this->formatStr($msnArr[1][0]),
   'fax'=>$this->formatStr($faxArr[1][0]),
   'email'=>$this->formatStr($emailArr[1][0]),
   'address'=>$this->formatStr($addressArr[1][0]),
   'country'=>$this->formatStr($countryArr[1][0]),
   'region'=>$this->formatStr($regionArr[1][0]),
   'zip'=>$this->formatStr($zipArr[1][0]),
   'web'=>$this->formatStr($webArr[1][0]),
   'shopUrl'=>$url
  );
  return $infoArr;
}

/**
* Get the supplier url array based on the page
* @param string $re
* @return ArrayObject
*/
private function shopUrlMatchReArr($re) {
preg_match_all('/

[]*.*[]*s*

/Usi', $re, $arr);
  $arr = $this->formatUrlArr(array_unique($arr[1]));
  return $arr;
}

/**
* Format array
* @param Array $arr
* @return ArrayObject
*/
private function formatUrlArr($arr) {
  $newArr = array();
  foreach ($arr as $key=>$value) {
   if ($this->isExistsHttp($value)) {
    $newArr[$key] = $value;
   }
  }
  return $newArr;
}

/**
* Format QQ
* @param string $str
* @return string
*/
private function formatQqMsn($str, $e='QQ') {
  if (empty($str)) return '';
  preg_match_all('/alt="'.$e.':(.+)"/Usi', $str, $arr);
  if (count($arr[1])==1) return $arr[1][0];
  $newStr = null;
  foreach ($arr[1] as $value) {
   $newStr .= $value . ' ';
  }
  return rtrim($newStr, ' ');
}

/**
* Add supplier store link contact.asp
* @param array $arr
* @return string
*/
private function shopAddContact($arr) {
  foreach ($arr as $k=>$v) {
   if (stristr($v, 'contact.asp')===FALSE)
    $newArr[$k] = $this->addContact($v);
   else
    $newArr[$k] = $v;
  }
  return $newArr;
}

/**
* Add link contact.asp
* @param string $str
* @return string
*/
private function addContact($str) {
  return $str . '/contact.asp';
}

/**
* Remove the A tag from the URL
* @param string $site
* @return string
*/
private function stripATags($site) {
$site = preg_replace('/(.+)/', '1', $site);
return $site;
}

/**
* Check if the url has http
* @param string $url
* @return bool
*/
private function isExistsHttp($url) {
  if (stristr($url, 'http://')) {
   return true;
  } else {
   return false;
  }
}

/**
* Get page content
* @param Number $page
* @return string
*/
private function getContent($page=1) {
  $re = file_get_contents($this->getUrl($this->key, $page));
  return $re;
}

/**
* Get page number
* @return Number
*/
private function getPageNum() {
  preg_match_all('/共.*条记录分(.*)页显示/Usi', $this->getContent(), $arr);
  return $arr[1][0];
}

/**
* Get URL link
* @param string $str
* @param int $page page number
* @return string
*/
private function getUrl($str, $page=1) {
return "http://www.ic37.com/sell/search.asp?keyword={$str}&x=86&y=22&page={$page}";
}

/**
* Get page content
* @param string $url
* @return string
*/
private function getUrlInfo($url) {
$re = file_get_contents($url);
return $re;
}
}

/*
程序运行思路：根据“中国 IC 网”的IC搜索功能，输入型号进行搜索，然后抓取供应商信息

数据库结构
CREATE TABLE `ic37` (
`id` mediumint(8) unsigned NOT NULL auto_increment,
`company` varchar(500) default NULL,
`person` varchar(500) default NULL,
`phone` varchar(500) default NULL,
`mobile` varchar(500) default NULL,
`qq` varchar(500) default NULL,
`msn` varchar(500) default NULL,
`fax` varchar(500) default NULL,
`email` varchar(500) default NULL,
`address` varchar(1000) default NULL,
`country` varchar(500) default NULL,
`region` varchar(500) default NULL,
`zip` varchar(500) default NULL,
`web` varchar(500) default NULL,
`shopUrl` varchar(500) default NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=gb2312
*/

$k = new ic37();
$arr = array_unique(array('MAX3232', 'AML8613', 'MT6225A', 'OM8373PS/N3/A', 'PT7313', 'MAX8212ESA', 'TL431', 'S3C2440', 'TMS320F2812PGFA', 'PCM1704', 'AN6717', 'CA3162E', 'CA3161E', 'LM393N', 'DS18B20', 'SHT10', 'AML8613', 'AN6717', 'LM393N', 'CA3161E', 'CA3162E', 'PCM1704', 'STK392-040', 'K1667', 'MAX232', 'STM32F103', 'LM358', 'NE555', '78L05', 'LM324', 'TL431', 'PC817', '7805', 'LM339', 'LM317', '46A-3GRI', 'MODEL', '78L05', '93C46-3GRI', '8050', 'DS18B20', 'TDA2030', 'LM393', '74HC595', '6N137', 'SN75176BDR'));
foreach ($arr as $v) {
$k->go($v);
}
?>

摘自 Lee.的专栏