Perangkak Python: Bagaimana untuk mendapatkan maklumat sewa bandar?-Tutorial Python-php.cn

Idea: Mulakan dengan perangkak satu benang, uji bahawa ia boleh merangkak dengan jayanya, kemudian optimumkannya kepada berbilang benang, dan akhirnya simpannya dalam pangkalan data

Ambil maklumat sewaan merangkak di Zhengzhou sebagai contoh

Nota: Projek praktikal ini hanya untuk tujuan pembelajaran Untuk mengelak daripada meletakkan terlalu banyak tekanan di tapak web, sila tukar nombor dalam kod Ubah suai kepada nombor yang lebih kecil dan tukar saiz utas

1. Perangkak berulir tunggal

# 用session取代requests
# 解析库使用bs4
# 并发库使用concurrent
import requests
# from lxml import etree    # 使用xpath解析
from bs4 import BeautifulSoup
from urllib import parse
import re
import time
 
headers = {
    &#39;referer&#39;: &#39;https://zz.zu.fang.com/&#39;,
    &#39;user-agent&#39;: &#39;Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36&#39;,
    &#39;cookie&#39;: &#39;global_cookie=ffzvt3kztwck05jm6twso2wjw18kl67hqft; city=zz; integratecover=1; __utma=147393320.427795962.1613371106.1613371106.1613371106.1; __utmc=147393320; __utmz=147393320.1613371106.1.1.utmcsr=zz.fang.com|utmccn=(referral)|utmcmd=referral|utmcct=/; __utmt_t0=1; __utmt_t1=1; __utmt_t2=1; ASP.NET_SessionId=aamzdnhzct4i5mx3ak4cyoyp; Rent_StatLog=23d82b94-13d6-4601-9019-ce0225c092f6; Captcha=61584F355169576F3355317957376E4F6F7552365351342B7574693561766E63785A70522F56557370586E3376585853346651565256574F37694B7074576B2B34536C5747715856516A4D3D; g_sourcepage=zf_fy%5Elb_pc; unique_cookie=U_ffzvt3kztwck05jm6twso2wjw18kl67hqft*6; __utmb=147393320.12.10.1613371106&#39;
}
data={
    &#39;agentbid&#39;:&#39;&#39;
}
 
session = requests.session()
session.headers = headers
 
# 获取页面
def getHtml(url):
    try:
        re = session.get(url)
        re.encoding = re.apparent_encoding
        return re.text
    except:
        print(re.status_code)
 
# 获取页面总数量
def getNum(text):
    soup = BeautifulSoup(text, &#39;lxml&#39;)
    txt = soup.select(&#39;.fanye .txt&#39;)[0].text
    # 取出“共**页”中间的数字
    num = re.search(r&#39;\d+&#39;, txt).group(0)
    return num
 
# 获取详细链接
def getLink(tex):
    soup=BeautifulSoup(text,&#39;lxml&#39;)
    links=soup.select(&#39;.title a&#39;)
    for link in links:
        href=parse.urljoin(&#39;https://zz.zu.fang.com/&#39;,link[&#39;href&#39;])
        hrefs.append(href)
 
# 解析页面
def parsePage(url):
    res=session.get(url)
    if res.status_code==200:
        res.encoding=res.apparent_encoding
        soup=BeautifulSoup(res.text,&#39;lxml&#39;)
        try:
            title=soup.select(&#39;div .title&#39;)[0].text.strip().replace(&#39; &#39;,&#39;&#39;)
            price=soup.select(&#39;div .trl-item&#39;)[0].text.strip()
            block=soup.select(&#39;.rcont #agantzfxq_C02_08&#39;)[0].text.strip()
            building=soup.select(&#39;.rcont #agantzfxq_C02_07&#39;)[0].text.strip()
            try:
                address=soup.select(&#39;.trl-item2 .rcont&#39;)[2].text.strip()
            except:
                address=soup.select(&#39;.trl-item2 .rcont&#39;)[1].text.strip()
            detail1=soup.select(&#39;.clearfix&#39;)[4].text.strip().replace(&#39;\n\n\n&#39;,&#39;,&#39;).replace(&#39;\n&#39;,&#39;&#39;)
            detail2=soup.select(&#39;.clearfix&#39;)[5].text.strip().replace(&#39;\n\n\n&#39;,&#39;,&#39;).replace(&#39;\n&#39;,&#39;&#39;)
            detail=detail1+detail2
            name=soup.select(&#39;.zf_jjname&#39;)[0].text.strip()
            buserid=re.search(&#39;buserid: \&#39;(\d+)\&#39;&#39;,res.text).group(1)
            phone=getPhone(buserid)
            print(title,price,block,building,address,detail,name,phone)
            house = (title, price, block, building, address, detail, name, phone)
            info.append(house)
        except:
            pass
    else:
        print(re.status_code,re.text)
 
# 获取代理人号码
def getPhone(buserid):
    url=&#39;https://zz.zu.fang.com/RentDetails/Ajax/GetAgentVirtualMobile.aspx&#39;
    data[&#39;agentbid&#39;]=buserid
    res=session.post(url,data=data)
    if res.status_code==200:
        return res.text
    else:
        print(res.status_code)
        return
 
if __name__ == &#39;__main__&#39;:
    start_time=time.time()
    hrefs=[]
    info=[]
    init_url = &#39;https://zz.zu.fang.com/house/&#39;
    num=getNum(getHtml(init_url))
    for i in range(0,num):
        url = f&#39;https://zz.zu.fang.com/house/i3{i+1}/&#39;
        text=getHtml(url)
        getLink(text)
    print(hrefs)
    for href in hrefs:
        parsePage(href)
 
    print("共获取%d条数据"%len(info))
    print("共耗时{}".format(time.time()-start_time))
    session.close()

Salin selepas log masuk

2. Optimumkan kepada berbilang- perangkak berulir

# 用session取代requests
# 解析库使用bs4
# 并发库使用concurrent
import requests
# from lxml import etree    # 使用xpath解析
from bs4 import BeautifulSoup
from concurrent.futures import ThreadPoolExecutor
from urllib import parse
import re
import time
 
headers = {
    &#39;referer&#39;: &#39;https://zz.zu.fang.com/&#39;,
    &#39;user-agent&#39;: &#39;Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36&#39;,
    &#39;cookie&#39;: &#39;global_cookie=ffzvt3kztwck05jm6twso2wjw18kl67hqft; integratecover=1; city=zz; keyWord_recenthousezz=%5b%7b%22name%22%3a%22%e6%96%b0%e5%af%86%22%2c%22detailName%22%3a%22%22%2c%22url%22%3a%22%2fhouse-a014868%2f%22%2c%22sort%22%3a1%7d%2c%7b%22name%22%3a%22%e4%ba%8c%e4%b8%83%22%2c%22detailName%22%3a%22%22%2c%22url%22%3a%22%2fhouse-a014864%2f%22%2c%22sort%22%3a1%7d%2c%7b%22name%22%3a%22%e9%83%91%e4%b8%9c%e6%96%b0%e5%8c%ba%22%2c%22detailName%22%3a%22%22%2c%22url%22%3a%22%2fhouse-a0842%2f%22%2c%22sort%22%3a1%7d%5d; __utma=147393320.427795962.1613371106.1613558547.1613575774.5; __utmc=147393320; __utmz=147393320.1613575774.5.4.utmcsr=zz.fang.com|utmccn=(referral)|utmcmd=referral|utmcct=/; ASP.NET_SessionId=vhrhxr1tdatcc1xyoxwybuwv; g_sourcepage=zf_fy%5Elb_pc; Captcha=4937566532507336644D6557347143746B5A6A6B4A7A48445A422F2F6A51746C67516F31357446573052634562725162316152533247514250736F72775566574A2B33514357304B6976343D; __utmt_t0=1; __utmt_t1=1; __utmt_t2=1; __utmb=147393320.9.10.1613575774; unique_cookie=U_0l0d1ilf1t0ci2rozai9qi24k1pkl9lcmrs*4&#39;
}
data={
    &#39;agentbid&#39;:&#39;&#39;
}
 
session = requests.session()
session.headers = headers
 
# 获取页面
def getHtml(url):
    res = session.get(url)
    if res.status_code==200:
        res.encoding = res.apparent_encoding
        return res.text
    else:
        print(res.status_code)
 
# 获取页面总数量
def getNum(text):
    soup = BeautifulSoup(text, &#39;lxml&#39;)
    txt = soup.select(&#39;.fanye .txt&#39;)[0].text
    # 取出“共**页”中间的数字
    num = re.search(r&#39;\d+&#39;, txt).group(0)
    return num
 
# 获取详细链接
def getLink(url):
    text=getHtml(url)
    soup=BeautifulSoup(text,&#39;lxml&#39;)
    links=soup.select(&#39;.title a&#39;)
    for link in links:
        href=parse.urljoin(&#39;https://zz.zu.fang.com/&#39;,link[&#39;href&#39;])
        hrefs.append(href)
 
# 解析页面
def parsePage(url):
    res=session.get(url)
    if res.status_code==200:
        res.encoding=res.apparent_encoding
        soup=BeautifulSoup(res.text,&#39;lxml&#39;)
        try:
            title=soup.select(&#39;div .title&#39;)[0].text.strip().replace(&#39; &#39;,&#39;&#39;)
            price=soup.select(&#39;div .trl-item&#39;)[0].text.strip()
            block=soup.select(&#39;.rcont #agantzfxq_C02_08&#39;)[0].text.strip()
            building=soup.select(&#39;.rcont #agantzfxq_C02_07&#39;)[0].text.strip()
            try:
                address=soup.select(&#39;.trl-item2 .rcont&#39;)[2].text.strip()
            except:
                address=soup.select(&#39;.trl-item2 .rcont&#39;)[1].text.strip()
            detail1=soup.select(&#39;.clearfix&#39;)[4].text.strip().replace(&#39;\n\n\n&#39;,&#39;,&#39;).replace(&#39;\n&#39;,&#39;&#39;)
            detail2=soup.select(&#39;.clearfix&#39;)[5].text.strip().replace(&#39;\n\n\n&#39;,&#39;,&#39;).replace(&#39;\n&#39;,&#39;&#39;)
            detail=detail1+detail2
            name=soup.select(&#39;.zf_jjname&#39;)[0].text.strip()
            buserid=re.search(&#39;buserid: \&#39;(\d+)\&#39;&#39;,res.text).group(1)
            phone=getPhone(buserid)
            print(title,price,block,building,address,detail,name,phone)
            house = (title, price, block, building, address, detail, name, phone)
            info.append(house)
        except:
            pass
    else:
        print(re.status_code,re.text)
 
# 获取代理人号码
def getPhone(buserid):
    url=&#39;https://zz.zu.fang.com/RentDetails/Ajax/GetAgentVirtualMobile.aspx&#39;
    data[&#39;agentbid&#39;]=buserid
    res=session.post(url,data=data)
    if res.status_code==200:
        return res.text
    else:
        print(res.status_code)
        return
 
if __name__ == &#39;__main__&#39;:
    start_time=time.time()
    hrefs=[]
    info=[]
    init_url = &#39;https://zz.zu.fang.com/house/&#39;
    num=getNum(getHtml(init_url))
    with ThreadPoolExecutor(max_workers=5) as t:
        for i in range(0,num):
            url = f&#39;https://zz.zu.fang.com/house/i3{i+1}/&#39;
            t.submit(getLink,url)
    print("共获取%d个链接"%len(hrefs))
    print(hrefs)
    with ThreadPoolExecutor(max_workers=30) as t:
        for href in hrefs:
            t.submit(parsePage,href)
    print("共获取%d条数据"%len(info))
    print("耗时{}".format(time.time()-start_time))
    session.close()

Salin selepas log masuk

3. Gunakan asyncio dioptimumkan lagi

# 用session取代requests
# 解析库使用bs4
# 并发库使用concurrent
import requests
# from lxml import etree    # 使用xpath解析
from bs4 import BeautifulSoup
from concurrent.futures import ThreadPoolExecutor
from urllib import parse
import re
import time
import asyncio
 
headers = {
    &#39;referer&#39;: &#39;https://zz.zu.fang.com/&#39;,
    &#39;user-agent&#39;: &#39;Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36&#39;,
    &#39;cookie&#39;: &#39;global_cookie=ffzvt3kztwck05jm6twso2wjw18kl67hqft; integratecover=1; city=zz; keyWord_recenthousezz=%5b%7b%22name%22%3a%22%e6%96%b0%e5%af%86%22%2c%22detailName%22%3a%22%22%2c%22url%22%3a%22%2fhouse-a014868%2f%22%2c%22sort%22%3a1%7d%2c%7b%22name%22%3a%22%e4%ba%8c%e4%b8%83%22%2c%22detailName%22%3a%22%22%2c%22url%22%3a%22%2fhouse-a014864%2f%22%2c%22sort%22%3a1%7d%2c%7b%22name%22%3a%22%e9%83%91%e4%b8%9c%e6%96%b0%e5%8c%ba%22%2c%22detailName%22%3a%22%22%2c%22url%22%3a%22%2fhouse-a0842%2f%22%2c%22sort%22%3a1%7d%5d; __utma=147393320.427795962.1613371106.1613558547.1613575774.5; __utmc=147393320; __utmz=147393320.1613575774.5.4.utmcsr=zz.fang.com|utmccn=(referral)|utmcmd=referral|utmcct=/; ASP.NET_SessionId=vhrhxr1tdatcc1xyoxwybuwv; g_sourcepage=zf_fy%5Elb_pc; Captcha=4937566532507336644D6557347143746B5A6A6B4A7A48445A422F2F6A51746C67516F31357446573052634562725162316152533247514250736F72775566574A2B33514357304B6976343D; __utmt_t0=1; __utmt_t1=1; __utmt_t2=1; __utmb=147393320.9.10.1613575774; unique_cookie=U_0l0d1ilf1t0ci2rozai9qi24k1pkl9lcmrs*4&#39;
}
data={
    &#39;agentbid&#39;:&#39;&#39;
}
 
session = requests.session()
session.headers = headers
 
# 获取页面
def getHtml(url):
    res = session.get(url)
    if res.status_code==200:
        res.encoding = res.apparent_encoding
        return res.text
    else:
        print(res.status_code)
 
# 获取页面总数量
def getNum(text):
    soup = BeautifulSoup(text, &#39;lxml&#39;)
    txt = soup.select(&#39;.fanye .txt&#39;)[0].text
    # 取出“共**页”中间的数字
    num = re.search(r&#39;\d+&#39;, txt).group(0)
    return num
 
# 获取详细链接
def getLink(url):
    text=getHtml(url)
    soup=BeautifulSoup(text,&#39;lxml&#39;)
    links=soup.select(&#39;.title a&#39;)
    for link in links:
        href=parse.urljoin(&#39;https://zz.zu.fang.com/&#39;,link[&#39;href&#39;])
        hrefs.append(href)
 
# 解析页面
def parsePage(url):
    res=session.get(url)
    if res.status_code==200:
        res.encoding=res.apparent_encoding
        soup=BeautifulSoup(res.text,&#39;lxml&#39;)
        try:
            title=soup.select(&#39;div .title&#39;)[0].text.strip().replace(&#39; &#39;,&#39;&#39;)
            price=soup.select(&#39;div .trl-item&#39;)[0].text.strip()
            block=soup.select(&#39;.rcont #agantzfxq_C02_08&#39;)[0].text.strip()
            building=soup.select(&#39;.rcont #agantzfxq_C02_07&#39;)[0].text.strip()
            try:
                address=soup.select(&#39;.trl-item2 .rcont&#39;)[2].text.strip()
            except:
                address=soup.select(&#39;.trl-item2 .rcont&#39;)[1].text.strip()
            detail1=soup.select(&#39;.clearfix&#39;)[4].text.strip().replace(&#39;\n\n\n&#39;,&#39;,&#39;).replace(&#39;\n&#39;,&#39;&#39;)
            detail2=soup.select(&#39;.clearfix&#39;)[5].text.strip().replace(&#39;\n\n\n&#39;,&#39;,&#39;).replace(&#39;\n&#39;,&#39;&#39;)
            detail=detail1+detail2
            name=soup.select(&#39;.zf_jjname&#39;)[0].text.strip()
            buserid=re.search(&#39;buserid: \&#39;(\d+)\&#39;&#39;,res.text).group(1)
            phone=getPhone(buserid)
            print(title,price,block,building,address,detail,name,phone)
            house = (title, price, block, building, address, detail, name, phone)
            info.append(house)
        except:
            pass
    else:
        print(re.status_code,re.text)
 
# 获取代理人号码
def getPhone(buserid):
    url=&#39;https://zz.zu.fang.com/RentDetails/Ajax/GetAgentVirtualMobile.aspx&#39;
    data[&#39;agentbid&#39;]=buserid
    res=session.post(url,data=data)
    if res.status_code==200:
        return res.text
    else:
        print(res.status_code)
        return
 
# 获取详细链接的线程池
async def Pool1(num):
    loop=asyncio.get_event_loop()
    task=[]
    with ThreadPoolExecutor(max_workers=5) as t:
        for i in range(0,num):
            url = f&#39;https://zz.zu.fang.com/house/i3{i+1}/&#39;
            task.append(loop.run_in_executor(t,getLink,url))
 
# 解析页面的线程池
async def Pool2(hrefs):
    loop=asyncio.get_event_loop()
    task=[]
    with ThreadPoolExecutor(max_workers=30) as t:
        for href in hrefs:
            task.append(loop.run_in_executor(t,parsePage,href))
 
if __name__ == &#39;__main__&#39;:
    start_time=time.time()
    hrefs=[]
    info=[]
    task=[]
    init_url = &#39;https://zz.zu.fang.com/house/&#39;
    num=getNum(getHtml(init_url))
    loop = asyncio.get_event_loop()
    loop.run_until_complete(Pool1(num))
    print("共获取%d个链接"%len(hrefs))
    print(hrefs)
    loop.run_until_complete(Pool2(hrefs))
    loop.close()
    print("共获取%d条数据"%len(info))
    print("耗时{}".format(time.time()-start_time))
    session.close()

Salin selepas log masuk

4. Simpan dalam pangkalan data Mysql

(1) Cipta jadual

from sqlalchemy import create_engine
from sqlalchemy import String, Integer, Column, Text
from sqlalchemy.orm import sessionmaker
from sqlalchemy.orm import scoped_session  # 多线程爬虫时避免出现线程安全问题
from sqlalchemy.ext.declarative import declarative_base
 
BASE = declarative_base()  # 实例化
engine = create_engine(
    "mysql+pymysql://root:root@127.0.0.1:3306/pytest?charset=utf8",
    max_overflow=300,  # 超出连接池大小最多可以创建的连接
    pool_size=100,  # 连接池大小
    echo=False,  # 不显示调试信息
)
 
 
class House(BASE):
    __tablename__ = &#39;house&#39;
    id = Column(Integer, primary_key=True, autoincrement=True)
    title=Column(String(200))
    price=Column(String(200))
    block=Column(String(200))
    building=Column(String(200))
    address=Column(String(200))
    detail=Column(Text())
    name=Column(String(20))
    phone=Column(String(20))
 
 
BASE.metadata.create_all(engine)
Session = sessionmaker(engine)
sess = scoped_session(Session)

Salin selepas log masuk

(2) Simpan data dalam pangkalan data

# 用session取代requests
# 解析库使用bs4
# 并发库使用concurrent
import requests
from bs4 import BeautifulSoup
from concurrent.futures import ThreadPoolExecutor
from urllib import parse
from mysqldb import sess, House
import re
import time
import asyncio
 
headers = {
    &#39;referer&#39;: &#39;https://zz.zu.fang.com/&#39;,
    &#39;user-agent&#39;: &#39;Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36&#39;,
    &#39;cookie&#39;: &#39;global_cookie=ffzvt3kztwck05jm6twso2wjw18kl67hqft; integratecover=1; city=zz; __utmc=147393320; ASP.NET_SessionId=vhrhxr1tdatcc1xyoxwybuwv; __utma=147393320.427795962.1613371106.1613575774.1613580597.6; __utmz=147393320.1613580597.6.5.utmcsr=zz.fang.com|utmccn=(referral)|utmcmd=referral|utmcct=/; __utmt_t0=1; __utmt_t1=1; __utmt_t2=1; Rent_StatLog=c158b2a7-4622-45a9-9e69-dcf6f42cf577; keyWord_recenthousezz=%5b%7b%22name%22%3a%22%e4%ba%8c%e4%b8%83%22%2c%22detailName%22%3a%22%22%2c%22url%22%3a%22%2fhouse-a014864%2f%22%2c%22sort%22%3a1%7d%2c%7b%22name%22%3a%22%e9%83%91%e4%b8%9c%e6%96%b0%e5%8c%ba%22%2c%22detailName%22%3a%22%22%2c%22url%22%3a%22%2fhouse-a0842%2f%22%2c%22sort%22%3a1%7d%2c%7b%22name%22%3a%22%e7%bb%8f%e5%bc%80%22%2c%22detailName%22%3a%22%22%2c%22url%22%3a%22%2fhouse-a014871%2f%22%2c%22sort%22%3a1%7d%5d; g_sourcepage=zf_fy%5Elb_pc; Captcha=6B65716A41454739794D666864397178613772676C75447A4E746C657144775A347A6D42554F446532357649643062344F6976756E563450554E59594B7833712B413579506C4B684958343D; unique_cookie=U_0l0d1ilf1t0ci2rozai9qi24k1pkl9lcmrs*14; __utmb=147393320.21.10.1613580597&#39;
}
data={
    &#39;agentbid&#39;:&#39;&#39;
}
 
session = requests.session()
session.headers = headers
 
# 获取页面
def getHtml(url):
    res = session.get(url)
    if res.status_code==200:
        res.encoding = res.apparent_encoding
        return res.text
    else:
        print(res.status_code)
 
# 获取页面总数量
def getNum(text):
    soup = BeautifulSoup(text, &#39;lxml&#39;)
    txt = soup.select(&#39;.fanye .txt&#39;)[0].text
    # 取出“共**页”中间的数字
    num = re.search(r&#39;\d+&#39;, txt).group(0)
    return num
 
# 获取详细链接
def getLink(url):
    text=getHtml(url)
    soup=BeautifulSoup(text,&#39;lxml&#39;)
    links=soup.select(&#39;.title a&#39;)
    for link in links:
        href=parse.urljoin(&#39;https://zz.zu.fang.com/&#39;,link[&#39;href&#39;])
        hrefs.append(href)
 
# 解析页面
def parsePage(url):
    res=session.get(url)
    if res.status_code==200:
        res.encoding=res.apparent_encoding
        soup=BeautifulSoup(res.text,&#39;lxml&#39;)
        try:
            title=soup.select(&#39;div .title&#39;)[0].text.strip().replace(&#39; &#39;,&#39;&#39;)
            price=soup.select(&#39;div .trl-item&#39;)[0].text.strip()
            block=soup.select(&#39;.rcont #agantzfxq_C02_08&#39;)[0].text.strip()
            building=soup.select(&#39;.rcont #agantzfxq_C02_07&#39;)[0].text.strip()
            try:
                address=soup.select(&#39;.trl-item2 .rcont&#39;)[2].text.strip()
            except:
                address=soup.select(&#39;.trl-item2 .rcont&#39;)[1].text.strip()
            detail1=soup.select(&#39;.clearfix&#39;)[4].text.strip().replace(&#39;\n\n\n&#39;,&#39;,&#39;).replace(&#39;\n&#39;,&#39;&#39;)
            detail2=soup.select(&#39;.clearfix&#39;)[5].text.strip().replace(&#39;\n\n\n&#39;,&#39;,&#39;).replace(&#39;\n&#39;,&#39;&#39;)
            detail=detail1+detail2
            name=soup.select(&#39;.zf_jjname&#39;)[0].text.strip()
            buserid=re.search(&#39;buserid: \&#39;(\d+)\&#39;&#39;,res.text).group(1)
            phone=getPhone(buserid)
            print(title,price,block,building,address,detail,name,phone)
            house = (title, price, block, building, address, detail, name, phone)
            info.append(house)
            try:
                house_data=House(
                    title=title,
                    price=price,
                    block=block,
                    building=building,
                    address=address,
                    detail=detail,
                    name=name,
                    phone=phone
                )
                sess.add(house_data)
                sess.commit()
            except Exception as e:
                print(e)    # 打印错误信息
                sess.rollback()  # 回滚
        except:
            pass
    else:
        print(re.status_code,re.text)
 
# 获取代理人号码
def getPhone(buserid):
    url=&#39;https://zz.zu.fang.com/RentDetails/Ajax/GetAgentVirtualMobile.aspx&#39;
    data[&#39;agentbid&#39;]=buserid
    res=session.post(url,data=data)
    if res.status_code==200:
        return res.text
    else:
        print(res.status_code)
        return
 
# 获取详细链接的线程池
async def Pool1(num):
    loop=asyncio.get_event_loop()
    task=[]
    with ThreadPoolExecutor(max_workers=5) as t:
        for i in range(0,num):
            url = f&#39;https://zz.zu.fang.com/house/i3{i+1}/&#39;
            task.append(loop.run_in_executor(t,getLink,url))
 
# 解析页面的线程池
async def Pool2(hrefs):
    loop=asyncio.get_event_loop()
    task=[]
    with ThreadPoolExecutor(max_workers=30) as t:
        for href in hrefs:
            task.append(loop.run_in_executor(t,parsePage,href))
 
if __name__ == &#39;__main__&#39;:
    start_time=time.time()
    hrefs=[]
    info=[]
    task=[]
    init_url = &#39;https://zz.zu.fang.com/house/&#39;
    num=getNum(getHtml(init_url))
    loop = asyncio.get_event_loop()
    loop.run_until_complete(Pool1(num))
    print("共获取%d个链接"%len(hrefs))
    print(hrefs)
    loop.run_until_complete(Pool2(hrefs))
    loop.close()
    print("共获取%d条数据"%len(info))
    print("耗时{}".format(time.time()-start_time))
    session.close()

Salin selepas log masuk

5. Penyampaian akhir (berkod)

Perangkak Python: Bagaimana untuk mendapatkan maklumat sewa bandar?

Atas ialah kandungan terperinci Perangkak Python: Bagaimana untuk mendapatkan maklumat sewa bandar?. Untuk maklumat lanjut, sila ikut artikel berkaitan lain di laman web China PHP!