Example tutorial of implementing crawler with requests and lxml

# requests module to request pages
# lxml module's html build selector selector (formatted response response)
# from lxml import html
# import requests

response = requests.get(url).content

# selector = html.formatstring(response)

hrefs = selector.xpath('/html/body//div[@class='feed-item _j_feed_item']/a/@href' )

# Take url = 'https://www.mafengwo.cn/gonglve/ziyouxing/2033.html' as an example

# python 2.7import requestsfrom lxml import htmlimport os

1 # 获取首页中子页的url链接2 def get_page_urls(url):3     response = requests.get(url).content4     # 通过lxml的html来构建选择器5     selector = html.fromstring(response)6     urls = []7     for i in selector.xpath("/html/body//div[@class='feed-item _j_feed_item']/a/@href"):8         urls.append(i)9     return urls

1 # get title from a child's html(div[@class='title'])2 def get_page_a_title(url):3     '''url is ziyouxing's a@href'''4     response = requests.get(url).content5     selector = html.fromstring(response)6     # get xpath by chrome's tool  -->  /html/body//div[@class='title']/text()7     a_title = selector.xpath("/html/body//div[@class='title']/text()")8     return a_title

 1 # 获取页面选择器(通过lxml的html构建) 2 def get_selector(url): 3     response = requests.get(url).content 4     selector = html.fromstring(response) 5     return selector

# 通过chrome的开发者工具分析html页面结构后发现，我们需要获取的文本内容主要显示在div[@class='l-topic']和div[@class='p-section']中

1  # 获取所需的文本内容2  def get_page_content(selector):3      # /html/body/div[2]/div[2]/div[1]/div[@class='l-topic']/p/text()4      page_title = selector.xpath("//div[@class='l-topic']/p/text()")5      # /html/body/div[2]/div[2]/div[1]/div[2]/div[15]/div[@class='p-section']/text()6      page_content = selector.xpath("//div[@class='p-section']/text()")7      return page_title,page_content

1 # 获取页面中的图片url地址2 def get_image_urls(selector):3     imagesrcs = selector.xpath("//img[@class='_j_lazyload']/@src")4     return imagesrcs

  # 获取图片的标题

1 def get_image_title(selector, num)2     # num 是从2开始的3     url = "/html/body/div[2]/div[2]/div[1]/div[2]/div["+num+"]/span[@class='img-an']/text()"4     if selector.xpath(url) is not None:5         image_title = selector.xpath(url)6     else:7         image_title = "map"+str(num) # 没有就起一个8     return image_title

  # 下载图片

 1 def downloadimages(selector,number): 2     '''number是用来计数的''' 3     urls = get_image_urls() 4     num = 2 5     amount = len(urls) 6     for url in urls: 7         image_title = get_image_title(selector, num) 8         filename = "/home/WorkSpace/tour/words/result"+number+"/+"image_title+".jpg" 9         if not os.path.exists(filename):10             os.makedirs(filename)11         print('downloading %s image %s' %(number, image_title))12         with open(filename, 'wb') as f:13             f.write(requests.get(url).content)14         num += 115     print "已经下载了%s张图" %num

# 入口，启动并把获取的数据存入文件中if __name__ =='__main__':
    url = ''urls = get_page_urls(url)# turn to get response from htmlnumber = 1for i in urls:
        selector = get_selector(i)# download images      downloadimages(selector,number)# get text and write into a filepage_title, page_content = get_page_content(selector)
        result = page_title+'\n'+page_content+'\n\n'path = "/home/WorkSpace/tour/words/result"+num+"/"if not os.path.exists(filename):
            os.makedirs(filename)
        filename = path + "num"+".txt"with open(filename,'wb') as f:
            f.write(result)print result

This is the end of the crawler. Before crawling the page, you must carefully analyze the html structure. Some pages are generated by js. This page is relatively simple and does not involve js processing. There will be relevant sharing in future essays

The above is the detailed content of Example tutorial of implementing crawler with requests and lxml. For more information, please follow other related articles on the PHP Chinese website!