Example of Python crawling Json data-Python Tutorial-php.cn

Example of Python crawling Json data

coldplay.xixi

Release： 2020-12-07 17:41:04

forward

4950 people have browsed it

Python教程栏目介绍爬取Json数据实例

Example of Python crawling Json data

相关免费学习推荐：python教程（视频）

本文中以爬取其中的AI流转率数据为例。

该地址返回的响应内容为Json类型，其中红框标记的项即为AI流转率值：

实现代码如下：

import requests
import json
import csv
 
# 爬虫地址
url = &#39;https://databank.yushanfang.com/api/ecapi?path=/databank/crowdFullLink/flowInfo&fromCrowdId=3312&beginTheDate=201810{}&endTheDate=201810{}&toCrowdIdList[0]=3312&toCrowdIdList[1]=3313&toCrowdIdList[2]=3314&toCrowdIdList[3]=3315&#39;
 
# 携带cookie进行访问
headers = {
&#39;Host&#39;:&#39;databank.yushanfang.com&#39;,
&#39;Referer&#39;:&#39;https://databank.yushanfang.com/&#39;,
&#39;Connection&#39;:&#39;keep-alive&#39;,
&#39;User-Agent&#39;:&#39;Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36&#39;,
&#39;Cookie&#39;:&#39;_tb_token_=iNkDeJLdM3MgvKjhsfdW; bs_n_lang=zh_CN; cna=aaj1EViI7x0CATo9kTKvjzgS; ck2=072de851f1c02d5c7bac555f64c5c66d; c_token=c74594b486f8de731e2608cb9526a3f2; an=5YWo5qOJ5pe25Luj5a6Y5pa55peX6Iiw5bqXOnpmeA%3D%3D; lg=true; sg=\"=19\"; lvc=sAhojs49PcqHQQ%3D%3D; isg=BPT0Md7dE_ic5Ie3Oa85RxaMxbLK3UqJMMiN6o5VjH8C-ZRDtt7aRXb3fXGEAVAP&#39;,
}
 
rows = []
for n in range(20, 31):
  row = []
  row.append(n)
  for m in range (21, 32):
    if m < n + 1:
      row.append("")
    else:
      
      # 格式化请求地址，更换请求参数
      reqUrl = url.format(n, m)
      
      # 打印本次请求地址
      print(url)
      
      # 发送请求，获取响应结果
      response = requests.get(url=reqUrl, headers=headers, verify=False)
      text = response.text
      
      # 打印本次请求响应内容
      print(text)
      
      # 将响应内容转换为Json对象
      jsonobj = json.loads(text)
      
      # 从Json对象获取想要的内容
      toCntPercent = jsonobj[&#39;data&#39;][&#39;interCrowdInfo&#39;][1][&#39;toCntPercent&#39;]
      
      # 生成行数据
      row.append(str(toCntPercent)+"%")
      
  # 保存行数据    
  rows.append(row)
  
# 生成Excel表头
header = [&#39;AI流转率&#39;, &#39;21&#39;, &#39;22&#39;, &#39;23&#39;, &#39;24&#39;, &#39;25&#39;, &#39;26&#39;, &#39;27&#39;, &#39;28&#39;, &#39;29&#39;, &#39;30&#39;, &#39;31&#39;]
 
# 将表头数据和爬虫数据导出到Excel文件
with open(&#39;D:\\res\\pachong\\tmall.csv&#39;, &#39;w&#39;, encoding=&#39;gb18030&#39;) as f :
  f_csv = csv.writer(f)
  f_csv.writerow(header)
  f_csv.writerows(rows)

Copy after login

import csv
import json
import ssl
import urllib.request
 
# 爬虫地址
url = &#39;https://databank.yushanfang.com/api/ecapi?path=/databank/crowdFullLink/flowInfo&fromCrowdId=3312&beginTheDate=201810{}&endTheDate=201810{}&toCrowdIdList[0]=3312&toCrowdIdList[1]=3313&toCrowdIdList[2]=3314&toCrowdIdList[3]=3315&#39;
 
# 不校验证书
ssl._create_default_https_context = ssl._create_unverified_context
 
# 携带cookie进行访问
headers = {
&#39;Host&#39;:&#39;databank.yushanfang.com&#39;,
&#39;Referer&#39;:&#39;https://databank.yushanfang.com/&#39;,
&#39;Connection&#39;:&#39;keep-alive&#39;,
&#39;User-Agent&#39;:&#39;Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36&#39;,
&#39;Cookie&#39;:&#39;_tb_token_=iNkDeJLdM3MgvKjhsfdW; bs_n_lang=zh_CN; cna=aaj1EViI7x0CATo9kTKvjzgS; ck2=072de851f1c02d5c7bac555f64c5c66d; c_token=c74594b486f8de731e2608cb9526a3f2; an=5YWo5qOJ5pe25Luj5a6Y5pa55peX6Iiw5bqXOnpmeA%3D%3D; lg=true; sg=\"=19\"; lvc=sAhojs49PcqHQQ%3D%3D; isg=BPT0Md7dE_ic5Ie3Oa85RxaMxbLK3UqJMMiN6o5VjH8C-ZRDtt7aRXb3fXGEAVAP&#39;,
}
 
rows = []
n = 20
while n <31:
  row = []
  row.append(n)
  
  m =21
  while m <32:
    
    if m < n + 1:
      row.append("")
    else:
      
      # 格式化请求地址，更换请求参数
      reqUrl = url.format(n, m)
      
      # 打印本次请求地址
      print(reqUrl)
      
      # 发送请求，获取响应结果
      request = urllib.request.Request(url=reqUrl, headers=headers)
      response = urllib.request.urlopen(request)
      text = response.read().decode(&#39;utf8&#39;)
      
      # 打印本次请求响应内容
      print(text)
      
      # 将响应内容转换为Json对象
      jsonobj = json.loads(text)
      
      # 从Json对象获取想要的内容
      toCntPercent = jsonobj[&#39;data&#39;][&#39;interCrowdInfo&#39;][1][&#39;toCntPercent&#39;]
      
      # 生成行数据
      row.append(str(toCntPercent) + "%")
      
    m = m+1
    
  rows.append(row)    
  n = n+1
  
# 生成Excel表头
header = [&#39;AI流转率&#39;, &#39;21&#39;, &#39;22&#39;, &#39;23&#39;, &#39;24&#39;, &#39;25&#39;, &#39;26&#39;, &#39;27&#39;, &#39;28&#39;, &#39;29&#39;, &#39;30&#39;, &#39;31&#39;]
 
# 将表头数据和爬虫数据导出到Excel文件
with open(&#39;D:\\res\\pachong\\tmall.csv&#39;, &#39;w&#39;, encoding=&#39;gb18030&#39;) as f :
  f_csv = csv.writer(f)
  f_csv.writerow(header)
  f_csv.writerows(rows)

Copy after login

导出内容如下：

到此这篇关于使用Python爬取Json数据的文章就介绍到这了。

相关免费学习推荐：php编程（视频）

The above is the detailed content of Example of Python crawling Json data. For more information, please follow other related articles on the PHP Chinese website!