python数据分析:赶集网北京各城区二手物品发帖量
运用python将前一节爬取的赶集网数据进行分析了解北京城区二手物品发帖量
效果是这样的：图表展示我的代码：from pymongo import MongoClientfrom string import punctuationimport chartsclient=MongoClient('localhost',27017)#链接数据库ganjiDB=client['ganjiDB']detail_info=ganjiDB['detail_info']以下为过程调试查看信息，非代码部分'''print(detail_info.find().count())#查看collection有多少条document-----------------------------------------------------------------------------------------------输出信息：86850-----------------------------------------------------------------------------------------------for i in detail_info.find().limit(1): #过程查看 print(i)-----------------------------------------------------------------------------------------------输出信息：{'area': ['朝阳', '-', '高碑店'], '_id': ObjectId('5698f524a98063dbe9e91ca8'), 'look': '-', 'url': 'http://bj.58.com/jiadian/24541664530488x.shtml', 'price': '450 元', 'pub_date': '2016.01.12', 'cates': ['北京58同城', '北京二手市场', '北京二手家电', '北京二手冰柜'], 'title': '【图】95成新小冰柜转让 - 朝阳高碑店二手家电 - 北京58同城'}-----------------------------------------------------------------------------------------------for i in detail_info.find().limit(7):#过程查看 print(i['area'])-----------------------------------------------------------------------------------------------输出信息：['朝阳', '-', '高碑店']['朝阳', '-', '定福庄']['西城', '-', '西单']['朝阳', '-', '望京']['丰台']['朝阳', '-', '定福庄']None-----------------------------------------------------------------------------------------------for i in detail_info.find().limit(7):#过程处理,将区域中的标点符号删除 if i['area']: area=[i for i in i['area'] if i not in punctuation]#去除标点符号 else: area=['不明']#将地区为‘None’调整为‘不明’ print(area)-----------------------------------------------------------------------------------------------输出信息：['朝阳', '高碑店']['朝阳', '定福庄']['西城', '西单']['朝阳', '望京']['丰台']['朝阳', '定福庄']['不明']-----------------------------------------------------------------------------------------------'''调试查看完毕，继续代码for i in detail_info.find(): if i['area']: area=[i for i in i['area'] if i not in punctuation]#去除标点符号 else: area=['不明']#将地区为‘None’调整为‘不明’ detail_info.update_one({'_id':i['_id']},{'$set':{'area':area}}) #将修改的条目更新到collection,注意此之前做好原collection的备份db.collection.copyTo('newcollection')area_list=[]#通过find条目信息查看，每个发帖条目都关联一个区域，那么可统一区域的数量进而推算当前区域的发帖量for i in detail_info.find(): area_list.append(i['area'][0])area_index=list(set(area_list))#将区域列表转换为集合剔除重复区域，运用集合的特点：无重复元素#print(area_index,len(area_index))post_times=[]for index in area_index: post_times.append(area_list.count(index))#以index为索引，查找每个index在原始区域表area_list中出现过多少次#print(post_times)def get_gen(types):#建议函数，运用生成器每次生成一条表象 length=0 if length<=len(area_index): for area,times in zip(area_index,post_times): info={ 'name':area, 'data':[times], 'type':types } yield info length+=1series=[data for data in get_gen('column')]#生成图标所需参数#print(series)charts.plot(series, show='inline', options=dict(title=dict(text='七日内北京城区二手物品')))#图标生成阅读本文章需要提前学习xpath，python等基本知识。本文章只适合python初学者学习了解python网络爬虫所用，禁止用于商业谋取利益。话不多说，直接上图
如上图所示，我们想抓取房源信息的标题、图片、区域、地址、价格和单价。那么直接上代码：
这里为模拟浏览器获输入网站的url，回车，然后得到网页源代码。
# 获取url对应的网页源码def getsource(url): headers={ 'User-Agent': 'User-Agent Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0)'} sourceHtml=requests.get(url, headers=headers) return sourceHtml.text此处为根据得到的网页源代码，通过xpath解析得到每一项数据，封装成我们需要的数组。
# 开始抓取并分析数据def spiderData(url): domtext=getsource(url) dom=etree.HTML(domtext) div_list=dom.xpath('//div[contains(@class, "js-tips-list")]/div[contains(@class, "f-list-item")]') data=[] for item in div_list: res={} res['title']=item.xpath('.//dd[contains(@class, "title")]/a/text()')[0] res['address']=item.xpath('.//dd[contains(@class, "address")]//a[@]/text()')[0] res['address-eara']=item.xpath('.//dd[contains(@class, "address")]//span[@]/text()')[0] res['price']=item.xpath('.//dd[contains(@class, "info")]//span[@]/text()')[0]+'万' res['singlePrice']=item.xpath('.//dd[contains(@class, "info")]//div[@]/text()')[0] res['images']=item.xpath('.//div[@]//img/@src')[0] data.append(res) return json.dumps(data, encoding="UTF-8", ensure_ascii=False )执行，查看部分json结果（因为太长，所以截取）
[{ "title": "9号线 外地人可买无需社保 70年产权 可落户上", "address-eara": "阳光理想城", "price": "53万", "singlePrice": "9636元/㎡", "address": "松江", "images": "http://pic7.58cdn.com.cn/anjuke_58/8205d4e70763f8a7bd7217d0e3cd574c?w=480&h=360&crop=1"}, { "title": "崇明品质小区 ，外地人可买 ，精装修带地暖 ，首", "address-eara": "明南佳苑", "price": "101万", "singlePrice": "17127元/㎡", "address": "崇明", "images": "http://pic6.58cdn.com.cn/anjuke_58/940dfa7b71d5e2a651e4fccb584c6170?w=480&h=360&crop=1"}, { "title": "经典小户型，外地可买，不受限购，送6万家具家电！", "address-eara": "临港17区", "price": "20万", "singlePrice": "15384元/㎡", "address": "浦东", "images": "http://pic2.58cdn.com.cn/anjuke_58/f7a9cd75d2d0439bf867d984ce06fcee?w=480&h=360&crop=1"}, { "title": "上海临港自贸区，不限贷，中国的第二个香港，精装修", "address-eara": "临港17区", "price": "105万", "singlePrice": "16153元/㎡", "address": "浦东", "images": "http://pic1.58cdn.com.cn/anjuke_58/6553ebe8ccb5e062e7e727b2dd48e4e4?w=480&h=360&crop=1"}···]这样我们就得到的所需要的爬虫数据。
欢迎喜欢网络爬虫的同学们批评指正，一起共同学习交流。
以下提供全部源码
# -*- coding: UTF-8 -*-import urllibfrom lxml import etreeimport jsonimport requests# 获取url对应的网页源码def getsource(url): headers={ 'User-Agent': 'User-Agent Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0)'} sourceHtml=requests.get(url, headers=headers) return sourceHtml.text# 开始抓取并分析数据def spiderData(url): domtext=getsource(url) dom=etree.HTML(domtext) div_list=dom.xpath('//div[contains(@class, "js-tips-list")]/div[contains(@class, "f-list-item")]') data=[] for item in div_list: res={} res['title']=item.xpath('.//dd[contains(@class, "title")]/a/text()')[0] res['address']=item.xpath('.//dd[contains(@class, "address")]//a[@]/text()')[0] res['address-eara']=item.xpath('.//dd[contains(@class, "address")]//span[@]/text()')[0] res['price']=item.xpath('.//dd[contains(@class, "info")]//span[@]/text()')[0]+'万' res['singlePrice']=item.xpath('.//dd[contains(@class, "info")]//div[@]/text()')[0] res['images']=item.xpath('.//div[@]//img/@src')[0] data.append(res) return json.dumps(data, encoding="UTF-8", ensure_ascii=False )result=spiderData('http://sh.ganji.com/ershoufang/')print result最后，小编想说：我是一名python开发工程师，整理了一套最新的python系统学习教程，想要这些资料的可以关注私信小编“01”即可，希望能对你有所帮助
发表评论