python爬⾍实例100例-Python爬⾍实例importrequestsfrom bs4 importBeautifulSoupfrom datetime importdatetimeimportreimportjsonimportpandas
news_total=[]
commentURL='ws.sina/page/info?version=1&format=js&channel=gn&newsid=comos-{}&group=&compress=0&ie=utf-8&oe=utf-8&page=1&page_size=20'url='ws.sina/zt_list? channel=news&cat_1=gnxw&cat_2==gdxw1||=gatxw||=zs-
pl||=mtjj&level==1||=2&show_ext=1&show_all=1&show_num=22&tag=1&format=json&page=
{}&callback=newsloadercallback&_=1509373193047'
defparseListLinks(url):
newsdetails=[]
(url)
jd= json.strip().lstrip('newsloadercallback(').rstrip(');'))for ent in jd['result']['data']:
newsdetails.append(getNewsDetail(ent['url']))returnnewsdetailsdefgetNewsDetail(newsurl):
result={}
(newsurl)
result['title']=soup.select('#artibodyTitle')[0].text
result['newssource']=soup.select('.time-source span a')[0].text
timesource=soup.select('.time-source')[0].contents[0].strip()
dt1=datetime.strptime(timesource,'%Y年%m⽉%d⽇%H:%M')
result['dt'] =dt1.strftime('%Y-%m-%d-%H:%M')
result['article']=' '.join([p.text.strip() for p in soup.select('#artibody p')[:-1]])
result['editor']=soup.select('.article-editor')[0].text.strip('责任编辑：')
result['comments']=getCommentCounts(newsurl)print('获得⼀条新闻')returnresultdefgetCommentCounts(newsurl):
m=re.search('doc-i(.+).shtml',newsurl)
up(1)
(commentURL.format(newsid))
jd=json.strip('var data='))return jd['result']['count']['total']for i in range(1,8):print('正在爬取
第'+str(i)+'页......')
newsurl=url.format(i)
newsary=parseListLinks(newsurl)
python新手编程100例d(newsary)print('抓取结束')
df=pandas.DataFrame(news_total)
<_excel('news.xlsx')

版权声明：本站内容均来自互联网，仅供演示用，请勿用于商业和其他非法用途。如果侵犯了您的权益请与我们联系QQ:729038198，我们将在24小时内删除。