python 爬⾍及数据可视化展⽰
python 爬⾍及数据可视化展⽰
filter是滤镜的意思吗学了有关python 爬⾍及数据可视化的知识，想着做⼀些总结，加强⾃⼰的学习成果，也能给各位⼩伙伴⼀些⼩⼩的启发。
1、做任何事情都要明确⾃⼰的⽬的，想要做什么，打算怎么做，做到什么样的程度，⾃⼰有⼀个清晰的定位，虽然计划永远赶不上变化，但是按计划⾛，见招拆招或许也是不错的选择。
tomcat启动后无法访问到8080页面2、本项⽬是爬取⾖瓣的250部电影，将电影名，电影链接，评分等信息爬取保存到本地。将相关信息以列表的形式展⽰在⽹页上，访问者可通我的⽹站直接挑转到⾖瓣查看电影，将评分制作评分⾛势图，将电影制作成词云图在⽹页上展⽰，共有五个⽹页，可相互跳转。
项⽬流程图：
数据爬取：# -*- codeing = utf-8 -*-# @Time : 2022/1/11 22:39# @Author : lj # @File : spider.PY # @Software: 4{PRODUCT_NAME}import bs4 # ⽹页解析，获取数据 对⽹页的数据进⾏拆分import re #正则表达式，进⾏⽂字匹配 对数据进⾏提炼import urllib .request ,urllib .error #指定url 获取⽹页数据 怕⽹页import xlwt #进⾏excel 操作 存再excel 中import sqlite3 #进⾏sqllite 数据库操作 存在数据库中
1
2
3
4
5
6
7
8
9
10
11
import time # 主函数def main (): # 调⽤函数 url = "movie.douban/top250?start=" datalist1 = allData (url ) # savepath = "⾖瓣电影top250.xls" # savedata(datalist1,savepath) dbpath = "move.db" savedatasql (datalist1,dbpath )#匹配所需内容的正则表达式linkpattern = re pile (r'<a href="(.*?)/">')#匹配图⽚的正则表达式imagepattern = re pile (r'<img .*src=".*"/>',re .S )#re.S 忽略换⾏符，.表⽰除了换⾏符以外的所有字符#匹配影名的正则表达式namepattern = re pile (r'<span class="title">(.*)</span>')# 影⽚评分gradepattern = re pile (r'<span class="rating_num" property="v:average">(.*)</span>')# 评价⼈数peoplepattern = re pile (r'<span>(\d*)⼈评价</span>')#(\d*) 零个或多个#概况thinkpattern = re pile (r'<span class="inq">(.*)</span>')#影⽚的相关内容contentpattern = re pile (r'<p class="">(.*?)</p>',re .S )#忽略换⾏符#1、爬取⼀个⽹页def getData (url1): head = { "User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:30.0) Gecko/20100101 Firefox/30.0"} request = urllib .request .Request (url1,headers =head ) html = "" try : response = urllib .request .urlopen (request )
html = response .read ().decode ("utf-8") # print(html) except urllib .error .URLError as e : if hasattr (e ,"code"): print (e .code ) if hasattr (e ,"reason"): print (e .reason ) return html #返回给调⽤的地⽅# 2、爬取所有⽹页，匹配分析def allData (url ): datalist = [] for i in range (0, 10): # 左闭右开 调⽤⼗次，每次⼆⼗五条信息 url1 = url + str (i * 25) html = getData (url1) #保存获取到的⽹页源码 time .sleep (1) # 逐页解析 soup = bs4.BeautifulSoup (html ,"html.parser") #返回树型结构 for item in soup .find_all ('div',class_="item"): #查符合要求的字符串，返回列表,class 加下划线 data = [] item = str (item ) #link 获取到影⽚的超链接 link = re .findall (linkpattern ,item )[0] data .append (link ) # 影⽚图⽚ image = re .findall (imagepattern ,item )[0] data .append (image ) # 影⽚名 name = re .findall (namepattern ,item ) if (len (name )==2): chinaname = name [0] data .append (chinaname ) outername = name [1].replace ("/","")#.replace("/","") 列表内置的⽅法，将/替换为空"" data .append (outername )
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
grep命令查看进程35
36
ue编辑器使用教程37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
递归图59
60
61
62
63
64
65
66
67
68
69
70
爬虫软件 app71
72
73
74
75
76
data .append (outername ) else : data .append (name [0]) data .append (' ')#外⽂名空出来 # 影⽚评分 grade = re .findall (gradepattern ,item )[0] data .append (grade ) # 影⽚评价⼈数 people = re .findall (peoplepattern , item )[0] data .append (people ) # 影⽚概况 think = re .findall (thinkpattern , item ) if len (t
hink ) != 0: think = think [0].replace ("。","") data .append (think ) else : data .append (" ") # 影⽚内容 content = re .findall (contentpattern , item )[0] content = re .sub ('<br(\s+)?/>(\s+)?'," ",content )#替换内容中多余的符号和内容 content = re .sub ('/'," ",content ) data .append (content .strip ())#去除列表中的空格 datalist .append (data ) return datalist #3、保存数据到excel # def savedata(datalist1,savepath):# workplace = xlwt.Workbook(encoding="utf-8",style_compression=0)#style_compression=0·压缩样式# worksheet = workplace.add_sheet("⾖瓣电影top250",cell_overwrite_ok="true")#cell_overwrite_ok=true 是否可以覆盖# col = ('电影详情链接','电影图⽚链接','影⽚中⽂名','影⽚外⽂名','评分','评价⼈数','概况','影⽚内容')# for i in range(0,8):# worksheet.write(0,i,col[i])# for i in range(0,250):# print("打印了%d 条" %(i+1))# databuffer = datalist1[i]# for j in range(0,8):# worksheet.write(i+1,j,databuffer[j])# workplace.save(savepath) #保存#3、保存数据到数据库def savedatasql (datalist1,dbpath ):#dbpath 数据库的路径位置 init_db (dbpath ) conn = t (dbpath ) cur = conn .cursor ()#获取⼀个游标，存放sql 语句的执⾏结果 #cur 获得了游标可存放执⾏结果的对象 #将datalist1中的数据依次遍历写⼊数据库 for data in datalist1: for index in range (len (data )): if index == 4 or index == 5: continue # data[index] = "'" + data[index].replace(u'\xa0','') + "'" data [index ] = data [index ].replace ("'", "") data [index ] = "'"+data [index ]+"'" sql = '''
insert into move250( move_link,img_link,move_chinaname,move_foriername,grade,numbers,introduction,content) values (%s)'''%",".join (data ) #print(sql) cur .execute (sql ) conn mit () cur .close () conn .close () print ("write successful")#创建数据库def init_db (dbpath ):
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141
数据库数据excel 数据
def init_db (dbpath ): sql = ''' create table move250 ( id integer primary key autoincrement, move_link text, img_link text, move_chinaname varchar, move_foriername varchar, grade numeric, numbers numeric, introduction text, content text ) ''' c = t (dbpath ) buffer = c .cursor () # 获取游标 buffer .execute (sql ) # 内置的⽅法执⾏sql 语句 c mit () # 数据提交，写⼊数据库 c .close () # 数据库关闭 print ("database create successful")if __name__ == "__main__": #调⽤函数 main () print ("爬取完毕！")
141142143144145146147148149150151152153154155156157158159160161162163164165166
可视化制作
路由分配，⽹页渲染
版权声明：本站内容均来自互联网，仅供演示用，请勿用于商业和其他非法用途。如果侵犯了您的权益请与我们联系QQ:729038198，我们将在24小时内删除。
发表评论