爬取天善问答首页并写入MySQL数据库

爬取数据，存入数据库中以方便后续分析！

只是爬了第一页的内容，如果所有的都爬下来，再对数据进行分析，看看天善问答的回复量，解决率，浏览量等，是否也能反映一下论坛的活跃性呢

下面上一下代码

# -*- coding: utf-8 -*-

'''

python 3.5

mysql 5.7.17 

提取天善问答社区第一页内容

url: https://ask.hellobi.com/question/sort_type-new__day-0__page-1

'''



import pymysql

import urllib.request

from bs4 import BeautifulSoup

class Crawling_TSWD :

	def __init__(self):

		self.url = 'https://ask.hellobi.com/question/sort_type-new__day-0__page-1'

		self.User_Agent = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.109 Safari/537.36'

		self.header = {'User-Agent':self.User_Agent}

		self.conn = pymysql.connect(host="127.0.0.1",port=3306,user="isme",passwd="1111",db="python",charset="utf8")

		#浏览信息

		self.ans = []

		#是否解决

		self.isSolve = []

		#链接

		self.href = []

		#问题

		self.question = []

		#问题所属类别

		self.qType = []

		

	#获取首页信息

	def getPage(self) :

		req = urllib.request.Request(self.url,headers = self.header)

		res = urllib.request.urlopen(req)

		data = res.read()

		data = data.decode('utf-8')

		#保存源代码，后续方便分析

		with open(r'D:\python学习笔记\爬虫\保存记录\9_天善智能\TSWDSource.txt','w',encoding = 'utf-8') as f:

			f.write(data)

		return data

	#解析数据

	def analysisData(self,pageSource) :

		'''

                #测试用

		file = open(r'D:\python学习笔记\爬虫\保存记录\9_天善智能\TSWDSource.txt','r',encoding='utf-8')

		data = file.read()

		file.close()

		'''

		self.pageSource = pageSource

		soup = BeautifulSoup(pageSource,'lxml')

		#分析获取投票浏览回复及是否解决

		for div in soup.find_all(name = 'div',class_='mini-counts') :

			#浏览量

			self.ans.append(div.span.text)

			#是否解决(通过父节点的class属性判断是否解决问题)

			self.isSolve.append(div.find_parent()['class'])

		#获取问题内容及链接

		for div in soup.find_all(name = 'div',class_="aw-question-content") :

			#问题列表url

			self.href.append(div.h4.a['href'])

			#问题内容

			self.question.append(div.h4.a.text)

		#获取所属类别

		for div in soup.find_all(name = 'div' ,class_='tags') :

			if div.a is None:

				self.qType.append('无')

			else:

				self.qType.append(div.a.text)

		#拆分是否解决及浏览量列表，方便组合

		#是否解决问题

		isSolve_spilt = [self.isSolve[i][2] for i in range(0,len(self.isSolve)) if len(self.isSolve[i])%3 ==0]

		

		#每个问题的投票量，回复量，浏览量

		ans_spilt = [self.ans[i:i+3] for i in range(0,len(self.ans),3)]

		

		#整合素有列表

		table = []

		for i in range(0,len(self.question)) :

			table.append([self.qType[i],isSolve_spilt[i],ans_spilt[i][0],ans_spilt[i][1],ans_spilt[i][2],self.question[i],self.href[i]])

		return table

	def saveDB(self,table) :

		try :

			self.table = table 

			#获取连接（数据库连接我改了，想要测试的写自己的）

			conn = pymysql.connect(host="*.*.*.*",port=3306,user="****",passwd="****",db="python",charset="utf8")

			cur = conn.cursor()

			#插入数据

			insert_sql = 'insert into TSZN(Q_TYPE,IS_SOLVE,Q_TPL,Q_HFL,Q_LLL,Q_TEXT,Q_HREF) VALUES(%s,%s,%s,%s,%s,%s,%s)'

			cur.executemany(insert_sql,self.table)

			conn.commit()

		finally :

			cur.close()

			conn.close()



S = Crawling_TSWD()

pageSource = S.getPage()

data = S.analysisData(pageSource)

S.saveDB(data)

入库结果