最近回老家了一趟，爬取简书的程序也就停下来了，回到长沙继续爬，很高兴的爬到了300W左右，导出来一看，好多重复的，我记得我说过设置过的，拿到代码一看，晕：

插入的是author_infos表，但判断却是author_url表，然后我就打算去重后调用url爬取用户详细信息，可耐mongodb去重不会，自己百度了下也没搞明白；再者，向右前辈说我爬取字段太少，那我就想重新修改爬取好了（已哭晕在厕所）。

代码

import requests

from lxml import etree

import time

import pymongo



client = pymongo.MongoClient('localhost', 27017)

jianshu = client['jianshu']

author_infos = jianshu['author_infos']



headers = {

    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36',

    'Connection':'keep-alive'

}



def get_article_url(url,page):

    link_view = '{}?order_by=added_at&page={}'.format(url,str(page))

    try:

        html = requests.get(link_view,headers=headers)

        selector = etree.HTML(html.text)

        infos = selector.xpath('//div[@class="name"]')

        for info in infos:

            author_name = info.xpath('a/text()')[0]

            authorurl = info.xpath('a/@href')[0]

            if 'http://www.jianshu.com'+ authorurl in [item['url'] for item in author_infos.find()]:

                pass

            else:

            # print('http://www.jianshu.com'+authorurl,author_name)

            #     author_infos.insert_one({'author_name':author_name,'author_url':'http://www.jianshu.com'+authorurl})

                get_all_info('http://www.jianshu.com'+authorurl)

                get_reader_url(authorurl)

        time.sleep(2)

    except requests.exceptions.ConnectionError:

        pass



# get_article_url('http://www.jianshu.com/c/bDHhpK',2)

def get_reader_url(url):

    link_views = ['http://www.jianshu.com/users/{}/followers?page={}'.format(url.split('/')[-1],str(i)) for i in range(1,100)]

    for link_view in link_views:

        try:

            html = requests.get(link_view,headers=headers)

            selector = etree.HTML(html.text)

            infos = selector.xpath('//li/div[@class="info"]')

            for info in infos:

                author_name = info.xpath('a/text()')[0]

                authorurl = info.xpath('a/@href')[0]

                # print(author_name,authorurl)

                # author_infos.insert_one({'author_name': author_name, 'author_url': 'http://www.jianshu.com' + authorurl})

                get_all_info('http://www.jianshu.com' + authorurl)

        except requests.exceptions.ConnectionError:

            pass

# get_reader_url('http://www.jianshu.com/u/7091a52ac9e5')



def get_all_info(url):

    html = requests.get(url,headers=headers)

    selector = etree.HTML(html.text)

    try:

        author_name = selector.xpath('//a[@class="name"]/text()')[0]

        author_focus = selector.xpath('//div[@class="info"]/ul/li[1]/div/p/text()')[0]

        author_fans = selector.xpath('//div[@class="info"]/ul/li[2]/div/p/text()')[0]

        author_article = selector.xpath('//div[@class="info"]/ul/li[3]/div/p/text()')[0]

        author_write_amount = selector.xpath('//div[@class="info"]/ul/li[4]/div/p/text()')[0]

        author_get_like = selector.xpath('//div[@class="info"]/ul/li[5]/div/p/text()')[0]

        author_intrus = selector.xpath('//div[1]/div/div[2]/div[2]/div/text()')

        author_intru = selector.xpath('//div[1]/div/div[2]/div[2]/div/text()') if len(author_intrus) != 0 else "无"

        if selector.xpath('//span[@class="author-tag"]'):

            author_title = '签约作者'

        else:

            author_title = '普通作者'

        infos = {

            'url':url,

            'name':author_name,

            'focus':author_focus,

            'fans':author_fans,

            'article':author_article,

            'write_amount':author_write_amount,

            'get_like':author_get_like,

            'intru':author_intru,

            'title':author_title

        }

        author_infos.insert_one(infos)

    except IndexError:

        pass

今天就先这样，主要是记录自己的学习过程。
本文已在版权印备案，如需转载请访问版权印。58708803