MongoDB爬蟲實踐：爬取虎撲論壇

---

網站地址為：https://bbs.hupu.com/bxj

1.網站分析

首先，定位網頁上帖子名稱、帖子鏈接、作者、作者鏈接、創建時間、回復數目、瀏覽數目、最后回復用戶、最后回復時間等信息的位置，之后，我們使用BeautifulSoup在網頁中定位這些。

數據所在的位置

數據	位置
某帖子所有數據	‘li’
帖子名稱	div?class="titlelink box"? ?>? a
帖子鏈接	div?class="titlelink box"? ?>? a['href']
作者	div class="author box"? ?>? a
作者鏈接	div class="author box"? ?>? a['href']
創建時間	div class="author box"? ?>? contents[5]
回復數	span class="ansour box"?
瀏覽數	span class="ansour box"
最后回復用戶	div? class="endreply box"? > span
最后回復時間	div? class="endreply box"? > a

另外，當打開第二頁時，網頁的URL地址變成了https://bbs.hupu.com/bxj-2,以此類推。

2.項目實踐

首先嘗試獲取第一頁數據，代碼如下：

            
              #!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@File  : Save05.py
@Author: Xinzhe.Pang
@Date  : 2019/7/10 0:14
@Desc  : 
"""
# 爬取虎撲論壇數據 https://bbs.hupu.com/bxj

import requests
from bs4 import BeautifulSoup
import datetime


def get_page(link):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'}
    r = requests.get(link, headers=headers)
    html = r.content
    html = html.decode('UTF-8')
    soup = BeautifulSoup(html, 'lxml')
    return soup


def get_data(post_list):
    data_list = []
    for post in post_list:
        title = post.find('div', class_='titlelink box').a.text.strip()
        post_link = post.find('div', class_='titlelink box').a['href']
        post_link = "https://bbs.hupu.com" + post_link

        author = post.find('div', class_='author box').a.text.strip()
        author_page = post.find('div', class_='author box').a['href']
        start_date = post.find('div', class_='author box').contents[5].text.strip()

        reply_view = post.find('span', class_='ansour box').text.strip()
        reply = reply_view.split('/')[0].strip()
        view = reply_view.split('/')[1].strip()

        reply_time = post.find('div', class_='endreply box').a.text.strip()
        last_reply = post.find('div', class_='endreply box').span.text.strip()

        if ':' in reply_time:  # 時間是11:27
            date_time = str(datetime.date.today()) + ' ' + reply_time
            date_time = datetime.datetime.strptime(date_time, '%Y-%m-%d %H:%M')
        else:
            date_time = datetime.datetime.strptime('2019-' + reply_time, '%Y-%m-%d').date()

        data_list.append([title, post_link, author, author_page, start_date, reply, last_reply, date_time])

    return data_list


link = "https://bbs.hupu.com/bxj"
soup = get_page(link)
post_all = soup.find('ul', class_="for-list")
post_list = post_all.find_all('li')
data_list = get_data(post_list)
for each in data_list:
    print(each)

            
          

3.獲取前50頁數據

注意一個問題：當翻到第二頁的時候，可能新回復的帖子已經將原來第一頁的帖子推到了第二頁，如果還用insert_one方法將數據存入MongoDB中，那么同一個帖子可能會在數據庫中出現兩次記錄，因此需要改用update方法。

代碼如下：

            
              #!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@File  : MongoAPI.py
@Author: Xinzhe.Pang
@Date  : 2019/7/10 22:15
@Desc  : 
"""
import requests
import datetime
import time
from pymongo import MongoClient
from bs4 import BeautifulSoup

# 定義一個使用MongoDB的類，方便連接數據庫、提取數據庫中內容以及加入或更新數據
class MongoAPI(object):
    def __init__(self, db_ip, db_port, db_name, table_name):
        self.db_ip = db_ip
        self.db_port = db_port
        self.db_name = db_name
        self.table_name = table_name
        self.conn = MongoClient(host=self.db_ip, port=self.db_port)
        self.db = self.conn[self.db_name]
        self.table = self.db[self.table_name]

    def get_one(self, query):
        return self.table.find_one(query, projection={"_id": False})

    def get_all(self, query):
        return self.table.find(query)

    def add(self, kv_dict):
        return self.table.insert(kv_dict)

    def delete(self, query):
        return self.table.delete_many(query)

    def check_exist(self, query):
        ret = self.table.find_one(query)
        return ret != None

    # 如果不存在則會新建
    def update(self, query, kv_dict):
        self.table.update_one(query, {'$set': kv_dict}, upsert=True)

# 獲取頁面內容
def get_page(link):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'}
    r = requests.get(link, headers=headers)
    html = r.content
    html = html.decode('UTF-8')
    soup = BeautifulSoup(html, 'lxml')
    return soup

# 解析頁面數據
def get_data(post_list):
    data_list = []
    for post in post_list:
        title = post.find('div', class_='titlelink box').a.text.strip()
        post_link = post.find('div', class_='titlelink box').a['href']
        post_link = "https://bbs.hupu.com" + post_link

        author = post.find('div', class_='author box').a.text.strip()
        author_page = post.find('div', class_='author box').a['href']
        start_date = post.find('div', class_='author box').contents[5].text.strip()

        reply_view = post.find('span', class_='ansour box').text.strip()
        reply = reply_view.split('/')[0].strip()
        view = reply_view.split('/')[1].strip()

        reply_time = post.find('div', class_='endreply box').a.text.strip()
        last_reply = post.find('div', class_='endreply box').span.text.strip()

        if ':' in reply_time:  # 時間是11:27
            date_time = str(datetime.date.today()) + ' ' + reply_time
            date_time = datetime.datetime.strptime(date_time, '%Y-%m-%d %H:%M')
        else:
            date_time = datetime.datetime.strptime('2019-' + reply_time, '%Y-%m-%d').date()

        data_list.append([title, post_link, author, author_page, start_date, reply, last_reply, date_time])

    return data_list


hupu_post = MongoAPI('111.230.95.186', 27017, 'hupu', 'post')
for i in range(1, 100):
    link = "https://bbs.hupu.com/bxj-" + str(i)
    soup = get_page(link)

    post_all = soup.find('ul', class_="for-list")
    if post_all is None:
        continue
    post_list = post_all.find_all('li')
    data_list = get_data(post_list)

    for each in data_list:
        hupu_post.update({"post_link": each[1]}, {"title": each[0],
                                                  "post_link": each[1],
                                                  "author": each[2],
                                                  "author_page": each[3],
                                                  "start_date": str(each[4]),
                                                  "reply": each[5],
                                                  "last_reply": each[6],
                                                  "last_reply_time": str(each[7])})
    time.sleep(3)
    print('第', i, '頁數據獲取完成，暫停3秒')

            
          

查看數據庫是否寫入：

遇到的問題：'NoneType' object has no attribute 'find_all'

解決方法：在post_list = post_all.find_all('li')之前加入如下代碼：

            if post_all is None:
    continue
          

?參考資料：《Python網絡爬蟲從入門到實踐》