Qt與Python腳本實戰之一(爬蟲)

• Python環境安裝

  1.Python版本選擇(2.7 or 3.6.x) 版本不同區別蠻大

  2.安裝pip 一個Python包管理工具 類似nodejs的npm(都是提供了海量第三方包)

• 編寫python代碼實現爬取.

  1.需要用到的庫有: Requests lxml 如果沒有安裝的請自己安裝一下(pip install xxx)

  2.IDE : pycharm or Qtcreator

  3.python 版本: 3.6

  4.代碼實現的是多線程下載

• 實現功能

1. 爬取指定網站的圖片

2. 按分類寫入本地目錄

3. 按分類將本地圖片寫成ppt

4. QML界面展示爬取的圖片內容

5. 代碼展示

            
              # This Python file uses the following encoding: utf-8

# if__name__ == "__main__":
#     pass
import sys
import requests
import os
import pathlib
import pptx
from pptx.util import Inches
from lxml import etree
from threading import *
from time import sleep

nMaxThread = 3  #這里設置需要開啟幾條線程
ThreadLock = BoundedSemaphore(nMaxThread)

gHeads = {
    "User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36",
}
#開始將該目錄下的圖片進行插入到ppt操作
def writeppt(subDir):
    print('subDir: ',subDir)
    pptFile = pptx.Presentation()

    picFiles = [fn for fn in os.listdir(subDir) if fn.endswith('.JPG') or fn.endswith('.jpg')]
#    print('fn: ',fn)
    # 按圖片編號順序導入
    for fn in sorted(picFiles, key=lambda item:str(item[:item.rindex('.')])):
        slide = pptFile.slides.add_slide(pptFile.slide_layouts[1])
        print('fn: ',fn)
        # 為PPTX文件當前幻燈片中第一個文本框設置文字，本文代碼中可忽略
        slide.shapes.placeholders[0].text = fn[:fn.rindex('.')]
        fullfn = subDir+'\\'+fn
        # 導入并為當前幻燈片添加圖片，起始位置和尺寸可修改
        slide.shapes.add_picture(fullfn, Inches(0), Inches(0), Inches(10), Inches(7.5))
    pptFile.save("%s.pptx"%(subDir))

#開始輪詢某個目錄下的子目錄
def lookupRootDir(root):
    dirs = os.listdir( root )
    print('dirs: ',dirs,root)
    # 輸出所有文件和文件夾
    for file in dirs:
       full = root+'\\'+file;
       path = pathlib.Path(full)
       print('file: ',path.is_dir())
       if path.is_dir():
           writeppt(full)

#開始將該網站下的圖片進行分析 下載等操作的類
class JinTu(Thread):
    def __init__(self,mainReferer,url,title):
        Thread.__init__(self)
        self.MainReferer = mainReferer
        self.url = url
        self.title = title[20:]  #這里是為了把
              
              給刪除
        self.dir  =  title[20:-8];
        print('dir: ',self.dir)

    def run(self):
        try:
            urlList = [self.url];
            if len(urlList) > 0 and urlList != None:
                self.SavePath(urlList)

        finally:
            ThreadLock.release()

    def GetPhotoUrl(self):
        heads={
            "Referer":self.MainReferer
        }
        heads.update(gHeads)
        html = requests.get(self.url,headers=heads)
        if html.status_code == 200:
            xmlContent = etree.HTML(html.text)
            urlList = xmlContent.xpath("http://div/@datasrc")
            print('url list: '+urlList)
            return urlList
        else:
            return None

    def SavePath(self,urlList):
        heads = {
            "Referer": self.url
        }
        heads.update(gHeads)
        savePath = "./photo/%s" % self.dir
        if not os.path.exists(savePath):
            os.makedirs(savePath)
        for i in range(len(urlList)):
            j = 0
            while j<5:
                #print("Download : %s/%d.jpg" % (self.title.encode("gbk"), i + 1))
                print("Download Url: %s" %(urlList[i]))
                html = requests.get(urlList[i],headers=heads)
                if html.status_code == 200:
                    with open(savePath + "/%s"%(self.title),"wb") as f:
                        f.write(html.content)
                    break
                elif html.status_code == 404:
                    j+=1
                    sleep(0.05)
                    continue
                else:
                    return None

#開始進行爬蟲操作
def startSplider():
    nNum = 35
    for i in range(nNum):
        myid = str(i);
        myid = myid.zfill(2)
        url = "http://www.jinfutech.com/wx/pyitem/item/PPTView.Aspx?ID=%s"%(myid)
        html = requests.get(url,headers=gHeads)
        if html.status_code == 200:
            xmlContent = etree.HTML(html.content)
#http://www.jinfutech.com/wx/pyitem/Style/PPT/Images/Item02-113.JPG?id=1
#../Style/PPT/Images/Item01-003.JPG?id=1
            hrefList = xmlContent.xpath("http://div/@datasrc")
            for i in range(len(hrefList)):
                ThreadLock.acquire()
                partUrl = hrefList[i][3:]
                t = JinTu(url,"http://www.jinfutech.com/wx/pyitem/"+partUrl,hrefList[i])
                t.start()
 #主函數入口
if __name__ == '__main__':
    startSplider()
    lookupRootDir(sys.path[0]+'\photo')


            
          

            
              import QtQuick 2.10
import QtQuick.Window 2.10
import io.thp.pyotherside 1.3 //導入qml插件 具體插件見底部說明

//http://www.jinfutech.com/wx/pyitem/item/PPTView.Aspx?ID=02
Window {
    visible: true
    width: 640
    height: 480
    title: qsTr("Hello World")
    Python {
        id: py //Python實例
        Component.onCompleted: {
            addImportPath(Qt.resolvedUrl('.'));
            //jintu 為py文件 注意如果直接使用python.exe 運行則需要去掉py文件中的main函數
            importModule('jintu', function (success) {
                console.log('module imported: ' + success);
                //等待模塊初始化完進行py方法調用
                getCoinlist("startSplider",0);
            });
        }
    }
    function getCoinlist(functionName,pageId){
        var functionId = 'jintu.'+functionName;
        py.call(functionId, '', function(result) {
            console.log();
        });
    }
}