【百度云搜索，搜各種資料:http://www.bdyss.cn】

【搜網(wǎng)盤，搜各種資料:http://www.swpan.cn】

注意：數(shù)據(jù)保存的操作都是在pipelines.py文件里操作的

將數(shù)據(jù)保存為json文件

spider是一個(gè)信號(hào)檢測(cè)

          
            #?-*-?coding:?utf-8?-*-

#?Define?your?item?pipelines?here
#
#?Don't?forget?to?add?your?pipeline?to?the?ITEM_PIPELINES?setting
#?See:?http://doc.scrapy.org/en/latest/topics/item-pipeline.html
from?scrapy.pipelines.images?import?ImagesPipeline??#導(dǎo)入圖片下載器模塊
import?codecs
import?json

class?AdcPipeline(object):??????????????????????#定義數(shù)據(jù)處理類，必須繼承object
????def?__init__(self):
????????self.file?=?codecs.open('shuju.json',?'w',?encoding='utf-8')??#初始化時(shí)打開json文件
????def?process_item(self,?item,?spider):???????#process_item(item)為數(shù)據(jù)處理函數(shù)，接收一個(gè)item，item里就是爬蟲最后yield?item?來的數(shù)據(jù)對(duì)象
????????#?print('文章標(biāo)題是：'?+?item['title'][0])
????????#?print('文章縮略圖url是：'?+?item['img'][0])
????????#?print('文章縮略圖保存路徑是：'?+?item['img_tplj'])??#接收?qǐng)D片下載器填充的，圖片下載后的路徑

????????#將數(shù)據(jù)保存為json文件
????????lines?=?json.dumps(dict(item),?ensure_ascii=False)?+?'\n'???#將數(shù)據(jù)對(duì)象轉(zhuǎn)換成json格式
????????self.file.write(lines)??????????#將json格式數(shù)據(jù)寫入文件
????????return?item
def?spider_closed(self,spider):?????#創(chuàng)建一個(gè)方法繼承spider，spider是一個(gè)信號(hào)，當(dāng)前數(shù)據(jù)操作完成后觸發(fā)這個(gè)方法
????????self.file.close()???????????????#關(guān)閉打開文件

class?imgPipeline(ImagesPipeline):??????????????????????#自定義一個(gè)圖片下載內(nèi)，繼承crapy內(nèi)置的ImagesPipeline圖片下載器類
????def?item_completed(self,?results,?item,?info):??????#使用ImagesPipeline類里的item_completed()方法獲取到圖片下載后的保存路徑
????????for?ok,?value?in?results:
????????????img_lj?=?value['path']?????#接收?qǐng)D片保存路徑
????????????#?print(ok)
????????????item['img_tplj']?=?img_lj??#將圖片保存路徑填充到items.py里的字段里
????????return?item????????????????????#將item給items.py?文件的容器函數(shù)

????#注意：自定義圖片下載器設(shè)置好后，需要在
          
        

將數(shù)據(jù)保存到數(shù)據(jù)庫(kù)

我們使用一個(gè)ORM框架sqlalchemy模塊，保存數(shù)據(jù)

數(shù)據(jù)庫(kù)操作文件

          
            #!/usr/bin/env?python
#?-*-?coding:utf-8?-*-

from?sqlalchemy.ext.declarative?import?declarative_base
from?sqlalchemy?import?Column
from?sqlalchemy?import?Integer,?String,?TIMESTAMP
from?sqlalchemy?import?ForeignKey,?UniqueConstraint,?Index
from?sqlalchemy.orm?import?sessionmaker,?relationship
from?sqlalchemy?import?create_engine

#配置數(shù)據(jù)庫(kù)引擎信息
ENGINE?=?create_engine("mysql+pymysql://root:279819@127.0.0.1:3306/cshi?charset=utf8",?max_overflow=10,?echo=True)

Base?=?declarative_base()???????#創(chuàng)建一個(gè)SQLORM基類

class?SendMsg(Base):????????????#設(shè)計(jì)表
????__tablename__?=?'sendmsg'

????id?=?Column(Integer,?primary_key=True,?autoincrement=True)
????title?=?Column(String(300))
????img_tplj?=?Column(String(300))

def?init_db():
????Base.metadata.create_all(ENGINE)????????#向數(shù)據(jù)庫(kù)創(chuàng)建指定表

def?drop_db():
????Base.metadata.drop_all(ENGINE)??????????#向數(shù)據(jù)庫(kù)刪除指定表

def?session():
????cls?=?sessionmaker(bind=ENGINE)?????????#創(chuàng)建sessionmaker類,操作表
????return?cls()

#?drop_db()?????????#刪除表
#?init_db()?????????#創(chuàng)建表
          
        

pipelines.py文件

          
            #?-*-?coding:?utf-8?-*-

#?Define?your?item?pipelines?here
#
#?Don't?forget?to?add?your?pipeline?to?the?ITEM_PIPELINES?setting
#?See:?http://doc.scrapy.org/en/latest/topics/item-pipeline.html
from?scrapy.pipelines.images?import?ImagesPipeline??#導(dǎo)入圖片下載器模塊
from?adc?import?shujuku?as?ORM??????????????????????#導(dǎo)入數(shù)據(jù)庫(kù)文件

class?AdcPipeline(object):??????????????????????#定義數(shù)據(jù)處理類，必須繼承object
????def?__init__(self):
????????ORM.init_db()???????????????????????????#創(chuàng)建數(shù)據(jù)庫(kù)表
????def?process_item(self,?item,?spider):???????#process_item(item)為數(shù)據(jù)處理函數(shù)，接收一個(gè)item，item里就是爬蟲最后yield?item?來的數(shù)據(jù)對(duì)象
????????print('文章標(biāo)題是：'?+?item['title'][0])
????????print('文章縮略圖url是：'?+?item['img'][0])
????????print('文章縮略圖保存路徑是：'?+?item['img_tplj'])??#接收?qǐng)D片下載器填充的，圖片下載后的路徑

????????mysq?=?ORM.session()
????????shuju?=?ORM.SendMsg(title=item['title'][0],?img_tplj=item['img_tplj'])
????????mysq.add(shuju)
????????mysq.commit()
????????return?item

class?imgPipeline(ImagesPipeline):??????????????????????#自定義一個(gè)圖片下載內(nèi)，繼承crapy內(nèi)置的ImagesPipeline圖片下載器類
????def?item_completed(self,?results,?item,?info):??????#使用ImagesPipeline類里的item_completed()方法獲取到圖片下載后的保存路徑
????????for?ok,?value?in?results:
????????????img_lj?=?value['path']?????#接收?qǐng)D片保存路徑
????????????#?print(ok)
????????????item['img_tplj']?=?img_lj??#將圖片保存路徑填充到items.py里的字段里
????????return?item????????????????????#將item給items.py?文件的容器函數(shù)

????#注意：自定義圖片下載器設(shè)置好后，需要在
          
        

【轉(zhuǎn)載自：http://www.lqkweb.com】