Share the example code stored in the pipeline

When the last function in the spider returns item, scrapy will call the

process_item(self, item, spider):
函数并传入item，spider等参数
在这里可以将数据进行持久化储存
我的piple代码

# -*- coding: utf-8 -*-# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.htmlimport MySQLdb
import MySQLdb.cursorsfrom twisted.enterprise import adbapiclass MyPipeline(object):  ##这里的函数名于setting中的对应

    def __init__(self, dbpool):
        self.dbpool = dbpool
    @classmethod 　　 ##得到数据库的连接
    def from_settings(cls, settings):
        dbargs = dict(
                host = settings['MYSQL_HOST'],
                db = settings['MYSQL_DBNAME'],
                port = settings['MYSQL_PORT'],
                user = settings['MYSQL_USER'],
                passwd = settings['MYSQL_PASSWD'],
                charset = 'utf8',
                cursorclass = MySQLdb.cursors.DictCursor,
                use_unicode = True,
            )
        dbpool = adbapi.ConnectionPool('MySQLdb', **dbargs)return cls(dbpool)

    def process_item(self, item, spider): ##这个函数会在spider返回时调用
        d = self.dbpool.runInteraction(self._do_upinsert, item, spider)return item

    def _do_upinsert(self, conn, item, spider):
        valid = Truefor data in item:if not data:
                valid = Falseif valid: 　　　　　　　　##执行sql
            result = conn.execute(‘sql’)  if result:
                print 'added a record'else:
                print 'failed insert into table'

代码git地址：过几天会上传

The above is the detailed content of Share the example code stored in the pipeline. For more information, please follow other related articles on the PHP Chinese website!