爬虫之pipeline保存数据到redis样本代码

import redis

from scrapy import signals

import json
import codecs
from collections import OrderedDict

class RedisPipeline(object):

    def __init__(self):
        self.r = redis.StrictRedis(host='localhost', port=6379)

    def process_item(self, item, spider):
        if not item[‘id’]:
            print( 'no id item!!')

        str_recorded_item = self.r.get(item[‘id’])
        final_item = None
        if str_recorded_item is None:
            final_item = item
        else:
            ritem = eval(self.r.get(item[‘id’]))
            final_item = dict(item.items() + ritem.items())
        self.r.set(item[‘id’], final_item)

    def close_spider(self, spider):
        return