python scrapy多个items多管道的使用

追影

多个items
这个比较简单，在items.py文件内创建相应的类，在spider中引入即可
items.py

1. import scrapy
   
2. class MymultispiderItem(scrapy.Item):
   
3.     # define the fields for your item here like:
   
4.     # name = scrapy.Field()
   
5.     pass
   
6. 
   
7. class Myspd1spiderItem(scrapy.Item):
   
8.     name = scrapy.Field()
   
9. 
   
10. class Myspd2spiderItem(scrapy.Item):
    
11.     name = scrapy.Field()
    
12. 
    
13. class Myspd3spiderItem(scrapy.Item):
    
14.     name = scrapy.Field()

复制代码

spider内使用对应的items

1. import scrapy
   
2. from mymultispider.items import Myspd1spiderItem
   
3. 
   
4. class Myspd1Spider(scrapy.Spider):
   
5.     name = 'myspd1'
   
6.     allowed_domains = ['sina.com.cn']
   
7.     start_urls = ['http://sina.com.cn/']
   
8. 
   
9.     def parse(self, response):
   
10.         print('myspd1')
    
11.         item = Myspd1spiderItem()
    
12.         item['name'] = 'myspd1的pipelines'
    
13.         yield item

复制代码

四，指定pipelines


1,这个也有两种方法，方法一，定义多个pipeline类：


pipelines.py文件内：

1. 
   
2. class Myspd1spiderPipeline:
   
3.     def process_item(self,item,spider):
   
4.         print(item['name'])
   
5.         return item
   
6. 
   
7. class Myspd2spiderPipeline:
   
8.     def process_item(self,item,spider):
   
9.         print(item['name'])
   
10.         return item
    
11. 
    
12. class Myspd3spiderPipeline:
    
13.     def process_item(self,item,spider):
    
14.         print(item['name'])
    
15.         return item

复制代码

1.1settings.py文件开启管道

1. ITEM_PIPELINES = {
   
2.    'mymultispider.pipelines.Myspd1spiderPipeline': 300,
   
3.    'mymultispider.pipelines.Myspd2spiderPipeline': 300,
   
4.    'mymultispider.pipelines.Myspd3spiderPipeline': 300,
   
5. }

复制代码

1.2spider中设置管道

1. mport scrapy
   
2. from mymultispider.items import Myspd1spiderItem
   
3. 
   
4. class Myspd1Spider(scrapy.Spider):
   
5.     name = 'myspd1'
   
6.     allowed_domains = ['sina.com.cn']
   
7.     start_urls = ['http://sina.com.cn/']
   
8.     custom_settings = {
   
9.         'ITEM_PIPELINES': {'mymultispider.pipelines.Myspd1spiderPipeline': 300},
   
10.     }
    
11. 
    
12.     def parse(self, response):
    
13.         print('myspd1')
    
14.         item = Myspd1spiderItem()
    
15.         item['name'] = 'myspd1的pipelines'
    
16.         yield item

复制代码

指定管道的代码

1. custom_settings = {
   
2.         'ITEM_PIPELINES': {'mymultispider.pipelines.Myspd1spiderPipeline': 300},
   
3.     }

复制代码