next_page = response.xpath('//*[@id="content-left"]/ul/li[last()]/a').attrib['href']

  if next_page is not None:      yield response.follow(next_page, callback=self.parse)

# if next_page is not None:      # next_page = response.urljoin(next_page)      # yield scrapy.Request(next_page, callback=self.parse)

import scrapy

class QiushibaikeItem(scrapy.Item):    # define the fields for your item here like:    # name = scrapy.Field()    author = scrapy.Field()    content = scrapy.Field()    _id = scrapy.Field()

    def parse(self, response):
        content_left_div = response.xpath('//*[@id="content-left"]')        content_list_div = content_left_div.xpath('./div')
        for content_div in content_list_div:            item = QiushibaikeItem()            item['author'] = content_div.xpath('./div/a[2]/h2/text()').get()            item['content'] = content_div.xpath('./a/div/span/text()').getall()            item['_id'] = content_div.attrib['id']            yield item
        next_page = response.xpath('//*[@id="content-left"]/ul/li[last()]/a').attrib['href']
        if next_page is not None:            yield response.follow(next_page, callback=self.parse)

class QiushibaikePipeline(object):
    def __init__(self):        self.connection = pymongo.MongoClient('localhost', 27017)        self.db = self.connection.scrapy          self.collection = self.db.qiushibaike 
    def process_item(self, item, spider):        if not self.connection or not item:            return        self.collection.save(item)
    def __del__(self):        if self.connection:            self.connection.close()

# See https://doc.scrapy.org/en/latest/topics/item-pipeline.htmlITEM_PIPELINES = {   'qiushibaike.pipelines.QiushibaikePipeline': 300,}

# Crawl responsibly by identifying yourself (and your website) on the user-agentUSER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/73.0.3683.86 Chrome/73.0.3683.86 Safari/537.36'

scrapy crawl qiushibaike