Python 爬虫小Demo 学堂在线课程

import scrapy

class MyscrapyItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    school = scrapy.Field()
    num = scrapy.Field()
    pass

import json

class MyscrapyPipeline(object):
    def open_spider(self, spider):
        try:
            # 这个就是爬虫生成的文件，可以支持好多种格式，这里使用的是json文件
            self.file = open('MyData.json', 'w', encoding="utf-8")
        except Exception as err:
            print(err)

    def process_item(self, item, spider):
        dict_item = dict(item)
        json_str = json.dumps(dict_item, ensure_ascii=False) + "\n"
        self.file.write(json_str)
        return item

    def close_spider(self, spider):
        self.file.close()

import scrapy
import re
from myScrapy.myScrapy.items import MyscrapyItem

class mySpider(scrapy.spiders.Spider):
    # spider的名字
    name = "xuetang"
    # 限制spider爬取的域名
    allowed_domains = ["www.xuetangx.com/"]
	# 爬虫要爬取的网页，是一个列表，按顺序爬取
    start_urls = ["http://www.xuetangx.com/partners"]
	# 这是一种方式，可以爬取网页中所有的项
    # def parse(self, response):
    #     item = MyscrapyItem()
    #     for each in response.xpath("/html/body/article[1]/section/ul/*"):
    #         item['school'] = each.xpath("a/div[2]/h3/text()").extract()
    #         item['num'] = each.xpath("a/div[2]/p[1]/text()").extract()
    #         if item['num']:
    #             item['num'] = re.findall(r'\d+', item['num'][0])
    #         if item['school'] and item['num']:
    #             yield (item)
    # 这是第二种方式，使用for循环，制定爬取项的数目
    def parse(self, response):
        item = MyscrapyItem()
        for i in range(1, 144):
            item['school'] = response.xpath \
                ("/html/body/article[1]/section/ul/li[{}]/a/div[2]/h3/text()".format(i)).extract()
            item['num'] = response.xpath \
                ("/html/body/article[1]/section/ul/li[{}]/a/div[2]/p[1]/text()".format(i)).extract()
            # 判断爬取的项目是否为空，把非空的项目提交
            if item['school'] and item['num']:
                yield (item)

from scrapy import cmdline
# "xuetang"是我们上面spider.py中定义的爬虫名
cmdline.execute("scrapy crawl xuetang".split())