docker-scrapy-ping/ping.py

import re

from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule


class PingSpider(CrawlSpider):

    name = 'ping'

    custom_settings = {
        # default: 3 min
        'DOWNLOAD_TIMEOUT': 30, # [s]
        # WORKAROUND
        # [boto] ERROR: Unable to read instance data, giving up
        'DOWNLOAD_HANDLERS': {'s3': None},
    }

    def __init__(self, start_url):
        self.start_urls = [start_url]
        self.rules = (
            Rule(LinkExtractor(allow='^'+re.escape(start_url))),
            Rule(LinkExtractor(tags='img', attrs='src', deny_extensions=set())),
            Rule(LinkExtractor(tags='link', attrs='href', deny_extensions=set())),
        )
        super(PingSpider, self).__init__()