Custom spider argument support

arachnado/handlers.py

@@ -76,6 +76,19 @@ def find_spider_cls(spider_name, spider_packages):
                     return spider_cls
 
 
+def set_spider_class_args(spider_cls, **kwargs):
+    """ Creates new spider class based on given spider class to keep the
+    original spider class consistent.
+
+    :param spider_cls: Original spider class
+    :param kwargs: Custom spider attributed to be set for this particular crawl
+    """
+    if kwargs:
+        return type(
+            '{}Customized'.format(spider_cls.__name__), (spider_cls,), kwargs)
+    return spider_cls
+
+
 class BaseRequestHandler(RequestHandler):
 
     def initialize(self, crawler_process, opts):
@@ -91,6 +104,25 @@ def render(self, *args, **kwargs):
         kwargs['initial_process_stats_json'] = json_encode(proc_stats)
         return super(BaseRequestHandler, self).render(*args, **kwargs)
 
+    def get_custom_spider_arguments(self, ignore=None):
+        """ Returns custom spider arguments embedded in request
+
+        :param ignore: List of arguments to be ignored. Defaults to 'domain'.
+        """
+        ignore = ignore or ['domain']
+        if self.json_args:
+            args = self.json_args
+            getter = self.json_args.get
+        else:
+            content_type = self.request.headers.get('Content-Type', '').lower()
+            if 'x-www-form-urlencoded' in content_type:
+                args = self.request.body_arguments
+                getter = self.get_body_argument
+            else:
+                args = self.request.query_arguments
+                getter = self.get_query_argument
+        return {arg: getter(arg) for arg in args if arg not in ignore}
+
 
 class Index(NoEtagsMixin, BaseRequestHandler):
 
@@ -109,7 +141,7 @@ class StartCrawler(ApiHandler, BaseRequestHandler):
     """
     This endpoint starts crawling for a domain.
     """
-    def crawl(self, domain):
+    def crawl(self, domain, **kwargs):
         storage_opts = self.opts['arachnado.storage']
         settings = {
             'MOTOR_PIPELINE_ENABLED': storage_opts['enabled'],
@@ -120,22 +152,24 @@ def crawl(self, domain):
         spider_cls = get_spider_cls(domain, self._get_spider_package_names())
 
         if spider_cls is not None:
+            spider_cls = set_spider_class_args(spider_cls, **kwargs)
             self.crawler = create_crawler(settings, spider_cls=spider_cls)
             self.crawler_process.crawl(self.crawler, domain=domain)
             return True
         return False
 
     def post(self):
+        spider_args = self.get_custom_spider_arguments()
         if self.is_json:
             domain = self.json_args['domain']
-            if self.crawl(domain):
+            if self.crawl(domain, **spider_args):
                 self.write({"status": "ok",
                             "job_id": self.crawler.spider.crawl_id})
             else:
                 self.write({"status": "error"})
         else:
             domain = self.get_body_argument('domain')
-            if self.crawl(domain):
+            if self.crawl(domain, **spider_args):
                 self.redirect("/")
             else:
                 raise HTTPError(400)

arachnado/mixins.py

@@ -0,0 +1,28 @@
+import logging
+import datetime
+
+
+class ArachnadoSpiderMixin(object):
+    """
+    An arachnado spider mixin that contains common attributes and utilities for
+    all Arachnado spiders
+    """
+    crawl_id = None
+    domain = None
+    motor_job_id = None
+
+    def __init__(self, *args, **kwargs):
+        super(ArachnadoSpiderMixin, self).__init__(*args, **kwargs)
+        # don't log scraped items
+        logging.getLogger("scrapy.core.scraper").setLevel(logging.INFO)
+
+    def get_page_item(self, response, type_='page'):
+        return {
+            'crawled_at': datetime.datetime.utcnow(),
+            'url': response.url,
+            'status': response.status,
+            'headers': response.headers,
+            'body': response.body_as_unicode(),
+            'meta': response.meta,
+            '_type': type_,
+        }

arachnado/spider.py

@@ -1,14 +1,14 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
-import datetime
-import logging
 
 import scrapy
 from scrapy.linkextractors import LinkExtractor
 from scrapy.http.response.html import HtmlResponse
+from scrapy.spiders.crawl import CrawlSpider
 
 from .utils import MB, add_scheme_if_missing, get_netloc
 from .crawler_process import ArachnadoCrawler
+from .mixins import ArachnadoSpiderMixin
 
 
 DEFAULT_SETTINGS = {
@@ -68,30 +68,12 @@ def create_crawler(settings=None, spider_cls=None):
     return ArachnadoCrawler(spider_cls, _settings)
 
 
-class ArachnadoSpider(scrapy.Spider):
-    """
-    A base spider that contains common attributes and utilities for all
-    Arachnado spiders
-    """
-    crawl_id = None
-    domain = None
-    motor_job_id = None
+class ArachnadoSpider(scrapy.Spider, ArachnadoSpiderMixin):
+    pass
 
-    def __init__(self, *args, **kwargs):
-        super(ArachnadoSpider, self).__init__(*args, **kwargs)
-        # don't log scraped items
-        logging.getLogger("scrapy.core.scraper").setLevel(logging.INFO)
-
-    def get_page_item(self, response, type_='page'):
-        return {
-            'crawled_at': datetime.datetime.utcnow(),
-            'url': response.url,
-            'status': response.status,
-            'headers': response.headers,
-            'body': response.body_as_unicode(),
-            'meta': response.meta,
-            '_type': type_,
-        }
+
+class ArachnadoCrawlSpider(CrawlSpider, ArachnadoSpiderMixin):
+    pass
 
 
 class CrawlWebsiteSpider(ArachnadoSpider):