Merge pull request #166 from redapple/spidertest

Move tests one-level up + add code coverage + codecov on Travis CI

.coveragerc

@@ -0,0 +1,3 @@
+[run]
+branch = true
+source = frontera

.travis.yml

@@ -1,17 +1,25 @@
 language: python
 python: 2.7
+
 env:
   - TOXENV=py27
   - TOXENV=flake8
+
 install:
-  - pip install -U tox wheel
+  - pip install -U tox wheel codecov
   - pip install -r requirements/tests.txt
+
 before_script:
   - mysql -u root -e "set global innodb_large_prefix=1;"
   - mysql -u root -e "set global innodb_file_format='Barracuda';"
   - mysql -u root -e "set global innodb_file_per_table=true;"
-  - frontera/tests/run_zmq_broker.sh
+  - tests/run_zmq_broker.sh
+
 script: tox
+
+after_success:
+  - codecov
+
 deploy:
   provider: pypi
   distributions: sdist bdist_wheel

examples/scrapy_recording/scrapy_recording/spiders/recorder.py

@@ -1,7 +1,6 @@
-from scrapy.contrib.linkextractors.lxmlhtml import LxmlLinkExtractor
-from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
-from scrapy.contrib.linkextractors.regex import RegexLinkExtractor
-from scrapy.contrib.spiders import CrawlSpider, Rule
+from scrapy.linkextractors import LinkExtractor
+from scrapy.linkextractors.regex import RegexLinkExtractor
+from scrapy.spiders import CrawlSpider, Rule
 
 
 DOMAIN = 'diffeo.com'
@@ -26,8 +25,7 @@ class MySpider(CrawlSpider):
     allowed_domains = [DOMAIN]
 
     rules = [Rule(FallbackLinkExtractor([
-        LxmlLinkExtractor(allow=ALLOWED_RE),
-        SgmlLinkExtractor(allow=ALLOWED_RE),
+        LinkExtractor(allow=ALLOWED_RE),
         RegexLinkExtractor(allow=ALLOWED_RE),
     ]), callback='parse_page', follow=True)]
 

requirements/tests.txt

@@ -9,4 +9,4 @@ cachetools
 pyzmq
 msgpack-python
 kafka-python<=0.9.5
-
+pytest-cov

frontera/contrib/backends/memory/test_backend.py → tests/contrib/backends/memory/test_backend_memory.py

@@ -1,5 +1,5 @@
-from frontera.tests.test_overused_buffer import DFSOverusedBackendTest
-from frontera.tests import backends
+from tests.test_overused_buffer import DFSOverusedBackendTest
+from tests import backends
 
 
 class TestFIFO(backends.FIFOBackendTest):

frontera/contrib/backends/sqlalchemy/test_backend.py → tests/contrib/backends/sqlalchemy/test_backend_sqlalchemy.py

@@ -4,8 +4,8 @@
 from psycopg2 import connect
 from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
 
-from frontera.tests import backends
-from frontera.tests.test_revisiting_backend import RevisitingBackendTest
+from tests import backends
+from tests.test_revisiting_backend import RevisitingBackendTest
 
 
 #----------------------------------------------------

frontera/tests/scrapy_spider/settings.py → tests/scrapy_spider/settings.py

@@ -3,8 +3,8 @@
 #--------------------------------------------------------------------------
 BOT_NAME = 'scrapy_spider'
 
-SPIDER_MODULES = ['frontera.tests.scrapy_spider.spiders']
-NEWSPIDER_MODULE = 'frontera.tests.scrapy_spider.spiders'
+SPIDER_MODULES = ['tests.scrapy_spider.spiders']
+NEWSPIDER_MODULE = 'tests.scrapy_spider.spiders'
 
 HTTPCACHE_ENABLED = False
 REDIRECT_ENABLED = True
@@ -30,7 +30,7 @@
     {'frontera.contrib.scrapy.middlewares.schedulers.SchedulerDownloaderMiddleware': 999}
 )
 SCHEDULER = 'frontera.contrib.scrapy.schedulers.frontier.FronteraScheduler'
-FRONTERA_SETTINGS = 'frontera.tests.scrapy_spider.frontera.settings'
+FRONTERA_SETTINGS = 'tests.scrapy_spider.frontera.settings'
 
 
 #--------------------------------------------------------------------------

frontera/tests/scrapy_spider/spiders/example.py → tests/scrapy_spider/spiders/example.py

@@ -1,6 +1,6 @@
-from scrapy.contrib.linkextractors.lxmlhtml import LxmlLinkExtractor
-from scrapy.contrib.linkextractors.regex import RegexLinkExtractor
-from scrapy.contrib.spiders import CrawlSpider, Rule
+from scrapy.linkextractors import LinkExtractor
+from scrapy.linkextractors.regex import RegexLinkExtractor
+from scrapy.spiders import CrawlSpider, Rule
 
 
 class FallbackLinkExtractor(object):
@@ -19,7 +19,7 @@ class MySpider(CrawlSpider):
     callback_calls = 0
 
     rules = [Rule(FallbackLinkExtractor([
-        LxmlLinkExtractor(),
+        LinkExtractor(),
         RegexLinkExtractor(),
     ]), callback='parse_page', follow=True)]
 

frontera/tests/test_frontera_scheduler.py → tests/test_frontera_scheduler.py

@@ -1,6 +1,6 @@
 from frontera.contrib.scrapy.schedulers.frontier import FronteraScheduler
-from frontera.tests.mocks.frontier_manager import FakeFrontierManager
-from frontera.tests.mocks.crawler import FakeCrawler
+from tests.mocks.frontier_manager import FakeFrontierManager
+from tests.mocks.crawler import FakeCrawler
 from frontera.core.models import Request as FRequest
 from frontera.core.models import Response as FResponse
 from scrapy.http import Request, Response

frontera/tests/test_frontier_manager.py → tests/test_frontier_manager.py

@@ -12,12 +12,12 @@ class TestFrontierManager(object):
 
     def setup_frontier_manager(self, settings=None):
         settings = settings or Settings()
-        settings.BACKEND = 'frontera.tests.mocks.components.FakeBackend'
-        settings.MIDDLEWARES = ['frontera.tests.mocks.components.FakeMiddleware',
-                                'frontera.tests.mocks.components.FakeMiddlewareModifySeeds',
-                                'frontera.tests.mocks.components.FakeMiddlewareModifyResponse',
-                                'frontera.tests.mocks.components.FakeMiddlewareModifyLinks']
-        settings.CANONICAL_SOLVER = 'frontera.tests.mocks.components.FakeCanonicalSolver'
+        settings.BACKEND = 'tests.mocks.components.FakeBackend'
+        settings.MIDDLEWARES = ['tests.mocks.components.FakeMiddleware',
+                                'tests.mocks.components.FakeMiddlewareModifySeeds',
+                                'tests.mocks.components.FakeMiddlewareModifyResponse',
+                                'tests.mocks.components.FakeMiddlewareModifyLinks']
+        settings.CANONICAL_SOLVER = 'tests.mocks.components.FakeCanonicalSolver'
         return FrontierManager.from_settings(settings)
 
     def test_start(self):
@@ -98,13 +98,13 @@ def test_max_requests_reached(self):
 
     def test_blocking_middleware(self):
         settings = Settings()
-        settings.BACKEND = 'frontera.tests.mocks.components.FakeBackend'
-        settings.MIDDLEWARES = ['frontera.tests.mocks.components.FakeMiddleware',
-                                'frontera.tests.mocks.components.FakeMiddlewareModifySeeds',
-                                'frontera.tests.mocks.components.FakeMiddlewareBlocking',
-                                'frontera.tests.mocks.components.FakeMiddlewareModifyResponse',
-                                'frontera.tests.mocks.components.FakeMiddlewareModifyLinks']
-        settings.CANONICAL_SOLVER = 'frontera.tests.mocks.components.FakeCanonicalSolver'
+        settings.BACKEND = 'tests.mocks.components.FakeBackend'
+        settings.MIDDLEWARES = ['tests.mocks.components.FakeMiddleware',
+                                'tests.mocks.components.FakeMiddlewareModifySeeds',
+                                'tests.mocks.components.FakeMiddlewareBlocking',
+                                'tests.mocks.components.FakeMiddlewareModifyResponse',
+                                'tests.mocks.components.FakeMiddlewareModifyLinks']
+        settings.CANONICAL_SOLVER = 'tests.mocks.components.FakeCanonicalSolver'
         fm = FrontierManager.from_settings(settings)
         fm.add_seeds([r1, r2, r3])
         response = Response(r1.url, request=r1)

frontera/tests/test_overused_buffer.py → tests/test_overused_buffer.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from frontera.tests.backends import BackendSequenceTest, TEST_SITES
+from tests.backends import BackendSequenceTest, TEST_SITES
 from frontera.utils.tester import DownloaderSimulator, BaseDownloaderSimulator
 from urlparse import urlparse
 

frontera/tests/test_revisiting_backend.py → tests/test_revisiting_backend.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from frontera.tests.backends import BackendSequenceTest, TEST_SITES
+from tests.backends import BackendSequenceTest, TEST_SITES
 from frontera.utils.tester import FrontierTester
 
 from datetime import timedelta

frontera/tests/test_scrapy_settings_adapter.py → tests/test_scrapy_settings_adapter.py

@@ -10,6 +10,6 @@ def test_fallsback_to_crawler_settings():
 
 def test_frontera_settings_have_precedence_over_crawler_settings():
     crawler_settings = {'MAX_REQUESTS': 10,
-                        'FRONTERA_SETTINGS': 'frontera.tests.scrapy_spider.frontera.settings'}
+                        'FRONTERA_SETTINGS': 'tests.scrapy_spider.frontera.settings'}
     settings = ScrapySettingsAdapter(crawler_settings)
     assert settings.get('MAX_REQUESTS') == 5

frontera/tests/test_scrapy_spider.py → tests/test_scrapy_spider.py

@@ -1,14 +1,14 @@
 # -*- coding: utf-8 -*-
 from twisted.internet import reactor
 from scrapy.crawler import Crawler
-from scrapy import log, signals
+from scrapy import signals
 from scrapy.settings import Settings
-from scrapy_spider.spiders.example import MySpider
+from tests.scrapy_spider.spiders.example import MySpider
 
 
 def test_scrapy_spider():
     settings = Settings()
-    settings.setmodule("frontera.tests.scrapy_spider.settings")
+    settings.setmodule("tests.scrapy_spider.settings")
     crawler = Crawler(MySpider, settings=settings)
     crawler.signals.connect(reactor.stop, signal=signals.spider_closed)
     crawler.crawl()

frontera/tests/test_settings.py → tests/test_settings.py

@@ -4,7 +4,7 @@
 
 
 def test_settings_on_a_python_module_are_loaded():
-    settings = Settings('frontera.tests.scrapy_spider.frontera.settings')
+    settings = Settings('tests.scrapy_spider.frontera.settings')
     assert settings.get('MAX_REQUESTS') == 5
 
 
@@ -30,7 +30,7 @@ def test_settings_attributes_can_be_assigned():
 
 
 def test_object_from_loads_settings_from_a_module():
-    module = 'frontera.tests.scrapy_spider.frontera.settings'
+    module = 'tests.scrapy_spider.frontera.settings'
     settings = BaseSettings.object_from(module)
     assert settings.get('MAX_REQUESTS') == 5
 

frontera/tests/test_utils_misc.py → tests/test_utils_misc.py

@@ -29,19 +29,19 @@ def test_non_multiple_length(self):
 class TestLoadObject(object):
 
     def test_load_class(self):
-        obj = load_object('frontera.tests.mocks.load_objects.MockClass')
+        obj = load_object('tests.mocks.load_objects.MockClass')
         assert obj.val == 10
 
     def test_load_instance(self):
-        obj = load_object('frontera.tests.mocks.load_objects.mock_instance')
+        obj = load_object('tests.mocks.load_objects.mock_instance')
         assert obj.val == 5
 
     def test_load_variable(self):
-        obj = load_object('frontera.tests.mocks.load_objects.mock_variable')
+        obj = load_object('tests.mocks.load_objects.mock_variable')
         assert obj == 'test'
 
     def test_load_function(self):
-        obj = load_object('frontera.tests.mocks.load_objects.mock_function')
+        obj = load_object('tests.mocks.load_objects.mock_function')
         assert obj() == 2
 
     def test_value_error(self):
@@ -57,6 +57,6 @@ def test_import_error(self):
 
     def test_name_error(self):
         with pytest.raises(NameError) as info:
-            load_object('frontera.tests.mocks.load_objects.non_existent_object')
-        assert info.value.message == ("Module 'frontera.tests.mocks.load_objects' doesn't define"
+            load_object('tests.mocks.load_objects.non_existent_object')
+        assert info.value.message == ("Module 'tests.mocks.load_objects' doesn't define"
                                       " any object named 'non_existent_object'")

tox.ini

@@ -13,7 +13,7 @@ deps =
     -r{toxinidir}/requirements.txt
     -r{toxinidir}/requirements/tests.txt
 commands =
-    py.test -s -v {envsitepackagesdir}/frontera
+    py.test --cov-report=term --cov=frontera -s -v tests
 
 [testenv:flake8]
 changedir = {toxinidir}