diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..07931e9ea --- /dev/null +++ b/.coveragerc @@ -0,0 +1,3 @@ +[run] +branch = true +source = frontera diff --git a/.travis.yml b/.travis.yml index 61fc76a2e..23523240f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,17 +1,25 @@ language: python python: 2.7 + env: - TOXENV=py27 - TOXENV=flake8 + install: - - pip install -U tox wheel + - pip install -U tox wheel codecov - pip install -r requirements/tests.txt + before_script: - mysql -u root -e "set global innodb_large_prefix=1;" - mysql -u root -e "set global innodb_file_format='Barracuda';" - mysql -u root -e "set global innodb_file_per_table=true;" - - frontera/tests/run_zmq_broker.sh + - tests/run_zmq_broker.sh + script: tox + +after_success: + - codecov + deploy: provider: pypi distributions: sdist bdist_wheel diff --git a/examples/scrapy_recording/scrapy_recording/spiders/recorder.py b/examples/scrapy_recording/scrapy_recording/spiders/recorder.py index 8d62b5bfb..d53c10de0 100644 --- a/examples/scrapy_recording/scrapy_recording/spiders/recorder.py +++ b/examples/scrapy_recording/scrapy_recording/spiders/recorder.py @@ -1,7 +1,6 @@ -from scrapy.contrib.linkextractors.lxmlhtml import LxmlLinkExtractor -from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor -from scrapy.contrib.linkextractors.regex import RegexLinkExtractor -from scrapy.contrib.spiders import CrawlSpider, Rule +from scrapy.linkextractors import LinkExtractor +from scrapy.linkextractors.regex import RegexLinkExtractor +from scrapy.spiders import CrawlSpider, Rule DOMAIN = 'diffeo.com' @@ -26,8 +25,7 @@ class MySpider(CrawlSpider): allowed_domains = [DOMAIN] rules = [Rule(FallbackLinkExtractor([ - LxmlLinkExtractor(allow=ALLOWED_RE), - SgmlLinkExtractor(allow=ALLOWED_RE), + LinkExtractor(allow=ALLOWED_RE), RegexLinkExtractor(allow=ALLOWED_RE), ]), callback='parse_page', follow=True)] diff --git a/requirements/tests.txt b/requirements/tests.txt index 485f80ba0..935d5909c 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -9,4 +9,4 @@ cachetools pyzmq msgpack-python kafka-python<=0.9.5 - +pytest-cov diff --git a/frontera/tests/__init__.py b/tests/__init__.py similarity index 100% rename from frontera/tests/__init__.py rename to tests/__init__.py diff --git a/frontera/tests/backends.py b/tests/backends.py similarity index 100% rename from frontera/tests/backends.py rename to tests/backends.py diff --git a/frontera/contrib/backends/memory/test_backend.py b/tests/contrib/backends/memory/test_backend_memory.py similarity index 88% rename from frontera/contrib/backends/memory/test_backend.py rename to tests/contrib/backends/memory/test_backend_memory.py index 71a03e902..796852a72 100644 --- a/frontera/contrib/backends/memory/test_backend.py +++ b/tests/contrib/backends/memory/test_backend_memory.py @@ -1,5 +1,5 @@ -from frontera.tests.test_overused_buffer import DFSOverusedBackendTest -from frontera.tests import backends +from tests.test_overused_buffer import DFSOverusedBackendTest +from tests import backends class TestFIFO(backends.FIFOBackendTest): diff --git a/frontera/contrib/backends/sqlalchemy/test_backend.py b/tests/contrib/backends/sqlalchemy/test_backend_sqlalchemy.py similarity index 97% rename from frontera/contrib/backends/sqlalchemy/test_backend.py rename to tests/contrib/backends/sqlalchemy/test_backend_sqlalchemy.py index 45f8c6795..b4e5dc030 100644 --- a/frontera/contrib/backends/sqlalchemy/test_backend.py +++ b/tests/contrib/backends/sqlalchemy/test_backend_sqlalchemy.py @@ -4,8 +4,8 @@ from psycopg2 import connect from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT -from frontera.tests import backends -from frontera.tests.test_revisiting_backend import RevisitingBackendTest +from tests import backends +from tests.test_revisiting_backend import RevisitingBackendTest #---------------------------------------------------- diff --git a/frontera/tests/mocks/__init__.py b/tests/mocks/__init__.py similarity index 100% rename from frontera/tests/mocks/__init__.py rename to tests/mocks/__init__.py diff --git a/frontera/tests/mocks/components.py b/tests/mocks/components.py similarity index 100% rename from frontera/tests/mocks/components.py rename to tests/mocks/components.py diff --git a/frontera/tests/mocks/crawler.py b/tests/mocks/crawler.py similarity index 100% rename from frontera/tests/mocks/crawler.py rename to tests/mocks/crawler.py diff --git a/frontera/tests/mocks/frontier_manager.py b/tests/mocks/frontier_manager.py similarity index 100% rename from frontera/tests/mocks/frontier_manager.py rename to tests/mocks/frontier_manager.py diff --git a/frontera/tests/mocks/load_objects.py b/tests/mocks/load_objects.py similarity index 100% rename from frontera/tests/mocks/load_objects.py rename to tests/mocks/load_objects.py diff --git a/frontera/tests/run_zmq_broker.sh b/tests/run_zmq_broker.sh similarity index 100% rename from frontera/tests/run_zmq_broker.sh rename to tests/run_zmq_broker.sh diff --git a/frontera/tests/scrapy_spider/__init__.py b/tests/scrapy_spider/__init__.py similarity index 100% rename from frontera/tests/scrapy_spider/__init__.py rename to tests/scrapy_spider/__init__.py diff --git a/frontera/tests/scrapy_spider/frontera/__init__.py b/tests/scrapy_spider/frontera/__init__.py similarity index 100% rename from frontera/tests/scrapy_spider/frontera/__init__.py rename to tests/scrapy_spider/frontera/__init__.py diff --git a/frontera/tests/scrapy_spider/frontera/settings.py b/tests/scrapy_spider/frontera/settings.py similarity index 100% rename from frontera/tests/scrapy_spider/frontera/settings.py rename to tests/scrapy_spider/frontera/settings.py diff --git a/frontera/tests/scrapy_spider/settings.py b/tests/scrapy_spider/settings.py similarity index 85% rename from frontera/tests/scrapy_spider/settings.py rename to tests/scrapy_spider/settings.py index aef597eb1..c5df01097 100644 --- a/frontera/tests/scrapy_spider/settings.py +++ b/tests/scrapy_spider/settings.py @@ -3,8 +3,8 @@ #-------------------------------------------------------------------------- BOT_NAME = 'scrapy_spider' -SPIDER_MODULES = ['frontera.tests.scrapy_spider.spiders'] -NEWSPIDER_MODULE = 'frontera.tests.scrapy_spider.spiders' +SPIDER_MODULES = ['tests.scrapy_spider.spiders'] +NEWSPIDER_MODULE = 'tests.scrapy_spider.spiders' HTTPCACHE_ENABLED = False REDIRECT_ENABLED = True @@ -30,7 +30,7 @@ {'frontera.contrib.scrapy.middlewares.schedulers.SchedulerDownloaderMiddleware': 999} ) SCHEDULER = 'frontera.contrib.scrapy.schedulers.frontier.FronteraScheduler' -FRONTERA_SETTINGS = 'frontera.tests.scrapy_spider.frontera.settings' +FRONTERA_SETTINGS = 'tests.scrapy_spider.frontera.settings' #-------------------------------------------------------------------------- diff --git a/frontera/tests/scrapy_spider/spiders/__init__.py b/tests/scrapy_spider/spiders/__init__.py similarity index 100% rename from frontera/tests/scrapy_spider/spiders/__init__.py rename to tests/scrapy_spider/spiders/__init__.py diff --git a/frontera/tests/scrapy_spider/spiders/example.py b/tests/scrapy_spider/spiders/example.py similarity index 75% rename from frontera/tests/scrapy_spider/spiders/example.py rename to tests/scrapy_spider/spiders/example.py index c09ef6a78..d33190e69 100644 --- a/frontera/tests/scrapy_spider/spiders/example.py +++ b/tests/scrapy_spider/spiders/example.py @@ -1,6 +1,6 @@ -from scrapy.contrib.linkextractors.lxmlhtml import LxmlLinkExtractor -from scrapy.contrib.linkextractors.regex import RegexLinkExtractor -from scrapy.contrib.spiders import CrawlSpider, Rule +from scrapy.linkextractors import LinkExtractor +from scrapy.linkextractors.regex import RegexLinkExtractor +from scrapy.spiders import CrawlSpider, Rule class FallbackLinkExtractor(object): @@ -19,7 +19,7 @@ class MySpider(CrawlSpider): callback_calls = 0 rules = [Rule(FallbackLinkExtractor([ - LxmlLinkExtractor(), + LinkExtractor(), RegexLinkExtractor(), ]), callback='parse_page', follow=True)] diff --git a/frontera/tests/test_canonical_solver.py b/tests/test_canonical_solver.py similarity index 100% rename from frontera/tests/test_canonical_solver.py rename to tests/test_canonical_solver.py diff --git a/frontera/tests/test_codecs.py b/tests/test_codecs.py similarity index 100% rename from frontera/tests/test_codecs.py rename to tests/test_codecs.py diff --git a/frontera/tests/test_core_overused_buffer.py b/tests/test_core_overused_buffer.py similarity index 100% rename from frontera/tests/test_core_overused_buffer.py rename to tests/test_core_overused_buffer.py diff --git a/frontera/tests/test_domain_mware.py b/tests/test_domain_mware.py similarity index 100% rename from frontera/tests/test_domain_mware.py rename to tests/test_domain_mware.py diff --git a/frontera/tests/test_fingerprint.py b/tests/test_fingerprint.py similarity index 100% rename from frontera/tests/test_fingerprint.py rename to tests/test_fingerprint.py diff --git a/frontera/tests/test_frontera_scheduler.py b/tests/test_frontera_scheduler.py similarity index 98% rename from frontera/tests/test_frontera_scheduler.py rename to tests/test_frontera_scheduler.py index 07003938d..8a4e21958 100644 --- a/frontera/tests/test_frontera_scheduler.py +++ b/tests/test_frontera_scheduler.py @@ -1,6 +1,6 @@ from frontera.contrib.scrapy.schedulers.frontier import FronteraScheduler -from frontera.tests.mocks.frontier_manager import FakeFrontierManager -from frontera.tests.mocks.crawler import FakeCrawler +from tests.mocks.frontier_manager import FakeFrontierManager +from tests.mocks.crawler import FakeCrawler from frontera.core.models import Request as FRequest from frontera.core.models import Response as FResponse from scrapy.http import Request, Response diff --git a/frontera/tests/test_frontier_manager.py b/tests/test_frontier_manager.py similarity index 81% rename from frontera/tests/test_frontier_manager.py rename to tests/test_frontier_manager.py index bb9027aba..256cd4007 100644 --- a/frontera/tests/test_frontier_manager.py +++ b/tests/test_frontier_manager.py @@ -12,12 +12,12 @@ class TestFrontierManager(object): def setup_frontier_manager(self, settings=None): settings = settings or Settings() - settings.BACKEND = 'frontera.tests.mocks.components.FakeBackend' - settings.MIDDLEWARES = ['frontera.tests.mocks.components.FakeMiddleware', - 'frontera.tests.mocks.components.FakeMiddlewareModifySeeds', - 'frontera.tests.mocks.components.FakeMiddlewareModifyResponse', - 'frontera.tests.mocks.components.FakeMiddlewareModifyLinks'] - settings.CANONICAL_SOLVER = 'frontera.tests.mocks.components.FakeCanonicalSolver' + settings.BACKEND = 'tests.mocks.components.FakeBackend' + settings.MIDDLEWARES = ['tests.mocks.components.FakeMiddleware', + 'tests.mocks.components.FakeMiddlewareModifySeeds', + 'tests.mocks.components.FakeMiddlewareModifyResponse', + 'tests.mocks.components.FakeMiddlewareModifyLinks'] + settings.CANONICAL_SOLVER = 'tests.mocks.components.FakeCanonicalSolver' return FrontierManager.from_settings(settings) def test_start(self): @@ -98,13 +98,13 @@ def test_max_requests_reached(self): def test_blocking_middleware(self): settings = Settings() - settings.BACKEND = 'frontera.tests.mocks.components.FakeBackend' - settings.MIDDLEWARES = ['frontera.tests.mocks.components.FakeMiddleware', - 'frontera.tests.mocks.components.FakeMiddlewareModifySeeds', - 'frontera.tests.mocks.components.FakeMiddlewareBlocking', - 'frontera.tests.mocks.components.FakeMiddlewareModifyResponse', - 'frontera.tests.mocks.components.FakeMiddlewareModifyLinks'] - settings.CANONICAL_SOLVER = 'frontera.tests.mocks.components.FakeCanonicalSolver' + settings.BACKEND = 'tests.mocks.components.FakeBackend' + settings.MIDDLEWARES = ['tests.mocks.components.FakeMiddleware', + 'tests.mocks.components.FakeMiddlewareModifySeeds', + 'tests.mocks.components.FakeMiddlewareBlocking', + 'tests.mocks.components.FakeMiddlewareModifyResponse', + 'tests.mocks.components.FakeMiddlewareModifyLinks'] + settings.CANONICAL_SOLVER = 'tests.mocks.components.FakeCanonicalSolver' fm = FrontierManager.from_settings(settings) fm.add_seeds([r1, r2, r3]) response = Response(r1.url, request=r1) diff --git a/frontera/tests/test_kafka_import.py b/tests/test_kafka_import.py similarity index 100% rename from frontera/tests/test_kafka_import.py rename to tests/test_kafka_import.py diff --git a/frontera/tests/test_message_bus.py b/tests/test_message_bus.py similarity index 100% rename from frontera/tests/test_message_bus.py rename to tests/test_message_bus.py diff --git a/frontera/tests/test_overused_buffer.py b/tests/test_overused_buffer.py similarity index 96% rename from frontera/tests/test_overused_buffer.py rename to tests/test_overused_buffer.py index c0d1f9b75..a9c336037 100644 --- a/frontera/tests/test_overused_buffer.py +++ b/tests/test_overused_buffer.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from frontera.tests.backends import BackendSequenceTest, TEST_SITES +from tests.backends import BackendSequenceTest, TEST_SITES from frontera.utils.tester import DownloaderSimulator, BaseDownloaderSimulator from urlparse import urlparse diff --git a/frontera/tests/test_partitioners.py b/tests/test_partitioners.py similarity index 100% rename from frontera/tests/test_partitioners.py rename to tests/test_partitioners.py diff --git a/frontera/tests/test_revisiting_backend.py b/tests/test_revisiting_backend.py similarity index 96% rename from frontera/tests/test_revisiting_backend.py rename to tests/test_revisiting_backend.py index 234b07da3..3ae85f0a8 100644 --- a/frontera/tests/test_revisiting_backend.py +++ b/tests/test_revisiting_backend.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from frontera.tests.backends import BackendSequenceTest, TEST_SITES +from tests.backends import BackendSequenceTest, TEST_SITES from frontera.utils.tester import FrontierTester from datetime import timedelta diff --git a/frontera/tests/test_scrapy.py b/tests/test_scrapy.py similarity index 100% rename from frontera/tests/test_scrapy.py rename to tests/test_scrapy.py diff --git a/frontera/tests/test_scrapy_settings_adapter.py b/tests/test_scrapy_settings_adapter.py similarity index 83% rename from frontera/tests/test_scrapy_settings_adapter.py rename to tests/test_scrapy_settings_adapter.py index 12b10a32b..46e0c7e35 100644 --- a/frontera/tests/test_scrapy_settings_adapter.py +++ b/tests/test_scrapy_settings_adapter.py @@ -10,6 +10,6 @@ def test_fallsback_to_crawler_settings(): def test_frontera_settings_have_precedence_over_crawler_settings(): crawler_settings = {'MAX_REQUESTS': 10, - 'FRONTERA_SETTINGS': 'frontera.tests.scrapy_spider.frontera.settings'} + 'FRONTERA_SETTINGS': 'tests.scrapy_spider.frontera.settings'} settings = ScrapySettingsAdapter(crawler_settings) assert settings.get('MAX_REQUESTS') == 5 diff --git a/frontera/tests/test_scrapy_spider.py b/tests/test_scrapy_spider.py similarity index 77% rename from frontera/tests/test_scrapy_spider.py rename to tests/test_scrapy_spider.py index a9b235880..98e8dd171 100644 --- a/frontera/tests/test_scrapy_spider.py +++ b/tests/test_scrapy_spider.py @@ -1,14 +1,14 @@ # -*- coding: utf-8 -*- from twisted.internet import reactor from scrapy.crawler import Crawler -from scrapy import log, signals +from scrapy import signals from scrapy.settings import Settings -from scrapy_spider.spiders.example import MySpider +from tests.scrapy_spider.spiders.example import MySpider def test_scrapy_spider(): settings = Settings() - settings.setmodule("frontera.tests.scrapy_spider.settings") + settings.setmodule("tests.scrapy_spider.settings") crawler = Crawler(MySpider, settings=settings) crawler.signals.connect(reactor.stop, signal=signals.spider_closed) crawler.crawl() diff --git a/frontera/tests/test_settings.py b/tests/test_settings.py similarity index 89% rename from frontera/tests/test_settings.py rename to tests/test_settings.py index fcd0712e7..939e18bd0 100644 --- a/frontera/tests/test_settings.py +++ b/tests/test_settings.py @@ -4,7 +4,7 @@ def test_settings_on_a_python_module_are_loaded(): - settings = Settings('frontera.tests.scrapy_spider.frontera.settings') + settings = Settings('tests.scrapy_spider.frontera.settings') assert settings.get('MAX_REQUESTS') == 5 @@ -30,7 +30,7 @@ def test_settings_attributes_can_be_assigned(): def test_object_from_loads_settings_from_a_module(): - module = 'frontera.tests.scrapy_spider.frontera.settings' + module = 'tests.scrapy_spider.frontera.settings' settings = BaseSettings.object_from(module) assert settings.get('MAX_REQUESTS') == 5 diff --git a/frontera/tests/test_utils_async.py b/tests/test_utils_async.py similarity index 100% rename from frontera/tests/test_utils_async.py rename to tests/test_utils_async.py diff --git a/frontera/tests/test_utils_heap.py b/tests/test_utils_heap.py similarity index 100% rename from frontera/tests/test_utils_heap.py rename to tests/test_utils_heap.py diff --git a/frontera/tests/test_utils_misc.py b/tests/test_utils_misc.py similarity index 77% rename from frontera/tests/test_utils_misc.py rename to tests/test_utils_misc.py index 51d9b5ff0..de79dafd2 100644 --- a/frontera/tests/test_utils_misc.py +++ b/tests/test_utils_misc.py @@ -29,19 +29,19 @@ def test_non_multiple_length(self): class TestLoadObject(object): def test_load_class(self): - obj = load_object('frontera.tests.mocks.load_objects.MockClass') + obj = load_object('tests.mocks.load_objects.MockClass') assert obj.val == 10 def test_load_instance(self): - obj = load_object('frontera.tests.mocks.load_objects.mock_instance') + obj = load_object('tests.mocks.load_objects.mock_instance') assert obj.val == 5 def test_load_variable(self): - obj = load_object('frontera.tests.mocks.load_objects.mock_variable') + obj = load_object('tests.mocks.load_objects.mock_variable') assert obj == 'test' def test_load_function(self): - obj = load_object('frontera.tests.mocks.load_objects.mock_function') + obj = load_object('tests.mocks.load_objects.mock_function') assert obj() == 2 def test_value_error(self): @@ -57,6 +57,6 @@ def test_import_error(self): def test_name_error(self): with pytest.raises(NameError) as info: - load_object('frontera.tests.mocks.load_objects.non_existent_object') - assert info.value.message == ("Module 'frontera.tests.mocks.load_objects' doesn't define" + load_object('tests.mocks.load_objects.non_existent_object') + assert info.value.message == ("Module 'tests.mocks.load_objects' doesn't define" " any object named 'non_existent_object'") diff --git a/frontera/tests/test_utils_url.py b/tests/test_utils_url.py similarity index 100% rename from frontera/tests/test_utils_url.py rename to tests/test_utils_url.py diff --git a/tox.ini b/tox.ini index bd06a1693..06723dd19 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,7 @@ deps = -r{toxinidir}/requirements.txt -r{toxinidir}/requirements/tests.txt commands = - py.test -s -v {envsitepackagesdir}/frontera + py.test --cov-report=term --cov=frontera -s -v tests [testenv:flake8] changedir = {toxinidir}