Fixes for PY2 and older ckan

.github/workflows/test.yml

@@ -19,7 +19,7 @@ jobs:
     needs: lint
     strategy:
       matrix:
-        ckan-version: ["2.10", 2.9, 2.9-py2, 2.8, 2.7]
+        ckan-version: [2.9, 2.9-py2, 2.8, 2.7]
       fail-fast: false
 
     name: CKAN ${{ matrix.ckan-version }}
@@ -49,6 +49,7 @@ jobs:
       - uses: actions/checkout@v3
       - name: Install requirements
         run: |
+          apk add file
           pip install -r requirements.txt
           pip install -r dev-requirements.txt
           pip install -e .

ckanext/qa/commands.py

@@ -1,7 +1,7 @@
 import logging
 import sys
 import ckan.plugins as p
-import ckanext.qa.utils as utils
+from ckanext.qa.utils import init_db, update, sniff, view, clean, migrate1
 
 REQUESTS_HEADER = {'content-type': 'application/json',
                    'User-Agent': 'ckanext-qa commands'}
@@ -92,24 +92,24 @@ def command(self):
             self.log.error('Command "%s" not recognized' % (cmd,))
 
     def init_db(self):
-        utils.init_db()
+        init_db()
 
     def update(self):
         if len(self.args) > 1:
             ids = self.args[1:]
-        utils.update(ids, self.options.queue)
+        update(ids, self.options.queue)
 
     def sniff(self):
         if len(self.args) < 2:
             print('Not enough arguments', self.args)
             sys.exit(1)
-        utils.sniff(self.args[1:])
+        sniff(self.args[1:])
 
     def view(self, package_ref=None):
-        utils.view(package_ref)
+        view(package_ref)
 
     def clean(self):
-        utils.clean()
+        clean()
 
     def migrate1(self):
-        utils.migrate1()
+        migrate1()

ckanext/qa/lib.py

@@ -5,7 +5,7 @@
 from ckan.plugins.toolkit import config
 
 from ckan import plugins as p
-import ckanext.qa.tasks as tasks
+from ckanext.qa.tasks import update_package, update
 
 log = logging.getLogger(__name__)
 
@@ -45,10 +45,9 @@ def resource_format_scores():
     if not _RESOURCE_FORMAT_SCORES:
         _RESOURCE_FORMAT_SCORES = {}
         json_filepath = config.get('qa.resource_format_openness_scores_json')
-        import ckanext.qa.plugin
         if not json_filepath:
             json_filepath = os.path.join(
-                os.path.dirname(os.path.dirname(os.path.realpath(ckanext.qa.plugin.__file__))),
+                os.path.dirname(os.path.realpath(__file__)),
                 'resource_format_openness_scores.json'
             )
         with open(json_filepath) as format_file:
@@ -87,7 +86,7 @@ def munge_format_to_be_canonical(format_name):
 
 def create_qa_update_package_task(package, queue):
 
-    compat_enqueue('qa.update_package', tasks.update_package, queue,  args=[package.id])
+    compat_enqueue('qa.update_package', update_package, queue,  args=[package.id])
     log.debug('QA of package put into celery queue %s: %s',
               queue, package.name)
 
@@ -98,7 +97,7 @@ def create_qa_update_task(resource, queue):
     else:
         package = resource.package
 
-    compat_enqueue('qa.update', tasks.update, queue, args=[resource.id])
+    compat_enqueue('qa.update', update, queue, args=[resource.id])
 
     log.debug('QA of resource put into celery queue %s: %s/%s url=%r',
               queue, package.name, resource.id, resource.url)

ckanext/qa/plugin/__init__.py

@@ -7,8 +7,8 @@
 from ckanext.archiver.interfaces import IPipe
 from ckanext.qa.logic import action, auth
 from ckanext.qa.model import QA, aggregate_qa_for_a_dataset
-import ckanext.qa.helpers as helpers
-import ckanext.qa.lib as lib
+from ckanext.qa.helpers import qa_openness_stars_resource_html, qa_openness_stars_dataset_html
+from ckanext.qa.lib import create_qa_update_package_task
 from ckanext.report.interfaces import IReport
 
 
@@ -47,7 +47,7 @@ def receive_data(self, operation, queue, **params):
         dataset = model.Package.get(dataset_id)
         assert dataset
 
-        lib.create_qa_update_package_task(dataset, queue=queue)
+        create_qa_update_package_task(dataset, queue=queue)
 
     # IReport
 
@@ -77,9 +77,9 @@ def get_auth_functions(self):
     def get_helpers(self):
         return {
             'qa_openness_stars_resource_html':
-            helpers.qa_openness_stars_resource_html,
+            qa_openness_stars_resource_html,
             'qa_openness_stars_dataset_html':
-            helpers.qa_openness_stars_dataset_html,
+            qa_openness_stars_dataset_html,
             }
 
     # IPackageController

ckanext/qa/sniff_format.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-from io import BytesIO
+from io import BytesIO, open
 import sys
 import re
 import zipfile
@@ -12,7 +12,6 @@
 import magic
 import messytables
 
-import ckanext.qa.lib as lib
 from ckan.lib import helpers as ckan_helpers
 
 
@@ -389,6 +388,7 @@ def get_zipped_format(filepath):
     '''For a given zip file, return the format of file inside.
     For multiple files, choose by the most open, and then by the most
     popular extension.'''
+    from ckanext.qa.lib import resource_format_scores
     # just check filename extension of each file inside
     try:
         # note: Cannot use "with" with a zipfile before python 2.7
@@ -427,7 +427,7 @@ def get_zipped_format(filepath):
         extension = os.path.splitext(filepath)[-1][1:].lower()
         format_tuple = ckan_helpers.resource_formats().get(extension)
         if format_tuple:
-            score = lib.resource_format_scores().get(format_tuple[1])
+            score = resource_format_scores().get(format_tuple[1])
             if score is not None and score > top_score:
                 top_score = score
                 top_scoring_extension_counts = defaultdict(int)

ckanext/qa/tasks.py

@@ -13,7 +13,6 @@
 from ckan.plugins import toolkit
 import ckan.lib.helpers as ckan_helpers
 from ckanext.qa.sniff_format import sniff_file_format
-import ckanext.qa.lib as lib
 from ckanext.archiver.model import Archival, Status
 
 import logging
@@ -291,6 +290,7 @@ def score_by_sniffing_data(archival, resource, score_reasons):
       * If it cannot work out the format then format_string is None
       * If it cannot score it, then score is None
     '''
+    from ckanext.qa.lib import resource_format_scores
     if not archival or not archival.cache_filepath:
         score_reasons.append(_('This file had not been downloaded at the time of scoring it.'))
         return (None, None)
@@ -302,7 +302,7 @@ def score_by_sniffing_data(archival, resource, score_reasons):
     else:
         if filepath:
             sniffed_format = sniff_file_format(filepath)
-            score = lib.resource_format_scores().get(sniffed_format['format']) \
+            score = resource_format_scores().get(sniffed_format['format']) \
                 if sniffed_format else None
             if sniffed_format:
                 score_reasons.append(_('Content of file appeared to be format "%s" which receives openness score: %s.')
@@ -340,14 +340,15 @@ def score_by_url_extension(resource, score_reasons):
       * If it cannot work out the format then format is None
       * If it cannot score it, then score is None
     '''
+    from ckanext.qa.lib import resource_format_scores
     extension_variants_ = extension_variants(resource.url.strip())
     if not extension_variants_:
         score_reasons.append(_('Could not determine a file extension in the URL.'))
         return (None, None)
     for extension in extension_variants_:
         format_ = format_get(extension)
         if format_:
-            score = lib.resource_format_scores().get(format_)
+            score = resource_format_scores().get(format_)
             if score:
                 score_reasons.append(
                     _('URL extension "%s" relates to format "%s" and receives score: %s.') % (extension, format_, score))
@@ -393,16 +394,17 @@ def score_by_format_field(resource, score_reasons):
       * If it cannot work out the format then format_string is None
       * If it cannot score it, then score is None
     '''
+    from ckanext.qa.lib import resource_format_scores, munge_format_to_be_canonical
     format_field = resource.format or ''
     if not format_field:
         score_reasons.append(_('Format field is blank.'))
         return (None, None)
     format_tuple = ckan_helpers.resource_formats().get(format_field.lower()) or \
-        ckan_helpers.resource_formats().get(lib.munge_format_to_be_canonical(format_field))
+        ckan_helpers.resource_formats().get(munge_format_to_be_canonical(format_field))
     if not format_tuple:
         score_reasons.append(_('Format field "%s" does not correspond to a known format.') % format_field)
         return (None, None)
-    score = lib.resource_format_scores().get(format_tuple[1])
+    score = resource_format_scores().get(format_tuple[1])
     score_reasons.append(_('Format field "%s" receives score: %s.') %
                          (format_field, score))
     return (score, format_tuple[1])

ckanext/qa/tests/test_link_checker.py

@@ -1,6 +1,5 @@
 import pytest
 import logging
-from functools import wraps
 import json
 from nose.tools import assert_in
 
@@ -11,8 +10,6 @@
 
 from ckanext.archiver.tasks import update_package
 
-from .mock_remote_server import MockEchoTestServer
-
 # enable celery logging for when you run nosetests -s
 log = logging.getLogger('ckanext.archiver.tasks')
 
@@ -24,19 +21,6 @@ def get_logger():
 update_package.get_logger = get_logger
 
 
-def with_mock_url(url=''):
-    """
-    Start a MockEchoTestServer call the decorated function with the server's address prepended to ``url``.
-    """
-    def decorator(func):
-        @wraps(func)
-        def decorated(*args, **kwargs):
-            with MockEchoTestServer().serve() as serveraddr:
-                return func(*(args + ('%s/%s' % (serveraddr, url),)), **kwargs)
-        return decorated
-    return decorator
-
-
 @pytest.mark.usefixtures('with_plugins')
 @pytest.mark.ckan_config('ckan.plugins', 'archiver qa')
 class TestLinkChecker(object):
@@ -90,9 +74,20 @@ def test_format_by_mimetype_csv(self, client, app):
     def test_file_url(self, client, app):
         url = u'file:///home/root/test.txt'
         result = self.check_link(url, None, app)
-        log.debug(result)
-        assert_in(u'Invalid url scheme. Please use one of: http ftp https',
-                  result['url_errors'])
+
+        format_in_use = None
+        # htt/https/ftp comes in random order in url_errors so check if any possible format is used in url_error
+        schemes = [u'http ftp https', u'http https ftp', u'https ftp http', u'https http ftp', u'ftp https http',
+                   u'ftp http https']
+        for scheme in schemes:
+            if u'Invalid url scheme. Please use one of: %s' % scheme in result['url_errors']:
+                format_in_use = u'Invalid url scheme. Please use one of: %s' % scheme
+                break
+
+        if format_in_use:
+            assert_in(format_in_use, result['url_errors'])
+        else:
+            pytest.fail("Link check failed {}".format(result['url_errors']))
 
     def test_empty_url(self, client, app):
         url = u''

ckanext/qa/tests/test_tasks.py

@@ -8,12 +8,7 @@
 from ckan.logic import get_action
 from ckan import plugins as p
 import ckan.lib.helpers as ckan_helpers
-try:
-    from ckan.tests import factories as ckan_factories
-    from ckan.tests.legacy import BaseCase
-except ImportError:
-    from ckan.new_tests import factories as ckan_factories
-    from ckan.tests import BaseCase
+from ckantoolkit.tests import factories as ckan_factories
 
 import ckanext.qa.tasks
 from ckanext.qa.tasks import resource_score, extension_variants
@@ -62,12 +57,12 @@ def set_sniffed_format(format_name):
 TODAY_STR = TODAY.isoformat()
 
 
-class TestTask(BaseCase):
-
-    @classmethod
+@pytest.mark.usefixtures('with_plugins')
+@pytest.mark.ckan_config('ckan.plugins', 'qa archiver report')
+class TestTask():
+    @pytest.fixture(autouse=True)
     @pytest.mark.usefixtures('clean_db')
-    def setup_class(cls):
-        # reset_db()
+    def init_data(cls, clean_db):
         archiver_model.init_tables(model.meta.engine)
         qa_model.init_tables(model.meta.engine)
 
@@ -98,12 +93,12 @@ def test_trigger_on_archival(cls):
         # TODO run celery and check it actually ran...
 
 
-class TestResourceScore(BaseCase):
-
-    @classmethod
+@pytest.mark.usefixtures('with_plugins')
+@pytest.mark.ckan_config('ckan.plugins', 'qa archiver report')
+class TestResourceScore():
+    @pytest.fixture(autouse=True)
     @pytest.mark.usefixtures('clean_db')
-    def setup_class(cls):
-        # reset_db()
+    def init_data(cls, clean_db):
         archiver_model.init_tables(model.meta.engine)
         qa_model.init_tables(model.meta.engine)
         cls.fake_resource = {
@@ -271,7 +266,7 @@ def test_not_available_any_more(self):
                                                    ' Attempted on 10/10/2008. This URL last worked on: 01/10/2008.')
 
 
-class TestExtensionVariants:
+class TestExtensionVariants():
     def test_0_normal(self):
         assert extension_variants('http://dept.gov.uk/coins-data-1996.csv') == ['csv']
 
@@ -285,11 +280,12 @@ def test_3_none(self):
         assert extension_variants('http://dept.gov.uk/coins-data-1996') == []
 
 
-class TestSaveQaResult(object):
-    @classmethod
+@pytest.mark.usefixtures('with_plugins')
+@pytest.mark.ckan_config('ckan.plugins', 'qa archiver report')
+class TestSaveQaResult():
+    @pytest.fixture(autouse=True)
     @pytest.mark.usefixtures('clean_db')
-    def setup_class(cls):
-        # reset_db()
+    def init_data(cls, clean_db):
         archiver_model.init_tables(model.meta.engine)
         qa_model.init_tables(model.meta.engine)
 
@@ -318,11 +314,12 @@ def test_simple(self):
         assert qa.updated == qa.updated
 
 
-class TestUpdatePackage(object):
-    @classmethod
+@pytest.mark.usefixtures('with_plugins')
+@pytest.mark.ckan_config('ckan.plugins', 'qa archiver report')
+class TestUpdatePackage():
+    @pytest.fixture(autouse=True)
     @pytest.mark.usefixtures('clean_db')
-    def setup_class(cls):
-        # reset_db()
+    def init_data(cls, clean_db):
         archiver_model.init_tables(model.meta.engine)
         qa_model.init_tables(model.meta.engine)
 
@@ -343,11 +340,12 @@ def test_simple(self):
         assert qa.openness_score_reason == 'License not open'
 
 
-class TestUpdateResource(object):
-    @classmethod
+@pytest.mark.usefixtures('with_plugins')
+@pytest.mark.ckan_config('ckan.plugins', 'qa archiver report')
+class TestUpdateResource():
+    @pytest.fixture(autouse=True)
     @pytest.mark.usefixtures('clean_db')
-    def setup_class(cls):
-        # reset_db()
+    def init_data(cls, clean_db):
         archiver_model.init_tables(model.meta.engine)
         qa_model.init_tables(model.meta.engine)