Merge pull request #3 from vrk-kpa/AV-1703_ckan-2.9-support

Av 1703 ckan 2.9 support

.github/workflows/test.yml

@@ -0,0 +1,69 @@
+name: Tests
+on: [push, pull_request]
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.6'
+      - name: Install requirements
+        run: pip install flake8 pycodestyle
+      - name: Check syntax
+        run: flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan
+      - name: Run flake8
+        run: flake8 . --count --max-line-length=127 --statistics --exclude ckan
+
+  test:
+    needs: lint
+    strategy:
+      matrix:
+        ckan-version: [2.9, 2.9-py2, 2.8, 2.7]
+      fail-fast: false
+
+    name: CKAN ${{ matrix.ckan-version }}
+    runs-on: ubuntu-latest
+    container:
+      image: openknowledge/ckan-dev:${{ matrix.ckan-version }}
+    services:
+      solr:
+        image: ckan/ckan-solr:${{ matrix.ckan-version }}
+      postgres:
+        image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }}
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+      redis:
+        image: redis:3
+    env:
+      CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test
+      CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test
+      CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test
+      CKAN_SOLR_URL: http://solr:8983/solr/ckan
+      CKAN_REDIS_URL: redis://redis:6379/1
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install requirements
+        run: |
+          apk add file
+          pip install -r requirements.txt
+          pip install -r dev-requirements.txt
+          pip install -e .
+          # Replace default path to CKAN core config file with the one on the container
+          sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini
+      - name: Setup extension (CKAN >= 2.9)
+        if: ${{ matrix.ckan-version != '2.7' && matrix.ckan-version != '2.8' }}
+        run: |
+          ckan -c test.ini db init
+          ckan -c test.ini qa init
+      - name: Setup extension (CKAN < 2.9)
+        if: ${{ matrix.ckan-version == '2.7' || matrix.ckan-version == '2.8' }}
+        run: |
+          paster --plugin=ckan db init -c test.ini
+          paster --plugin=ckanext-qa qa init -c test.ini
+      - name: Run tests
+        run: pytest --ckan-ini=test.ini --cov=ckanext.qa --disable-warnings ckanext/qa/tests

README.rst

@@ -2,8 +2,8 @@
    these badges work. The necessary Travis and Coverage config files have been
    generated for you.
 
-.. image:: https://travis-ci.org/ckan/ckanext-qa.svg?branch=master
-    :target: https://travis-ci.org/ckan/ckanext-qa
+.. image:: https://github.com/ckan/ckanext-qa/actions/workflows/test.yml/badge.svg
+    :target: https://github.com/ckan/ckanext-qa/actions/workflows/test.yml
 
 CKAN QA Extension (Quality Assurance)
 =====================================
@@ -31,7 +31,7 @@ Requirements
 
 Before installing ckanext-qa, make sure that you have installed the following:
 
-* CKAN 2.1+
+* CKAN 2.1+ (tests are only running for CKAN 2.7+)
 * ckanext-archiver 2.0+ (https://github.com/ckan/ckanext-archiver)
 * ckanext-report (https://github.com/datagovuk/ckanext-report) for reporting
 

ckanext/qa/bin/common.py

@@ -1,29 +1,3 @@
-import os
-
-
-def load_config(config_filepath):
-    import paste.deploy
-    config_abs_path = os.path.abspath(config_filepath)
-    conf = paste.deploy.appconfig('config:' + config_abs_path)
-    import ckan
-    ckan.config.environment.load_environment(conf.global_conf,
-                                             conf.local_conf)
-
-
-def register_translator():
-    # Register a translator in this thread so that
-    # the _() functions in logic layer can work
-    from paste.registry import Registry
-    from pylons import translator
-    from ckan.lib.cli import MockTranslator
-    global registry
-    registry = Registry()
-    registry.prepare()
-    global translator_obj
-    translator_obj = MockTranslator()
-    registry.register(translator, translator_obj)
-
-
 def get_resources(state='active', publisher_ref=None, resource_id=None, dataset_name=None):
     ''' Returns all active resources, or filtered by the given criteria. '''
     from ckan import model
@@ -48,5 +22,5 @@ def get_resources(state='active', publisher_ref=None, resource_id=None, dataset_
         resources = resources.filter(model.Resource.id == resource_id)
         criteria.append('Resource:%s' % resource_id)
     resources = resources.all()
-    print '%i resources (%s)' % (len(resources), ' '.join(criteria))
+    print('%i resources (%s)' % (len(resources), ' '.join(criteria)))
     return resources

ckanext/qa/bin/migrate_task_status.py

@@ -59,7 +59,7 @@ def migrate(options):
         # time, so some timezone nonesense going on. Can't do much.
         archival = Archival.get_for_resource(res.id)
         if not archival:
-            print add_stat('QA but no Archival data', res, stats)
+            print(add_stat('QA but no Archival data', res, stats))
             continue
         archival_date = archival.updated
         # the state of the resource was as it was archived on the date of
@@ -112,10 +112,10 @@ def migrate(options):
                 model.Session.add(qa)
             add_stat('Added to QA table', res, stats)
 
-    print 'Summary\n', stats.report()
+    print('Summary\n', stats.report())
     if options.write:
         model.repo.commit_and_remove()
-        print 'Written'
+        print('Written')
 
 
 def add_stat(outcome, res, stats, extra_info=None):
@@ -154,10 +154,7 @@ def date_str_to_datetime_or_none(date_str):
     if len(args) != 1:
         parser.error('Wrong number of arguments (%i)' % len(args))
     config_ini = args[0]
-    print 'Loading CKAN config...'
-    common.load_config(config_ini)
-    common.register_translator()
-    print 'Done'
+
     # Setup logging to print debug out for local only
     rootLogger = logging.getLogger()
     rootLogger.setLevel(logging.WARNING)

ckanext/qa/bin/running_stats.py

@@ -110,6 +110,6 @@ def report_value(self, category):
     package_stats.add('Success', 'good3')
     package_stats.add('Success', 'good4')
     package_stats.add('Failure', 'bad1')
-    print package_stats.report()
+    print(package_stats.report())
 
-    print StatsList().report()
+    print(StatsList().report())

ckanext/qa/cli.py

@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+
+import sys
+import click
+import ckanext.qa.utils as utils
+
+
+def get_commands():
+    return [qa]
+
+
+@click.group()
+def qa():
+    """
+    QA analysis of CKAN resources
+
+    Usage::
+
+        ckan -c <path to CKAN config file> qa init
+           - Creates the database tables that QA expects for storing
+           results
+
+        ckan -c <path to CKAN config file> qa [options] update [dataset/group name/id]
+           - QA analysis on all resources in a given dataset, or on all
+           datasets if no dataset given
+
+        ckan -c <path to CKAN config file> qa sniff {filepath}
+           - Opens the file and determines its type by the contents
+
+        ckan -c <path to CKAN config file> qa view [dataset name/id]
+           - See package score information
+
+        ckan -c <path to CKAN config file> qa clean
+           - Remove all package score information
+
+        ckan -c <path to CKAN config file> qa migrate1
+           - Migrates the way results are stored in task_status,
+             with commit 6f63ab9e 20th March 2013
+             (from key='openness_score'/'openness_score_failure_count' to
+              key='status')
+
+    The commands should be run from the ckanext-qa directory and expect
+    a development.ini file to be present. Most of the time you will
+    specify the config explicitly though::
+
+        ckan -c <path to CKAN config file> qa update
+    """
+
+
+@qa.command()
+def init():
+    utils.init_db()
+
+
+@qa.command()
+@click.argument('ids', nargs=-1)
+@click.option('-q', '--queue', help='Send to a particular queue')
+def update(ids, queue):
+    utils.update(ids, queue)
+
+
+@qa.command()
+@click.argument('filepaths', nargs=-1)
+def sniff(filepaths):
+    if len(filepaths) < 1:
+        print('Not enough arguments', filepaths)
+        sys.exit(1)
+
+    utils.sniff(filepaths)
+
+
+@qa.command()
+@click.argument('package_ref')
+def view(package_ref=None):
+    utils.view(package_ref)
+
+
+@qa.command()
+def clean():
+    utils.clean()
+
+
+@qa.command()
+def migrate1():
+    utils.migrate1()