diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 00000000..3138593c --- /dev/null +++ b/.coveragerc @@ -0,0 +1,5 @@ +[report] +omit = + */site-packages/* + */python?.?/* + ckan/* \ No newline at end of file diff --git a/.gitignore b/.gitignore index da03614f..52950fc8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,52 @@ -*.pyc -*.egg-info -build -*.egg -.DS_Store -*.swp +# archiver settings should not be checked in - only its template ckanext/archiver/settings.py -*~ \ No newline at end of file + +.DS_Store +# vim +*.sw? +# emacs +*~ + +.ropeproject +node_modules +bower_components + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +sdist/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.cache +nosetests.xml +coverage.xml + +# Sphinx documentation +docs/_build/ + diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..4f3ed4eb --- /dev/null +++ b/.travis.yml @@ -0,0 +1,10 @@ +language: python +python: + - "2.7" +env: PGVERSION=9.1 +install: + - bash bin/travis-build.bash + - pip install coveralls +script: sh bin/travis-run.sh +after_success: + - coveralls diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..e258ae36 --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2015 Open Knowledge & Crown Copyright + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..22e204fc --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include README.rst +recursive-include ckanext/archiver *.html *.json *.js *.less *.css \ No newline at end of file diff --git a/README.rst b/README.rst index 22865614..8a5df99f 100644 --- a/README.rst +++ b/README.rst @@ -1,10 +1,25 @@ -CKAN Archiver Extension -======================= +.. You should enable this project on travis-ci.org and coveralls.io to make + these badges work. The necessary Travis and Coverage config files have been + generated for you. -**Status:** Production +.. image:: https://travis-ci.org/datagovuk/ckanext-archiver.svg?branch=master + :target: https://travis-ci.org/datagovuk/ckanext-archiver -**CKAN Version:** release-1.7.1-dgu +.. image:: https://pypip.in/py_versions/ckanext-archiver/badge.svg + :target: https://pypi.python.org/pypi/ckanext-archiver/ + :alt: Supported Python versions +.. image:: https://pypip.in/status/ckanext-archiver/badge.svg + :target: https://pypi.python.org/pypi/ckanext-archiver/ + :alt: Development Status + +.. image:: https://pypip.in/license/ckanext-archiver/badge.svg + :target: https://pypi.python.org/pypi/ckanext-archiver/ + :alt: License + +============= +ckanext-archiver +============= Overview -------- @@ -25,21 +40,32 @@ This means that the 'bulk' queue can happily run slowly, chugging through the do Installation ------------ -Install the extension source as usual, e.g. (from an activated virtualenv):: +To install ckanext-archiver: + +1. Activate your CKAN virtual environment, for example:: + + . /usr/lib/ckan/default/bin/activate - $ pip install -e git+http://github.com/datagovuk/ckanext-archiver.git#egg=ckanext-archiver +2. Install the ckanext-archiver Python package into your virtual environment:: -Or (primarily for developers) download the source, then from the ckanext-archiver directory run:: + pip install -e git+http://github.com/datagovuk/ckanext-archiver.git#egg=ckanext-archiver - $ pip install -e ./ +3. Now create the database tables:: -Whichever way you install the source, you now must create the database tables:: + paster --plugin=ckanext-archiver archiver init --config=production.ini - $ paster --plugin=ckanext-archiver archiver init --config=ckan.ini +4. Add ``archiver`` to the ``ckan.plugins`` setting in your CKAN + config file (by default the config file is located at + ``/etc/ckan/default/production.ini``). +5. Restart CKAN. For example if you've deployed CKAN with Apache on Ubuntu:: -Configuration -------------- + sudo service apache2 reload + + + +Config settings +--------------- 1. Enabling Archiver to listen to resource changes @@ -131,8 +157,6 @@ The Archiver can be used in two ways: Testing ------- -Tests should be run from the CKAN root directory (not the extension root). - -:: +To run the tests, from the CKAN root directory (not the extension root) do:: - (pyenv)~/pyenv/src/ckan$ nosetests --ckan ../ckanext-archiver/tests/ --with-pylons=../ckanext-archiver/test.ini + (pyenv)~/pyenv/src/ckan$ nosetests --ckan ../ckanext-archiver/tests/ --with-pylons=../ckanext-archiver/test-core.ini diff --git a/bin/travis-build.bash b/bin/travis-build.bash new file mode 100644 index 00000000..e7110dd8 --- /dev/null +++ b/bin/travis-build.bash @@ -0,0 +1,38 @@ +#!/bin/bash +set -e + +echo "This is travis-build.bash..." + +echo "Installing the packages that CKAN requires..." +sudo apt-get update -qq +sudo apt-get install postgresql-$PGVERSION solr-jetty libcommons-fileupload-java:amd64=1.2.2-1 + +echo "Installing CKAN and its Python dependencies..." +git clone https://github.com/ckan/ckan +cd ckan +export latest_ckan_release_branch=`git branch --all | grep remotes/origin/release-v | sort -r | sed 's/remotes\/origin\///g' | head -n 1` +echo "CKAN branch: $latest_ckan_release_branch" +git checkout $latest_ckan_release_branch +python setup.py develop +pip install -r requirements.txt --allow-all-external +pip install -r dev-requirements.txt --allow-all-external +cd - + +echo "Creating the PostgreSQL user and database..." +sudo -u postgres psql -c "CREATE USER ckan_default WITH PASSWORD 'pass';" +sudo -u postgres psql -c 'CREATE DATABASE ckan_test WITH OWNER ckan_default;' + +echo "Initialising the database..." +cd ckan +paster db init -c test-core.ini +cd - + +echo "Installing ckanext-ckanext-archiver and its requirements..." +python setup.py develop +pip install -r dev-requirements.txt + +echo "Moving test.ini into a subdir..." +mkdir subdir +mv test.ini subdir + +echo "travis-build.bash is done." \ No newline at end of file diff --git a/bin/travis-run.sh b/bin/travis-run.sh new file mode 100644 index 00000000..2db8e4b5 --- /dev/null +++ b/bin/travis-run.sh @@ -0,0 +1,6 @@ +#!/bin/sh -e + +echo "NO_START=0\nJETTY_HOST=127.0.0.1\nJETTY_PORT=8983\nJAVA_HOME=$JAVA_HOME" | sudo tee /etc/default/jetty +sudo cp ckan/ckan/config/solr/schema.xml /etc/solr/conf/schema.xml +sudo service jetty restart +nosetests --nologcapture --with-pylons=subdir/test.ini --with-coverage --cover-package=ckanext.archiver --cover-inclusive --cover-erase --cover-tests \ No newline at end of file diff --git a/ckanext/archiver/tasks.py b/ckanext/archiver/tasks.py index 07dbaa09..b02e4240 100644 --- a/ckanext/archiver/tasks.py +++ b/ckanext/archiver/tasks.py @@ -211,6 +211,8 @@ def _save(status_id, exception, resource, url_redirected_to=None, _save(Status.by_text('Archived successfully'), '', resource, download_result['url_redirected_to'], download_result, archive_result) # The return value is only used by tests. Serialized for Celery. + print download_result + print archive_result return json.dumps(dict(download_result, **archive_result)) diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..f64e6f6e --- /dev/null +++ b/setup.cfg @@ -0,0 +1,21 @@ +[extract_messages] +keywords = translate isPlural +add_comments = TRANSLATORS: +output_file = i18n/ckanext-archiver.pot +width = 80 + +[init_catalog] +domain = ckanext-archiver +input_file = i18n/ckanext-archiver.pot +output_dir = i18n + +[update_catalog] +domain = ckanext-archiver +input_file = i18n/ckanext-archiver.pot +output_dir = i18n +previous = true + +[compile_catalog] +domain = ckanext-archiver +directory = i18n +statistics = true \ No newline at end of file diff --git a/setup.py b/setup.py index abdae46b..04c5051c 100644 --- a/setup.py +++ b/setup.py @@ -1,43 +1,117 @@ -from setuptools import setup, find_packages +from setuptools import setup, find_packages # Always prefer setuptools over distutils +from codecs import open # To use a consistent encoding +from os import path + +here = path.abspath(path.dirname(__file__)) + +# Get the long description from the relevant file +with open(path.join(here, 'README.rst'), encoding='utf-8') as f: + long_description = f.read() setup( - name='ckanext-archiver', - version='0.1', - packages=find_packages(), - install_requires=[ - 'celery==2.4.2', - 'kombu==2.1.3', - 'kombu-sqlalchemy==1.1.0', - 'SQLAlchemy>=0.6.6', - 'requests==1.1.0', - 'flask==0.8' # flask needed for tests - ], - author='Open Knowledge Foundation', + name='''ckanext-archiver''', + + # Versions should comply with PEP440. For a discussion on single-sourcing + # the version across setup.py and the project code, see + # http://packaging.python.org/en/latest/tutorial.html#version + version='0.2.0', + + description='''Archives resources in CKAN (CKAN Extension)''', + long_description=long_description, + + # The project's main homepage. + url='https://github.com/datagovuk/ckanext-archiver', + + # Author details + author='''Open Knowledge / Cabinet Office''', author_email='info@okfn.org', - description='Archive ckan resources', - long_description='Archive ckan resources', + + # Choose your license license='MIT', - url='http://ckan.org/wiki/Extensions', - download_url='', - include_package_data=True, + + # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ - 'Development Status :: 3 - Alpha', + # How mature is this project? Common values are + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + 'Development Status :: 4 - Beta', + + # Pick your license as you wish (should match "license" above) + 'License :: OSI Approved :: MIT License', + + # Specify the Python versions you support here. In particular, ensure + # that you indicate whether you support Python 2, Python 3 or both. + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Environment :: Console', 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', 'Operating System :: OS Independent', 'Programming Language :: Python', 'Topic :: Software Development :: Libraries :: Python Modules' ], + + # What does your project relate to? + keywords='''CKAN''', + + # You can just specify the packages manually here if your project is + # simple. Or you can use find_packages(). + packages=find_packages(exclude=['contrib', 'docs', 'tests*']), + + # List run-time dependencies here. These will be installed by pip when your + # project is installed. For an analysis of "install_requires" vs pip's + # requirements files see: + # https://packaging.python.org/en/latest/technical.html#install-requires-vs-requirements-files + install_requires=[ + 'celery==2.4.2', + 'kombu==2.1.3', + 'kombu-sqlalchemy==1.1.0', + 'SQLAlchemy>=0.6.6', + 'requests==1.1.0', + 'flask==0.8' # flask needed for tests + ], + + # If there are data files included in your packages that need to be + # installed, specify them here. If using Python 2.6 or less, then these + # have to be included in MANIFEST.in as well. + include_package_data=True, + package_data={ + }, + + # Although 'package_data' is the preferred approach, in some case you may + # need to place data files outside of your packages. + # see http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files + # In this case, 'data_file' will be installed into '/my_data' + data_files=[], + + # To provide executable scripts, use entry points in preference to the + # "scripts" keyword. Entry points provide cross-platform support and allow + # pip to create the appropriate form of executable for the target platform. entry_points=''' - [paste.paster_command] - archiver = ckanext.archiver.commands:Archiver + [paste.paster_command] + archiver = ckanext.archiver.commands:Archiver + + [ckan.plugins] + archiver = ckanext.archiver.plugin:ArchiverPlugin + testipipe = ckanext.archiver.plugin:TestIPipePlugin + + [ckan.celery_task] + tasks = ckanext.archiver.celery_import:task_imports - [ckan.plugins] - archiver = ckanext.archiver.plugin:ArchiverPlugin - testipipe = ckanext.archiver.plugin:TestIPipePlugin + [babel.extractors] + ckan = ckan.lib.extract:extract_ckan + ''', - [ckan.celery_task] - tasks = ckanext.archiver.celery_import:task_imports - ''' + # If you are changing from the default layout of your extension, you may + # have to change the message extractors, you can read more about babel + # message extraction at + # http://babel.pocoo.org/docs/messages/#extraction-method-mapping-and-configuration + message_extractors={ + 'ckanext': [ + ('**.py', 'python', None), + ('**.js', 'javascript', None), + ('**/templates/**.html', 'ckan', None), + ], + } ) diff --git a/test.ini b/test.ini index 7c1094c4..a7c707a2 100644 --- a/test.ini +++ b/test.ini @@ -1,12 +1,5 @@ -# -# ckan - Pylons testing environment configuration -# -# The %(here)s variable will be replaced with the parent directory of this file -# [DEFAULT] -debug = true -# Uncomment and replace with the address which should receive any error reports -#email_to = you@yourdomain.com +debug = false smtp_server = localhost error_email_from = paste@localhost