Update main with local changes and upstream

.github/workflows/dependencies.yml

@@ -0,0 +1,41 @@
+name: update dependencies
+on:
+
+  schedule:
+    - cron:  '0 0 1 * *'
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: development
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+          cache: pip
+
+      - name: Install pre-commit and pip-tools
+        run: pip install pre-commit pip-tools
+
+      - name: Run pre-commit autoupdate
+        run: pre-commit autoupdate
+
+      - name: Run pip-compile
+        run: pip-compile --upgrade
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v4
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          base: development
+          branch: dependency-updates
+          delete-branch: true
+          title: Dependency Updates
+          commit-message: Dependency updates

.pre-commit-config.yaml

@@ -1,20 +1,20 @@
 repos:
 - repo: https://github.com/pre-commit/mirrors-autopep8
-  rev: v1.5
+  rev: v2.0.1
   hooks:
   - id: autopep8
     args:
-      - --diff
+      - --in-place
       - --aggressive
-- repo: https://gitlab.com/pycqa/flake8
-  rev: 3.7.9
+- repo: https://github.com/pycqa/flake8
+  rev: 6.0.0
   hooks:
   - id: flake8
-- repo: git://github.com/doublify/pre-commit-isort
-  rev: v4.3.0
+- repo: https://github.com/pycqa/isort
+  rev: 5.11.4
   hooks:
   - id: isort
 - repo: https://github.com/jazzband/pip-tools
-  rev: 6.2.0
+  rev: 6.12.1
   hooks:
     - id: pip-compile

.travis.yml

@@ -1,4 +1,6 @@
+dist: bionic
 language: python
+python: "3.10"
 cache:
   directories:
     - $HOME/.cache/pip

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.6
+FROM python:3.10
 
 ENV PYTHONUNBUFFERED 1
 RUN mkdir /code

Dockerfile.cron

@@ -0,0 +1,21 @@
+FROM python:3.10
+
+ENV PYTHONUNBUFFERED 1
+RUN apt-get update && apt-get install -y cron && which cron && rm -rf /etc/cron.*/*
+RUN apt-get -y install python3-pip
+RUN pip install --upgrade pip
+
+RUN mkdir /code
+WORKDIR /code
+ADD requirements.txt /code/
+RUN pip install --upgrade pip && pip install -r requirements.txt
+ADD . /code/
+RUN chmod +x /code
+
+RUN ln -s /usr/local/bin/python3 /code/scorpio/python
+ADD ./cron/scorpio_cron /etc/cron.d/scorpio_cron
+RUN chmod 644 /etc/cron.d/scorpio_cron
+RUN crontab /etc/cron.d/scorpio_cron
+RUN mkdir -p /var/log/scorpio-cron
+ENTRYPOINT ["/code/cron/entrypoint.sh"]
+CMD ["cron","-f","-l","2"]

Dockerfile.prod

@@ -0,0 +1,36 @@
+FROM python:3.10
+
+ENV PYTHONUNBUFFERED 1
+ARG SCORPIO_DNS
+ARG APPLICATION_PORT
+RUN apt-get update
+RUN apt-get install --yes apache2 apache2-dev 
+RUN apt-get install --yes postgresql
+RUN apt-get -y install python3-pip
+RUN pip install --upgrade pip
+
+RUN wget https://github.com/GrahamDumpleton/mod_wsgi/archive/refs/tags/4.9.0.tar.gz \
+    && tar xvfz 4.9.0.tar.gz \
+    && cd mod_wsgi-4.9.0 \
+    && ./configure --with-apxs=/usr/bin/apxs --with-python=/usr/local/bin/python \
+    && make \
+    && make install \
+    && make clean
+
+ADD ./apache/000-scorpio.conf /etc/apache2/sites-available/000-scorpio.conf
+RUN sed "s/ENV_SCORPIO_DNS/${SCORPIO_DNS}/" -i /etc/apache2/sites-available/000-scorpio.conf
+RUN sed "s/ENV_SCORPIO_PORT/${APPLICATION_PORT}/" -i /etc/apache2/sites-available/000-scorpio.conf
+ADD ./apache/wsgi.load /etc/apache2/mods-available/wsgi.load
+RUN a2dissite 000-default
+RUN a2ensite 000-scorpio.conf
+RUN a2enmod headers
+RUN a2enmod rewrite
+RUN a2enmod wsgi
+
+COPY . /var/www/scorpio
+WORKDIR /var/www/scorpio
+RUN pip install -r requirements.txt
+RUN ./manage.py collectstatic
+
+EXPOSE ${APPLICATION_PORT}
+CMD ["apache2ctl", "-D", "FOREGROUND"]

README.md

@@ -34,6 +34,8 @@ Scorpio configurations are stored in `/scorpio/config.py`. This file is excluded
 
 The first time the container is started, the example config file (`/scorpio/config.py.example`) will be copied to create the config file if it doesn't already exist.
 
+Deployment using the `Dockerfile.prod` file is intended to bring up a production image (based on Apache/WSGI) which is ready to be proxied by an apache, nginx, traefik or similar frontend. Dockerfile.prod expects two environment arguments to be available at build time: `SCORPIO_DNS` and `APPLICATION_PORT`. Apache will Listen on `${APPLICATION_PORT}` with a ServerName of `${SCORPIO_DNS}`.  If you don't anticipate exposing the Scorpio web UI publicly, a `SCORPIO_DNS` value of your container name or "localhost" is just fine.  Deployment of the `Dockerfile.cron` will launch a container which will run Scorpio tasks automatically, per `/cron/scorpio_cron`.
+
 
 ## Services
 

apache/000-scorpio.conf

@@ -0,0 +1,20 @@
+Listen ENV_SCORPIO_PORT
+<VirtualHost *:ENV_SCORPIO_PORT>
+    ErrorLog /var/log/apache2/scorpio_error_log
+    CustomLog /var/log/apache2/scorpio_access_log combined
+    ServerName ENV_SCORPIO_DNS
+    DocumentRoot /var/www/html
+    Alias /static /var/www/scorpio/static
+    <Directory /var/www/scorpio/static>
+        Options Includes FollowSymLinks MultiViews
+        Require all granted
+    </Directory>
+    <Directory /var/www/scorpio/scorpio>
+        WSGIProcessGroup scorpio
+        WSGIApplicationGroup %{GLOBAL}
+        Require all granted
+    </Directory>
+    WSGIDaemonProcess scorpio home=/var/www/scorpio
+    WSGIProcessGroup scorpio
+    WSGIScriptAlias / /var/www/scorpio/scorpio/wsgi.py
+</VirtualHost>

apache/wsgi.load

@@ -0,0 +1 @@
+LoadModule wsgi_module /usr/lib/apache2/modules/mod_wsgi.so

cron/entrypoint.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+env >> environment
+
+# execute CMD
+echo "$@"
+exec "$@"

cron/scorpio_cron

@@ -0,0 +1,7 @@
+# Don't remove empty line at end of file. It is required for cron to run.
+SCORPIO_ROOT=/code/scorpio
+00,30 * * * * $SCORPIO_ROOT/python -u /code/manage.py runcrons "indexer.cron.IndexAgents" >> /var/log/scorpio-cron/scorpio-agents.txt 2>&1
+10,40 * * * * $SCORPIO_ROOT/python -u /code/manage.py runcrons "indexer.cron.IndexCollections" >> /var/log/scorpio-cron/scorpio-collections.txt 2>&1
+20,50 * * * * $SCORPIO_ROOT/python -u /code/manage.py runcrons "indexer.cron.IndexObjects" >> /var/log/scorpio-cron/scorpio-objects.txt 2>&1
+5 0,12 * * * $SCORPIO_ROOT/python -u /code/manage.py runcrons "indexer.cron.CleanUpCompleted" >> /var/log/scorpio-cron/scorpio-cleanupcompleted.txt 2>&1
+

docker-compose.yml

@@ -2,7 +2,11 @@ version: '3'
 
 services:
   scorpio-db:
-    image: postgres:10.4
+    image: postgres:14.4
+    environment:
+      - POSTGRES_USER=postgres  # name of the application database user
+      - POSTGRES_PASSWORD=postgres  # name of the application database user
+      - POSTGRES_DB=postgres  # name of the application database
     volumes:
       - scorpiodbvolume:/var/lib/postgresql/data/
     networks:

indexer/cron.py

@@ -1,3 +1,4 @@
+import json
 from datetime import datetime
 
 from django_cron import CronJobBase, Schedule
@@ -12,20 +13,21 @@ class BaseCron(CronJobBase):
     clean = False
 
     def do(self):
-        result = False
+        result = ''
         start = datetime.now()
         action = "Full" if self.clean else "Incremental"
         object_type = self.object_type if self.object_type else "all"
         indexed = []
         print("{} indexing of {} records started at {}".format(action, object_type, start))
         try:
             indexed = Indexer().add(object_type=self.object_type, clean=self.clean)
+            end = datetime.now()
+            print("{} records indexed in {}".format(len(indexed), end - start))
+            result = "{} index of {} records complete at {}\n".format(action, object_type, end)
         except Exception as e:
+            result = str(e)
             print(e)
-        end = datetime.now()
-        print("{} records indexed in {}".format(len(indexed), end - start))
-        print("{} index of {} records complete at {}\n".format(action, object_type, end))
-        result = True
+        print(result)
         return result
 
 
@@ -92,7 +94,9 @@ class CleanUpCompleted(CronJobBase):
 
     def do(self):
         try:
-            return IndexRun.objects.filter(indexrunerror__isnull=True, status=IndexRun.FINISHED).delete()
+            retval = IndexRun.objects.filter(indexrunerror__isnull=True, status=IndexRun.FINISHED).delete()
+            result = json.dumps(retval)
         except Exception as e:
-            print("Error cleaning up completed IndexRun objects: {}".format(e))
-            return False
+            result = "Error cleaning up completed IndexRun objects: {}".format(e)
+            print(result)
+        return result

indexer/indexers.py

@@ -5,6 +5,7 @@
 from electronbonder.client import ElectronBond
 from rac_es.documents import (Agent, BaseDescriptionComponent, Collection,
                               Object, Term)
+
 from scorpio import settings
 
 from .models import IndexRun, IndexRunError

indexer/tests.py

@@ -9,6 +9,7 @@
 from elasticsearch_dsl import connections
 from rac_es.documents import BaseDescriptionComponent
 from rest_framework.test import APIClient, APIRequestFactory
+
 from scorpio import settings
 
 from .cron import (CleanUpCompleted, IndexAgents, IndexAgentsClean, IndexAll,
@@ -104,3 +105,7 @@ def test_cleanup(self):
     def test_process(self):
         self.index_objects()
         self.delete_objects()
+
+    def test_ping_view(self):
+        response = self.client.get(reverse('ping'))
+        self.assertEqual(response.status_code, 200)

requirements.in

@@ -1,12 +1,11 @@
-asterism==0.7.2
-Django==3.2.10
-django-cron==0.5.1
-djangorestframework==3.12.4
-elasticsearch==7.13.1
-elasticsearch-dsl==7.3.0
-ElectronBonder==0.7.2
-health-check==3.4.1
-odin==1.7.0
-psycopg2-binary==2.9.1
-rac-es==0.17.2
-requests==2.25.1
+asterism~=0.9
+Django~=4.1
+django4-cron~=0.5
+djangorestframework~=3.13
+elasticsearch~=7.17
+elasticsearch-dsl~=7.4
+ElectronBonder~=1.1
+odin~=1.7
+psycopg2-binary~=2.9
+rac-es~=1.0
+requests~=2.28