From 46fe5b8c1987a91fb067e20208e8a46eb6a95b04 Mon Sep 17 00:00:00 2001 From: Hillel Arnold Date: Sat, 26 Feb 2022 12:23:56 -0500 Subject: [PATCH 01/29] update pre-commit URLs and versions --- .pre-commit-config.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1be65d5..a6d52b0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,20 +1,20 @@ repos: - repo: https://github.com/pre-commit/mirrors-autopep8 - rev: v1.5 + rev: v1.6.0 hooks: - id: autopep8 args: - - --diff + - --in-place - --aggressive -- repo: https://gitlab.com/pycqa/flake8 - rev: 3.7.9 +- repo: https://github.com/pycqa/flake8 + rev: 4.0.1 hooks: - id: flake8 -- repo: git://github.com/doublify/pre-commit-isort - rev: v4.3.0 +- repo: https://github.com/pycqa/isort + rev: 5.10.1 hooks: - id: isort - repo: https://github.com/jazzband/pip-tools - rev: 6.2.0 + rev: 6.5.1 hooks: - id: pip-compile From 73d58390be14beb4cbef1f3e577528cb360e3062 Mon Sep 17 00:00:00 2001 From: Hillel Arnold Date: Sat, 26 Feb 2022 12:24:08 -0500 Subject: [PATCH 02/29] update python and dependencies --- Dockerfile | 2 +- requirements.in | 23 +++++++++++----------- requirements.txt | 50 ++++++++++++++++-------------------------------- 3 files changed, 28 insertions(+), 47 deletions(-) diff --git a/Dockerfile b/Dockerfile index a0d685a..2859e99 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.6 +FROM python:3.10 ENV PYTHONUNBUFFERED 1 RUN mkdir /code diff --git a/requirements.in b/requirements.in index cccced0..705218a 100644 --- a/requirements.in +++ b/requirements.in @@ -1,12 +1,11 @@ -asterism==0.7.2 -Django==3.2.10 -django-cron==0.5.1 -djangorestframework==3.12.4 -elasticsearch==7.13.1 -elasticsearch-dsl==7.3.0 -ElectronBonder==0.7.2 -health-check==3.4.1 -odin==1.7.0 -psycopg2-binary==2.9.1 -rac-es==0.17.2 -requests==2.25.1 +asterism==0.8 +Django==4.0.2 +django4-cron==0.5.1 +djangorestframework==3.13.1 +elasticsearch==7.17.0 +elasticsearch-dsl==7.4.0 +ElectronBonder==1.0 +odin==1.7.3 +psycopg2-binary==2.9.3 +rac-es==0.18 +requests==2.27.1 diff --git a/requirements.txt b/requirements.txt index e7ce5c5..decd4bf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,12 @@ # -# This file is autogenerated by pip-compile with python 3.6 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile # asgiref==3.4.1 # via django -asterism==0.7.2 +asterism==0.8 # via -r requirements.in bagit==1.8.1 # via asterism @@ -14,63 +14,51 @@ certifi==2021.5.30 # via # elasticsearch # requests -chardet==4.0.0 +charset-normalizer==2.0.12 # via requests -clinner==1.12.3 - # via health-check -colorlog==3.2.0 - # via clinner -django==3.2.10 +django==4.0.2 # via # -r requirements.in # asterism # django-common-helpers - # django-cron + # django4-cron # djangorestframework django-common-helpers==0.9.2 - # via django-cron -django-cron==0.5.1 + # via django4-cron +django4-cron==0.5.1 # via -r requirements.in -djangorestframework==3.12.4 +djangorestframework==3.13.1 # via # -r requirements.in # asterism -elasticsearch==7.13.1 +elasticsearch==7.17.0 # via # -r requirements.in # elasticsearch-dsl # rac-es -elasticsearch-dsl==7.3.0 +elasticsearch-dsl==7.4.0 # via # -r requirements.in # rac-es -electronbonder==0.7.2 - # via -r requirements.in -gitdb==4.0.7 - # via gitpython -gitpython==3.1.18 - # via health-check -health-check==3.4.1 +electronbonder==1.0 # via -r requirements.in idna==2.10 # via requests -odin==1.7.0 +odin==1.7.3 # via # -r requirements.in # asterism -psycopg2-binary==2.9.1 +psycopg2-binary==2.9.3 # via # -r requirements.in # asterism python-dateutil==2.8.2 # via elasticsearch-dsl pytz==2021.1 - # via django -pyyaml==5.4.1 - # via health-check -rac-es==0.17.2 + # via djangorestframework +rac-es==0.18 # via -r requirements.in -requests==2.25.1 +requests==2.27.1 # via # -r requirements.in # electronbonder @@ -80,14 +68,8 @@ six==1.16.0 # electronbonder # odin # python-dateutil -smmap==4.0.0 - # via gitdb sqlparse==0.4.2 # via django -typing-extensions==3.10.0.2 - # via - # asgiref - # gitpython urllib3==1.26.6 # via # elasticsearch From 9bdb549916848eb566102e5f5c2f61ed4538e32a Mon Sep 17 00:00:00 2001 From: Hillel Arnold Date: Sat, 26 Feb 2022 12:24:19 -0500 Subject: [PATCH 03/29] update travis configs --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 5ed0ecf..f7fd08e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,6 @@ +dist: bionic language: python +python: "3.10" cache: directories: - $HOME/.cache/pip From 8271d2b3df6b5c00504831fa059e68f3e4b86818 Mon Sep 17 00:00:00 2001 From: Hillel Arnold Date: Sat, 26 Feb 2022 12:24:38 -0500 Subject: [PATCH 04/29] replace health_check with asterism view --- indexer/tests.py | 5 +++++ scorpio/settings.py | 1 - scorpio/urls.py | 6 ++++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/indexer/tests.py b/indexer/tests.py index 3e2d43c..b5c5848 100644 --- a/indexer/tests.py +++ b/indexer/tests.py @@ -9,6 +9,7 @@ from elasticsearch_dsl import connections from rac_es.documents import BaseDescriptionComponent from rest_framework.test import APIClient, APIRequestFactory + from scorpio import settings from .cron import (CleanUpCompleted, IndexAgents, IndexAgentsClean, IndexAll, @@ -104,3 +105,7 @@ def test_cleanup(self): def test_process(self): self.index_objects() self.delete_objects() + + def test_ping_view(self): + response = self.client.get(reverse('ping')) + self.assertEqual(response.status_code, 200) diff --git a/scorpio/settings.py b/scorpio/settings.py index 20efa08..8853937 100644 --- a/scorpio/settings.py +++ b/scorpio/settings.py @@ -40,7 +40,6 @@ 'django.contrib.messages', 'django.contrib.staticfiles', 'rest_framework', - 'health_check', 'indexer', 'django_cron', ] diff --git a/scorpio/urls.py b/scorpio/urls.py index 3ff2b18..166247c 100644 --- a/scorpio/urls.py +++ b/scorpio/urls.py @@ -13,11 +13,13 @@ 1. Import the include() function: from django.urls import include, path 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ +from asterism.views import PingView from django.contrib import admin from django.urls import include, re_path +from rest_framework.schemas import get_schema_view + from indexer.views import (IndexAddView, IndexDeleteView, IndexResetView, IndexRunViewSet) -from rest_framework.schemas import get_schema_view from .routers import ScorpioRouter @@ -34,7 +36,7 @@ re_path(r'^index/add/', IndexAddView.as_view(), name='index-add'), re_path(r'^index/delete/', IndexDeleteView.as_view(), name='index-delete'), re_path(r'^index/reset/', IndexResetView.as_view(), name='index-reset'), - re_path(r'^status/', include('health_check.api.urls')), + re_path(r'^status/', PingView.as_view(), name='ping'), re_path(r'^schema/', schema_view, name='schema'), re_path(r'^', include(router.urls)), ] From ac247712f68116c40758225cb5ea91cb2d6ab048 Mon Sep 17 00:00:00 2001 From: Hillel Arnold Date: Sat, 26 Feb 2022 12:24:55 -0500 Subject: [PATCH 05/29] linting --- indexer/indexers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/indexer/indexers.py b/indexer/indexers.py index 1d89732..0f5eba1 100644 --- a/indexer/indexers.py +++ b/indexer/indexers.py @@ -5,6 +5,7 @@ from electronbonder.client import ElectronBond from rac_es.documents import (Agent, BaseDescriptionComponent, Collection, Object, Term) + from scorpio import settings from .models import IndexRun, IndexRunError From 45e5f9c1cc00bf70dbbee3e6fccff762a523649a Mon Sep 17 00:00:00 2001 From: Hillel Arnold Date: Mon, 9 May 2022 15:51:53 -0400 Subject: [PATCH 06/29] update requirements --- requirements.in | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.in b/requirements.in index 705218a..3ea2bdd 100644 --- a/requirements.in +++ b/requirements.in @@ -1,5 +1,5 @@ asterism==0.8 -Django==4.0.2 +Django==4.0.4 django4-cron==0.5.1 djangorestframework==3.13.1 elasticsearch==7.17.0 diff --git a/requirements.txt b/requirements.txt index decd4bf..4a519d2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ certifi==2021.5.30 # requests charset-normalizer==2.0.12 # via requests -django==4.0.2 +django==4.0.4 # via # -r requirements.in # asterism From 23ab4a0a1dadbaa4edb63c262f1b86999d4fad6c Mon Sep 17 00:00:00 2001 From: Hillel Arnold Date: Fri, 13 May 2022 10:51:39 -0400 Subject: [PATCH 07/29] update rac_es --- requirements.in | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.in b/requirements.in index 3ea2bdd..d3e6e28 100644 --- a/requirements.in +++ b/requirements.in @@ -7,5 +7,5 @@ elasticsearch-dsl==7.4.0 ElectronBonder==1.0 odin==1.7.3 psycopg2-binary==2.9.3 -rac-es==0.18 +rac-es==1.0.0 requests==2.27.1 diff --git a/requirements.txt b/requirements.txt index 4a519d2..8bdfb15 100644 --- a/requirements.txt +++ b/requirements.txt @@ -56,7 +56,7 @@ python-dateutil==2.8.2 # via elasticsearch-dsl pytz==2021.1 # via djangorestframework -rac-es==0.18 +rac-es==1.0.0 # via -r requirements.in requests==2.27.1 # via From 75419c0a5a43e77d482655d1945788b1d21b0510 Mon Sep 17 00:00:00 2001 From: Hannah Date: Thu, 21 Jul 2022 14:54:36 -0400 Subject: [PATCH 08/29] add gh-actions file, remove dependabot --- .github/workflows/dependencies.yml | 33 ++++++++++++++++++++++++++++++ dependabot.yml | 7 ------- 2 files changed, 33 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/dependencies.yml delete mode 100644 dependabot.yml diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml new file mode 100644 index 0000000..925762c --- /dev/null +++ b/.github/workflows/dependencies.yml @@ -0,0 +1,33 @@ +name: update dependencies +on: + + schedule: + - cron: '0 0 1 * *' + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + ref: development + + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + cache: pip + - run: | + pip install pip-tools + pip-compile --upgrade + - name: Create Pull Request + uses: peter-evans/create-pull-request@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + base: development + branch: dependency-updates + delete-branch: true + title: Dependency Updates + commit-message: Dependency updates \ No newline at end of file diff --git a/dependabot.yml b/dependabot.yml deleted file mode 100644 index f052884..0000000 --- a/dependabot.yml +++ /dev/null @@ -1,7 +0,0 @@ -version: 2 -updates: - - package-ecosystem: "pip" - directory: "/" - schedule: - interval: "monthly" - target-branch: "development" From fc07219dc539e9b923bd372c68484e9772684759 Mon Sep 17 00:00:00 2001 From: Hannah Date: Thu, 21 Jul 2022 15:09:52 -0400 Subject: [PATCH 09/29] update requirements --- requirements.in | 22 +++++++++++----------- requirements.txt | 24 ++++++++++++------------ 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/requirements.in b/requirements.in index d3e6e28..42e21fd 100644 --- a/requirements.in +++ b/requirements.in @@ -1,11 +1,11 @@ -asterism==0.8 -Django==4.0.4 -django4-cron==0.5.1 -djangorestframework==3.13.1 -elasticsearch==7.17.0 -elasticsearch-dsl==7.4.0 -ElectronBonder==1.0 -odin==1.7.3 -psycopg2-binary==2.9.3 -rac-es==1.0.0 -requests==2.27.1 +asterism~=0.9 +Django~=4.0 +django4-cron~=0.5 +djangorestframework~=3.13 +elasticsearch~=7.17 +elasticsearch-dsl~=7.4 +ElectronBonder~=1.1 +odin~=1.7 +psycopg2-binary~=2.9 +rac-es~=1.0 +requests~=2.28 diff --git a/requirements.txt b/requirements.txt index 8bdfb15..10d597b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,19 +4,19 @@ # # pip-compile # -asgiref==3.4.1 +asgiref==3.5.2 # via django -asterism==0.8 +asterism==0.9.0 # via -r requirements.in bagit==1.8.1 # via asterism -certifi==2021.5.30 +certifi==2022.6.15 # via # elasticsearch # requests -charset-normalizer==2.0.12 +charset-normalizer==2.1.0 # via requests -django==4.0.4 +django==4.0.6 # via # -r requirements.in # asterism @@ -31,7 +31,7 @@ djangorestframework==3.13.1 # via # -r requirements.in # asterism -elasticsearch==7.17.0 +elasticsearch==7.17.4 # via # -r requirements.in # elasticsearch-dsl @@ -40,11 +40,11 @@ elasticsearch-dsl==7.4.0 # via # -r requirements.in # rac-es -electronbonder==1.0 +electronbonder==1.1 # via -r requirements.in -idna==2.10 +idna==3.3 # via requests -odin==1.7.3 +odin==1.8.1 # via # -r requirements.in # asterism @@ -54,11 +54,11 @@ psycopg2-binary==2.9.3 # asterism python-dateutil==2.8.2 # via elasticsearch-dsl -pytz==2021.1 +pytz==2022.1 # via djangorestframework rac-es==1.0.0 # via -r requirements.in -requests==2.27.1 +requests==2.28.1 # via # -r requirements.in # electronbonder @@ -70,7 +70,7 @@ six==1.16.0 # python-dateutil sqlparse==0.4.2 # via django -urllib3==1.26.6 +urllib3==1.26.10 # via # elasticsearch # requests From 48a98de102909e8a0cfad78538cdbc022a2d0b8d Mon Sep 17 00:00:00 2001 From: HaSistrunk Date: Mon, 1 Aug 2022 00:22:10 +0000 Subject: [PATCH 10/29] Dependency updates --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 10d597b..4b4d74b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -70,7 +70,7 @@ six==1.16.0 # python-dateutil sqlparse==0.4.2 # via django -urllib3==1.26.10 +urllib3==1.26.11 # via # elasticsearch # requests From 07a201af7d21b886f7a0e5ea804fede3aab74126 Mon Sep 17 00:00:00 2001 From: Clinton Graham Date: Tue, 16 Aug 2022 17:59:59 -0400 Subject: [PATCH 11/29] #170: set cron returns to recent string values --- indexer/cron.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/indexer/cron.py b/indexer/cron.py index 1baf567..65594c1 100644 --- a/indexer/cron.py +++ b/indexer/cron.py @@ -5,6 +5,7 @@ from .indexers import Indexer from .models import IndexRun +import json class BaseCron(CronJobBase): RUN_EVERY_MINS = 0 @@ -12,7 +13,7 @@ class BaseCron(CronJobBase): clean = False def do(self): - result = False + result = '' start = datetime.now() action = "Full" if self.clean else "Incremental" object_type = self.object_type if self.object_type else "all" @@ -21,11 +22,12 @@ def do(self): try: indexed = Indexer().add(object_type=self.object_type, clean=self.clean) except Exception as e: + result = str(e) print(e) end = datetime.now() print("{} records indexed in {}".format(len(indexed), end - start)) - print("{} index of {} records complete at {}\n".format(action, object_type, end)) - result = True + result = "{} index of {} records complete at {}\n".format(action, object_type, end) + print(result) return result @@ -92,7 +94,10 @@ class CleanUpCompleted(CronJobBase): def do(self): try: - return IndexRun.objects.filter(indexrunerror__isnull=True, status=IndexRun.FINISHED).delete() + retval = IndexRun.objects.filter(indexrunerror__isnull=True, status=IndexRun.FINISHED).delete() + result = json.dumps(retval) except Exception as e: - print("Error cleaning up completed IndexRun objects: {}".format(e)) - return False + result = "Error cleaning up completed IndexRun objects: {}".format(e) + print(result) + return result + From 4d94f23535b8e3d6c028f8062545e00231d3eb5a Mon Sep 17 00:00:00 2001 From: Clinton Graham Date: Tue, 16 Aug 2022 18:09:57 -0400 Subject: [PATCH 12/29] #170: move cron sucess messages into try --- indexer/cron.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/indexer/cron.py b/indexer/cron.py index 65594c1..e16da40 100644 --- a/indexer/cron.py +++ b/indexer/cron.py @@ -21,12 +21,12 @@ def do(self): print("{} indexing of {} records started at {}".format(action, object_type, start)) try: indexed = Indexer().add(object_type=self.object_type, clean=self.clean) + end = datetime.now() + print("{} records indexed in {}".format(len(indexed), end - start)) + result = "{} index of {} records complete at {}\n".format(action, object_type, end) except Exception as e: result = str(e) print(e) - end = datetime.now() - print("{} records indexed in {}".format(len(indexed), end - start)) - result = "{} index of {} records complete at {}\n".format(action, object_type, end) print(result) return result From e49880c0b5b357749a649265861dde835af8abec Mon Sep 17 00:00:00 2001 From: Hillel Arnold Date: Tue, 16 Aug 2022 20:35:05 -0400 Subject: [PATCH 13/29] formatting --- indexer/cron.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/indexer/cron.py b/indexer/cron.py index e16da40..777878e 100644 --- a/indexer/cron.py +++ b/indexer/cron.py @@ -1,3 +1,4 @@ +import json from datetime import datetime from django_cron import CronJobBase, Schedule @@ -5,7 +6,6 @@ from .indexers import Indexer from .models import IndexRun -import json class BaseCron(CronJobBase): RUN_EVERY_MINS = 0 @@ -100,4 +100,3 @@ def do(self): result = "Error cleaning up completed IndexRun objects: {}".format(e) print(result) return result - From 46d42152c993431e3ef0512d07bb5265d93df546 Mon Sep 17 00:00:00 2001 From: HaSistrunk Date: Thu, 1 Sep 2022 00:20:10 +0000 Subject: [PATCH 14/29] Dependency updates --- requirements.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 4b4d74b..470faef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,9 +14,9 @@ certifi==2022.6.15 # via # elasticsearch # requests -charset-normalizer==2.1.0 +charset-normalizer==2.1.1 # via requests -django==4.0.6 +django==4.1 # via # -r requirements.in # asterism @@ -31,7 +31,7 @@ djangorestframework==3.13.1 # via # -r requirements.in # asterism -elasticsearch==7.17.4 +elasticsearch==7.17.6 # via # -r requirements.in # elasticsearch-dsl @@ -54,7 +54,7 @@ psycopg2-binary==2.9.3 # asterism python-dateutil==2.8.2 # via elasticsearch-dsl -pytz==2022.1 +pytz==2022.2.1 # via djangorestframework rac-es==1.0.0 # via -r requirements.in @@ -70,7 +70,7 @@ six==1.16.0 # python-dateutil sqlparse==0.4.2 # via django -urllib3==1.26.11 +urllib3==1.26.12 # via # elasticsearch # requests From 429f17019541af148961e07635023e1268165517 Mon Sep 17 00:00:00 2001 From: Clinton Graham Date: Fri, 2 Sep 2022 17:23:25 -0400 Subject: [PATCH 15/29] #175: Production-ready docker configuration --- Dockerfile.cron | 21 +++++++++++++++++++++ Dockerfile.prod | 36 ++++++++++++++++++++++++++++++++++++ README.md | 2 ++ apache/000-scorpio.conf | 20 ++++++++++++++++++++ apache/wsgi.load | 1 + cron/entrypoint.sh | 7 +++++++ cron/scorpio_cron | 7 +++++++ 7 files changed, 94 insertions(+) create mode 100644 Dockerfile.cron create mode 100644 Dockerfile.prod create mode 100644 apache/000-scorpio.conf create mode 100644 apache/wsgi.load create mode 100755 cron/entrypoint.sh create mode 100644 cron/scorpio_cron diff --git a/Dockerfile.cron b/Dockerfile.cron new file mode 100644 index 0000000..0f557d4 --- /dev/null +++ b/Dockerfile.cron @@ -0,0 +1,21 @@ +FROM python:3.10 + +ENV PYTHONUNBUFFERED 1 +RUN apt-get update && apt-get install -y cron && which cron && rm -rf /etc/cron.*/* +RUN apt-get -y install python3-pip +RUN pip install --upgrade pip + +RUN mkdir /code +WORKDIR /code +ADD requirements.txt /code/ +RUN pip install --upgrade pip && pip install -r requirements.txt +ADD . /code/ +RUN chmod +x /code + +RUN ln -s /usr/local/bin/python3 /code/scorpio/python +ADD ./cron/scorpio_cron /etc/cron.d/scorpio_cron +RUN chmod 644 /etc/cron.d/scorpio_cron +RUN crontab /etc/cron.d/scorpio_cron +RUN mkdir -p /var/log/scorpio-cron +ENTRYPOINT ["/code/cron/entrypoint.sh"] +CMD ["cron","-f","-l","2"] diff --git a/Dockerfile.prod b/Dockerfile.prod new file mode 100644 index 0000000..7f405a8 --- /dev/null +++ b/Dockerfile.prod @@ -0,0 +1,36 @@ +FROM python:3.10 + +ENV PYTHONUNBUFFERED 1 +ARG SCORPIO_DNS +ARG APPLICATION_PORT +RUN apt-get update +RUN apt-get install --yes apache2 apache2-dev +RUN apt-get install --yes postgresql +RUN apt-get -y install python3-pip +RUN pip install --upgrade pip + +RUN wget https://github.com/GrahamDumpleton/mod_wsgi/archive/refs/tags/4.9.0.tar.gz \ + && tar xvfz 4.9.0.tar.gz \ + && cd mod_wsgi-4.9.0 \ + && ./configure --with-apxs=/usr/bin/apxs --with-python=/usr/local/bin/python \ + && make \ + && make install \ + && make clean + +ADD ./apache/000-scorpio.conf /etc/apache2/sites-available/000-scorpio.conf +RUN sed "s/ENV_SCORPIO_DNS/${SCORPIO_DNS}/" -i /etc/apache2/sites-available/000-scorpio.conf +RUN sed "s/ENV_SCORPIO_PORT/${APPLICATION_PORT}/" -i /etc/apache2/sites-available/000-scorpio.conf +ADD ./apache/wsgi.load /etc/apache2/mods-available/wsgi.load +RUN a2dissite 000-default +RUN a2ensite 000-scorpio.conf +RUN a2enmod headers +RUN a2enmod rewrite +RUN a2enmod wsgi + +COPY . /var/www/scorpio +WORKDIR /var/www/scorpio +RUN pip install -r requirements.txt +RUN ./manage.py collectstatic + +EXPOSE ${APPLICATION_PORT} +CMD ["apache2ctl", "-D", "FOREGROUND"] diff --git a/README.md b/README.md index b133d7b..293d90c 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,8 @@ Scorpio configurations are stored in `/scorpio/config.py`. This file is excluded The first time the container is started, the example config file (`/scorpio/config.py.example`) will be copied to create the config file if it doesn't already exist. +Deployment using the `Dockerfile.prod` file is intended to bring up a production image (based on Apache/WSGI) which is ready to be proxied by an apache, nginx, traefik or similar frontend. Dockerfile.prod expects two environment arguments to be available at build time: `SCORPIO_DNS` and `APPLICATION_PORT`. Apache will Listen on `${APPLICATION_PORT}` with a ServerName of `${SCORPIO_DNS}`. If you don't anticipate exposing the Scorpio web UI publicly, a `SCORPIO_DNS` value of your container name or "localhost" is just fine. Deployment of the `Dockerfile.cron` will launch a container which will run Scorpio tasks automatically, per `/cron/scorpio_cron`. + ## Services diff --git a/apache/000-scorpio.conf b/apache/000-scorpio.conf new file mode 100644 index 0000000..e0eeceb --- /dev/null +++ b/apache/000-scorpio.conf @@ -0,0 +1,20 @@ +Listen ENV_SCORPIO_PORT + + ErrorLog /var/log/apache2/scorpio_error_log + CustomLog /var/log/apache2/scorpio_access_log combined + ServerName ENV_SCORPIO_DNS + DocumentRoot /var/www/html + Alias /static /var/www/scorpio/static + + Options Includes FollowSymLinks MultiViews + Require all granted + + + WSGIProcessGroup scorpio + WSGIApplicationGroup %{GLOBAL} + Require all granted + + WSGIDaemonProcess scorpio home=/var/www/scorpio + WSGIProcessGroup scorpio + WSGIScriptAlias / /var/www/scorpio/scorpio/wsgi.py + diff --git a/apache/wsgi.load b/apache/wsgi.load new file mode 100644 index 0000000..d76d1d7 --- /dev/null +++ b/apache/wsgi.load @@ -0,0 +1 @@ +LoadModule wsgi_module /usr/lib/apache2/modules/mod_wsgi.so diff --git a/cron/entrypoint.sh b/cron/entrypoint.sh new file mode 100755 index 0000000..662f6b9 --- /dev/null +++ b/cron/entrypoint.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +env >> environment + +# execute CMD +echo "$@" +exec "$@" diff --git a/cron/scorpio_cron b/cron/scorpio_cron new file mode 100644 index 0000000..f966422 --- /dev/null +++ b/cron/scorpio_cron @@ -0,0 +1,7 @@ +# Don't remove empty line at end of file. It is required for cron to run. +SCORPIO_ROOT=/code/scorpio +00,30 * * * * $SCORPIO_ROOT/python -u /code/manage.py runcrons "indexer.cron.IndexAgents" >> /var/log/scorpio-cron/scorpio-agents.txt 2>&1 +10,40 * * * * $SCORPIO_ROOT/python -u /code/manage.py runcrons "indexer.cron.IndexCollections" >> /var/log/scorpio-cron/scorpio-collections.txt 2>&1 +20,50 * * * * $SCORPIO_ROOT/python -u /code/manage.py runcrons "indexer.cron.IndexObjects" >> /var/log/scorpio-cron/scorpio-objects.txt 2>&1 +5 0,12 * * * $SCORPIO_ROOT/python -u /code/manage.py runcrons "indexer.cron.CleanUpCompleted" >> /var/log/scorpio-cron/scorpio-cleanupcompleted.txt 2>&1 + From f23e947766862b4452eddde3b8326fef130adce2 Mon Sep 17 00:00:00 2001 From: Patrick Date: Tue, 6 Sep 2022 15:13:16 -0400 Subject: [PATCH 16/29] Update django version Updates django version to match postgres requirements --- requirements.in | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.in b/requirements.in index 42e21fd..3267fc7 100644 --- a/requirements.in +++ b/requirements.in @@ -1,5 +1,5 @@ asterism~=0.9 -Django~=4.0 +Django~=4.0.7 django4-cron~=0.5 djangorestframework~=3.13 elasticsearch~=7.17 diff --git a/requirements.txt b/requirements.txt index 470faef..ce85262 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ certifi==2022.6.15 # requests charset-normalizer==2.1.1 # via requests -django==4.1 +django==4.0.7 # via # -r requirements.in # asterism From 8246de725d55c3423b8d123051a2f58d7b86e91f Mon Sep 17 00:00:00 2001 From: HaSistrunk Date: Sat, 1 Oct 2022 00:32:06 +0000 Subject: [PATCH 17/29] Dependency updates --- requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index ce85262..be32aca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ asterism==0.9.0 # via -r requirements.in bagit==1.8.1 # via asterism -certifi==2022.6.15 +certifi==2022.9.24 # via # elasticsearch # requests @@ -27,7 +27,7 @@ django-common-helpers==0.9.2 # via django4-cron django4-cron==0.5.1 # via -r requirements.in -djangorestframework==3.13.1 +djangorestframework==3.14.0 # via # -r requirements.in # asterism @@ -42,7 +42,7 @@ elasticsearch-dsl==7.4.0 # rac-es electronbonder==1.1 # via -r requirements.in -idna==3.3 +idna==3.4 # via requests odin==1.8.1 # via @@ -68,7 +68,7 @@ six==1.16.0 # electronbonder # odin # python-dateutil -sqlparse==0.4.2 +sqlparse==0.4.3 # via django urllib3==1.26.12 # via From 9e93aacfb7ce8d65d28d79ba52f0372a7e4bd6b7 Mon Sep 17 00:00:00 2001 From: HaSistrunk Date: Tue, 1 Nov 2022 00:26:29 +0000 Subject: [PATCH 18/29] Dependency updates --- requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index be32aca..7fd1563 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ certifi==2022.9.24 # requests charset-normalizer==2.1.1 # via requests -django==4.0.7 +django==4.0.8 # via # -r requirements.in # asterism @@ -31,7 +31,7 @@ djangorestframework==3.14.0 # via # -r requirements.in # asterism -elasticsearch==7.17.6 +elasticsearch==7.17.7 # via # -r requirements.in # elasticsearch-dsl @@ -48,13 +48,13 @@ odin==1.8.1 # via # -r requirements.in # asterism -psycopg2-binary==2.9.3 +psycopg2-binary==2.9.5 # via # -r requirements.in # asterism python-dateutil==2.8.2 # via elasticsearch-dsl -pytz==2022.2.1 +pytz==2022.6 # via djangorestframework rac-es==1.0.0 # via -r requirements.in From e4e18eb1ea02e30927811da83c5f02d7b3d08903 Mon Sep 17 00:00:00 2001 From: HaSistrunk Date: Thu, 1 Dec 2022 00:22:50 +0000 Subject: [PATCH 19/29] Dependency updates --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7fd1563..4851b05 100644 --- a/requirements.txt +++ b/requirements.txt @@ -70,7 +70,7 @@ six==1.16.0 # python-dateutil sqlparse==0.4.3 # via django -urllib3==1.26.12 +urllib3==1.26.13 # via # elasticsearch # requests From 53ee7ec01acbc6fcb3a6cf7a0099665dec010c7c Mon Sep 17 00:00:00 2001 From: Patrick Date: Fri, 9 Dec 2022 11:46:32 -0500 Subject: [PATCH 20/29] security patch updates dependencies --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 4851b05..bb35ffe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ asterism==0.9.0 # via -r requirements.in bagit==1.8.1 # via asterism -certifi==2022.9.24 +certifi==2022.12.7 # via # elasticsearch # requests @@ -31,7 +31,7 @@ djangorestframework==3.14.0 # via # -r requirements.in # asterism -elasticsearch==7.17.7 +elasticsearch==7.17.8 # via # -r requirements.in # elasticsearch-dsl From 1a5c22b4f500388e3414d4e36c6672a0e53ae388 Mon Sep 17 00:00:00 2001 From: HaSistrunk Date: Sun, 1 Jan 2023 00:20:21 +0000 Subject: [PATCH 21/29] Dependency updates --- requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index bb35ffe..e841b37 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ # -# This file is autogenerated by pip-compile with python 3.10 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: # # pip-compile # -asgiref==3.5.2 +asgiref==3.6.0 # via django asterism==0.9.0 # via -r requirements.in @@ -54,7 +54,7 @@ psycopg2-binary==2.9.5 # asterism python-dateutil==2.8.2 # via elasticsearch-dsl -pytz==2022.6 +pytz==2022.7 # via djangorestframework rac-es==1.0.0 # via -r requirements.in From 6f243e633debbb36ffbab3f36a203c924e37b933 Mon Sep 17 00:00:00 2001 From: Patrick Date: Tue, 3 Jan 2023 11:14:52 -0500 Subject: [PATCH 22/29] Update pre-commit conifg Updates hook versions for pre-commit config. --- .pre-commit-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a6d52b0..fbf80b3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,20 +1,20 @@ repos: - repo: https://github.com/pre-commit/mirrors-autopep8 - rev: v1.6.0 + rev: v2.0.1 hooks: - id: autopep8 args: - --in-place - --aggressive - repo: https://github.com/pycqa/flake8 - rev: 4.0.1 + rev: 6.0.0 hooks: - id: flake8 - repo: https://github.com/pycqa/isort - rev: 5.10.1 + rev: 5.11.4 hooks: - id: isort - repo: https://github.com/jazzband/pip-tools - rev: 6.5.1 + rev: 6.12.1 hooks: - id: pip-compile From 52dd4b756432bcf9526a205ce0ca6d08c8178eef Mon Sep 17 00:00:00 2001 From: Kevin Cloud Date: Fri, 27 Jan 2023 19:15:59 -0500 Subject: [PATCH 23/29] Implement cron as separate container #1 --- Dockerfile.cron | 21 +++++++++++++++++++++ cron/entrypoint.sh | 7 +++++++ cron/scorpio_cron | 7 +++++++ 3 files changed, 35 insertions(+) create mode 100644 Dockerfile.cron create mode 100755 cron/entrypoint.sh create mode 100644 cron/scorpio_cron diff --git a/Dockerfile.cron b/Dockerfile.cron new file mode 100644 index 0000000..0f557d4 --- /dev/null +++ b/Dockerfile.cron @@ -0,0 +1,21 @@ +FROM python:3.10 + +ENV PYTHONUNBUFFERED 1 +RUN apt-get update && apt-get install -y cron && which cron && rm -rf /etc/cron.*/* +RUN apt-get -y install python3-pip +RUN pip install --upgrade pip + +RUN mkdir /code +WORKDIR /code +ADD requirements.txt /code/ +RUN pip install --upgrade pip && pip install -r requirements.txt +ADD . /code/ +RUN chmod +x /code + +RUN ln -s /usr/local/bin/python3 /code/scorpio/python +ADD ./cron/scorpio_cron /etc/cron.d/scorpio_cron +RUN chmod 644 /etc/cron.d/scorpio_cron +RUN crontab /etc/cron.d/scorpio_cron +RUN mkdir -p /var/log/scorpio-cron +ENTRYPOINT ["/code/cron/entrypoint.sh"] +CMD ["cron","-f","-l","2"] diff --git a/cron/entrypoint.sh b/cron/entrypoint.sh new file mode 100755 index 0000000..662f6b9 --- /dev/null +++ b/cron/entrypoint.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +env >> environment + +# execute CMD +echo "$@" +exec "$@" diff --git a/cron/scorpio_cron b/cron/scorpio_cron new file mode 100644 index 0000000..f966422 --- /dev/null +++ b/cron/scorpio_cron @@ -0,0 +1,7 @@ +# Don't remove empty line at end of file. It is required for cron to run. +SCORPIO_ROOT=/code/scorpio +00,30 * * * * $SCORPIO_ROOT/python -u /code/manage.py runcrons "indexer.cron.IndexAgents" >> /var/log/scorpio-cron/scorpio-agents.txt 2>&1 +10,40 * * * * $SCORPIO_ROOT/python -u /code/manage.py runcrons "indexer.cron.IndexCollections" >> /var/log/scorpio-cron/scorpio-collections.txt 2>&1 +20,50 * * * * $SCORPIO_ROOT/python -u /code/manage.py runcrons "indexer.cron.IndexObjects" >> /var/log/scorpio-cron/scorpio-objects.txt 2>&1 +5 0,12 * * * $SCORPIO_ROOT/python -u /code/manage.py runcrons "indexer.cron.CleanUpCompleted" >> /var/log/scorpio-cron/scorpio-cleanupcompleted.txt 2>&1 + From 0df1a30fc1dabd515e423c16a076d1594b6429ac Mon Sep 17 00:00:00 2001 From: Kevin Cloud Date: Fri, 27 Jan 2023 19:23:23 -0500 Subject: [PATCH 24/29] Add Docker production config #2 --- Dockerfile.prod | 36 ++++++++++++++++++++++++++++++++++++ apache/000-scorpio.conf | 20 ++++++++++++++++++++ apache/wsgi.load | 1 + 3 files changed, 57 insertions(+) create mode 100644 Dockerfile.prod create mode 100644 apache/000-scorpio.conf create mode 100644 apache/wsgi.load diff --git a/Dockerfile.prod b/Dockerfile.prod new file mode 100644 index 0000000..7f405a8 --- /dev/null +++ b/Dockerfile.prod @@ -0,0 +1,36 @@ +FROM python:3.10 + +ENV PYTHONUNBUFFERED 1 +ARG SCORPIO_DNS +ARG APPLICATION_PORT +RUN apt-get update +RUN apt-get install --yes apache2 apache2-dev +RUN apt-get install --yes postgresql +RUN apt-get -y install python3-pip +RUN pip install --upgrade pip + +RUN wget https://github.com/GrahamDumpleton/mod_wsgi/archive/refs/tags/4.9.0.tar.gz \ + && tar xvfz 4.9.0.tar.gz \ + && cd mod_wsgi-4.9.0 \ + && ./configure --with-apxs=/usr/bin/apxs --with-python=/usr/local/bin/python \ + && make \ + && make install \ + && make clean + +ADD ./apache/000-scorpio.conf /etc/apache2/sites-available/000-scorpio.conf +RUN sed "s/ENV_SCORPIO_DNS/${SCORPIO_DNS}/" -i /etc/apache2/sites-available/000-scorpio.conf +RUN sed "s/ENV_SCORPIO_PORT/${APPLICATION_PORT}/" -i /etc/apache2/sites-available/000-scorpio.conf +ADD ./apache/wsgi.load /etc/apache2/mods-available/wsgi.load +RUN a2dissite 000-default +RUN a2ensite 000-scorpio.conf +RUN a2enmod headers +RUN a2enmod rewrite +RUN a2enmod wsgi + +COPY . /var/www/scorpio +WORKDIR /var/www/scorpio +RUN pip install -r requirements.txt +RUN ./manage.py collectstatic + +EXPOSE ${APPLICATION_PORT} +CMD ["apache2ctl", "-D", "FOREGROUND"] diff --git a/apache/000-scorpio.conf b/apache/000-scorpio.conf new file mode 100644 index 0000000..e0eeceb --- /dev/null +++ b/apache/000-scorpio.conf @@ -0,0 +1,20 @@ +Listen ENV_SCORPIO_PORT + + ErrorLog /var/log/apache2/scorpio_error_log + CustomLog /var/log/apache2/scorpio_access_log combined + ServerName ENV_SCORPIO_DNS + DocumentRoot /var/www/html + Alias /static /var/www/scorpio/static + + Options Includes FollowSymLinks MultiViews + Require all granted + + + WSGIProcessGroup scorpio + WSGIApplicationGroup %{GLOBAL} + Require all granted + + WSGIDaemonProcess scorpio home=/var/www/scorpio + WSGIProcessGroup scorpio + WSGIScriptAlias / /var/www/scorpio/scorpio/wsgi.py + diff --git a/apache/wsgi.load b/apache/wsgi.load new file mode 100644 index 0000000..d76d1d7 --- /dev/null +++ b/apache/wsgi.load @@ -0,0 +1 @@ +LoadModule wsgi_module /usr/lib/apache2/modules/mod_wsgi.so From a553f1108c99849deabde531c3e7947da628f93a Mon Sep 17 00:00:00 2001 From: HaSistrunk Date: Wed, 1 Feb 2023 00:21:06 +0000 Subject: [PATCH 25/29] Dependency updates --- requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index e841b37..87a35ca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ certifi==2022.12.7 # via # elasticsearch # requests -charset-normalizer==2.1.1 +charset-normalizer==3.0.1 # via requests django==4.0.8 # via @@ -54,11 +54,11 @@ psycopg2-binary==2.9.5 # asterism python-dateutil==2.8.2 # via elasticsearch-dsl -pytz==2022.7 +pytz==2022.7.1 # via djangorestframework rac-es==1.0.0 # via -r requirements.in -requests==2.28.1 +requests==2.28.2 # via # -r requirements.in # electronbonder @@ -70,7 +70,7 @@ six==1.16.0 # python-dateutil sqlparse==0.4.3 # via django -urllib3==1.26.13 +urllib3==1.26.14 # via # elasticsearch # requests From 1621556a38f1046a851ee3ace7f543ed04297187 Mon Sep 17 00:00:00 2001 From: Hillel Arnold Date: Wed, 1 Feb 2023 13:46:56 -0500 Subject: [PATCH 26/29] Update dependencies.yml --- .github/workflows/dependencies.yml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index 925762c..0d2cfc0 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -15,13 +15,21 @@ jobs: with: ref: development - - uses: actions/setup-python@v4 + - name: Set up Python + uses: actions/setup-python@v4 with: python-version: '3.10' cache: pip - - run: | - pip install pip-tools - pip-compile --upgrade + + - name: Install pre-commit and pip-tools + run: pip install pre-commit pip-tools + + - name: Run pre-commit autoupdate + run: pre-commit autoupdate + + - name: Run pip-compile + run: pip-compile --upgrade + - name: Create Pull Request uses: peter-evans/create-pull-request@v4 with: @@ -30,4 +38,4 @@ jobs: branch: dependency-updates delete-branch: true title: Dependency Updates - commit-message: Dependency updates \ No newline at end of file + commit-message: Dependency updates From 4fce04693c18d4cc0bb11f6334ad28a1228c1083 Mon Sep 17 00:00:00 2001 From: Hillel Arnold Date: Mon, 6 Feb 2023 10:07:18 -0500 Subject: [PATCH 27/29] bump django --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 87a35ca..7067534 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: +# This file is autogenerated by pip-compile with python 3.10 +# To update, run: # # pip-compile # From da53bb82a2df621f230b753d137110365665ee99 Mon Sep 17 00:00:00 2001 From: Hillel Arnold Date: Mon, 6 Feb 2023 10:36:38 -0500 Subject: [PATCH 28/29] updating --- requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7067534..5a027eb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.10 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: # # pip-compile # @@ -16,7 +16,7 @@ certifi==2022.12.7 # requests charset-normalizer==3.0.1 # via requests -django==4.0.8 +django==4.0.9 # via # -r requirements.in # asterism @@ -31,7 +31,7 @@ djangorestframework==3.14.0 # via # -r requirements.in # asterism -elasticsearch==7.17.8 +elasticsearch==7.17.9 # via # -r requirements.in # elasticsearch-dsl From 7835471a91fd2add0a1f256c984cec2a09627f3f Mon Sep 17 00:00:00 2001 From: Hillel Arnold Date: Wed, 15 Feb 2023 22:55:54 -0500 Subject: [PATCH 29/29] update django --- docker-compose.yml | 6 +++++- requirements.in | 2 +- requirements.txt | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4042f57..a8bdbb7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,11 @@ version: '3' services: scorpio-db: - image: postgres:10.4 + image: postgres:14.4 + environment: + - POSTGRES_USER=postgres # name of the application database user + - POSTGRES_PASSWORD=postgres # name of the application database user + - POSTGRES_DB=postgres # name of the application database volumes: - scorpiodbvolume:/var/lib/postgresql/data/ networks: diff --git a/requirements.in b/requirements.in index 3267fc7..a7bf819 100644 --- a/requirements.in +++ b/requirements.in @@ -1,5 +1,5 @@ asterism~=0.9 -Django~=4.0.7 +Django~=4.1 django4-cron~=0.5 djangorestframework~=3.13 elasticsearch~=7.17 diff --git a/requirements.txt b/requirements.txt index 5a027eb..b621bcf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ certifi==2022.12.7 # requests charset-normalizer==3.0.1 # via requests -django==4.0.9 +django==4.1.7 # via # -r requirements.in # asterism