From 5326c5c4c01281250d98610cc978b30e68ee9425 Mon Sep 17 00:00:00 2001 From: Wes Okes Date: Mon, 15 Mar 2021 22:20:15 -0400 Subject: [PATCH 01/11] not optimal but better than the old one --- entity_event/models.py | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/entity_event/models.py b/entity_event/models.py index 7b29518..bb842a8 100644 --- a/entity_event/models.py +++ b/entity_event/models.py @@ -6,7 +6,7 @@ from cached_property import cached_property from django.contrib.postgres.fields import JSONField from django.core.serializers.json import DjangoJSONEncoder -from django.db import models, transaction +from django.db import models, transaction, connection from django.db.models import Q from django.db.models.query import QuerySet from django.template.loader import render_to_string @@ -480,7 +480,8 @@ def get_filtered_events_queries(self, start_time, end_time, seen, include_expire # If we only want unseen events exclude events that have been seen for this medium elif seen is False: - filters.append(~Q(eventseen__medium=self)) + unseen_ids = _unseen_event_ids(medium=self) + filters.append(Q(id__in=unseen_ids)) # Filter by actor if actor is not None: @@ -1216,20 +1217,24 @@ def _unseen_event_ids(medium): """ Return all events that have not been seen on this medium. """ - # query = ''' - # SELECT event.id - # FROM entity_event_event AS event - # LEFT OUTER JOIN (SELECT * - # FROM entity_event_eventseen AS seen - # WHERE seen.medium_id=%s) AS eventseen - # ON event.id = eventseen.event_id - # WHERE eventseen.medium_id IS NULL - # ''' - unseen_events = Event.objects.filter( - ~Q(eventseen__medium=medium) - ) - ids = [e.id for e in unseen_events] - return ids + query = ''' + SELECT + event.id + FROM + entity_event_event AS event + LEFT JOIN ( + SELECT seen.event_id, seen.medium_id + FROM entity_event_eventseen AS seen + WHERE seen.medium_id=%s + ) AS eventseen + ON event.id = eventseen.event_id + WHERE eventseen.medium_id IS NULL + ''' + with connection.cursor() as cursor: + cursor.execute(query, [medium.id]) + rows = cursor.fetchall() + + return [row[0] for row in rows] class RenderingStyle(models.Model): From ccaf46776a5d99e21f24ecb37671d910972c6b34 Mon Sep 17 00:00:00 2001 From: Wes Okes Date: Tue, 24 Jan 2023 15:40:16 -0500 Subject: [PATCH 02/11] github actions --- .github/workflows/tests.yml | 84 +++++++++++++++++++++++++++++++++++++ docs/release_notes.rst | 6 +++ entity_event/version.py | 2 +- settings.py | 27 ++++++------ setup.py | 5 ++- 5 files changed, 110 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..947bf8e --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,84 @@ +# copied from django-cte +name: entity_event tests +on: + push: + branches: [master] + pull_request: + branches: [master,develop] + +jobs: + tests: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python: ['3.7'] + # Time to switch to pytest or nose2?? + # nosetests is broken on 3.10 + # AttributeError: module 'collections' has no attribute 'Callable' + # https://github.com/nose-devs/nose/issues/1099 + django: + - 'Django~=2.2.0' +# - 'Django~=3.0.0' +# - 'Django~=3.1.0' +# - 'Django~=3.2.0' +# - 'Django~=4.0.0' +# - 'Django~=4.1.0' + experimental: [false] +# include: +# - python: '3.9' +# django: 'https://github.com/django/django/archive/refs/heads/main.zip#egg=Django' +# experimental: true +# # NOTE this job will appear to pass even when it fails because of +# # `continue-on-error: true`. Github Actions apparently does not +# # have this feature, similar to Travis' allow-failure, yet. +# # https://github.com/actions/toolkit/issues/399 + exclude: + - python: '3.7' + django: 'Django~=4.0.0' + - python: '3.7' + django: 'Django~=4.1.0' + services: + postgres: + image: postgres:latest + env: + POSTGRES_DB: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_USER: postgres + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + - name: Setup + run: | + python --version + pip install --upgrade pip wheel + pip install -r requirements/requirements.txt + pip install -r requirements/requirements-testing.txt + pip install "${{ matrix.django }}" + pip freeze + - name: Run tests + env: + DB_SETTINGS: >- + { + "ENGINE":"django.db.backends.postgresql_psycopg2", + "NAME":"entity_event", + "USER":"postgres", + "PASSWORD":"postgres", + "HOST":"localhost", + "PORT":"5432" + } + run: | + coverage run manage.py test entity_event + coverage report --fail-under=99 + continue-on-error: ${{ matrix.experimental }} + - name: Check style + run: flake8 entity_event diff --git a/docs/release_notes.rst b/docs/release_notes.rst index d4ffd16..0a2192f 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -1,6 +1,12 @@ Release Notes ============= +v3.0.0 +------ +* drop python 3.6 +* support python 3.8, 3.9 +* django 3.2, 4.0, 4.1 + v2.2.1 ------ * Fix bad queryset check which would cause it to evaluate. Add explicit None check. diff --git a/entity_event/version.py b/entity_event/version.py index 36a511e..4eb28e3 100644 --- a/entity_event/version.py +++ b/entity_event/version.py @@ -1 +1 @@ -__version__ = '2.2.1' +__version__ = '3.0.0' diff --git a/settings.py b/settings.py index 14f1664..2e28e4d 100644 --- a/settings.py +++ b/settings.py @@ -1,4 +1,5 @@ import os +import json from django.conf import settings @@ -12,28 +13,30 @@ def configure_settings(): test_db = os.environ.get('DB', None) if test_db is None: db_config = { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'NAME': 'ambition', - 'USER': 'ambition', - 'PASSWORD': 'ambition', - 'HOST': 'db' + 'ENGINE': 'django.db.backends.postgresql', + 'NAME': 'entity', + 'USER': 'postgres', + 'PASSWORD': '', + 'HOST': 'db', } elif test_db == 'postgres': db_config = { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'ENGINE': 'django.db.backends.postgresql', + 'NAME': 'entity', 'USER': 'postgres', - 'NAME': 'entity_event', - } - elif test_db == 'sqlite': - db_config = { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': 'entity_event', + 'PASSWORD': '', + 'HOST': 'db', } else: raise RuntimeError('Unsupported test DB {0}'.format(test_db)) + # Check env for db override (used for github actions) + if os.environ.get('DB_SETTINGS'): + db_config = json.loads(os.environ.get('DB_SETTINGS')) + settings.configure( TEST_RUNNER='django_nose.NoseTestSuiteRunner', + SECRET_KEY='*', NOSE_ARGS=['--nocapture', '--nologcapture', '--verbosity=1'], MIDDLEWARE_CLASSES=(), DATABASES={ diff --git a/setup.py b/setup.py index 7b4dedb..2428484 100755 --- a/setup.py +++ b/setup.py @@ -38,9 +38,9 @@ def get_lines(file_path): packages=find_packages(), classifiers=[ 'Programming Language :: Python', - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', 'Operating System :: OS Independent', @@ -48,6 +48,9 @@ def get_lines(file_path): 'Framework :: Django :: 2.2', 'Framework :: Django :: 3.0', 'Framework :: Django :: 3.1', + 'Framework :: Django :: 3.2', + 'Framework :: Django :: 4.0', + 'Framework :: Django :: 4.1', ], license='MIT', install_requires=install_requires, From 59f20cdaf617d11d461e20e617bb40996b4a93fb Mon Sep 17 00:00:00 2001 From: Wes Okes Date: Tue, 24 Jan 2023 15:43:01 -0500 Subject: [PATCH 03/11] flake --- entity_event/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entity_event/models.py b/entity_event/models.py index cffad7c..e8180da 100644 --- a/entity_event/models.py +++ b/entity_event/models.py @@ -6,7 +6,7 @@ from cached_property import cached_property from django.contrib.postgres.fields import JSONField from django.core.serializers.json import DjangoJSONEncoder -from django.db import models, transaction, connection +from django.db import models, transaction from django.db.models import Q from django.db.models.query import QuerySet from django.template.loader import render_to_string From 177a5297938dc59f71152a359aca7781753a0a6b Mon Sep 17 00:00:00 2001 From: Wes Okes Date: Tue, 24 Jan 2023 15:43:07 -0500 Subject: [PATCH 04/11] versions --- .github/workflows/tests.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 947bf8e..ead1fd9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,18 +12,18 @@ jobs: strategy: fail-fast: false matrix: - python: ['3.7'] + python: ['3.7', '3.8', '3.9'] # Time to switch to pytest or nose2?? # nosetests is broken on 3.10 # AttributeError: module 'collections' has no attribute 'Callable' # https://github.com/nose-devs/nose/issues/1099 django: - 'Django~=2.2.0' -# - 'Django~=3.0.0' -# - 'Django~=3.1.0' -# - 'Django~=3.2.0' -# - 'Django~=4.0.0' -# - 'Django~=4.1.0' + - 'Django~=3.0.0' + - 'Django~=3.1.0' + - 'Django~=3.2.0' + - 'Django~=4.0.0' + - 'Django~=4.1.0' experimental: [false] # include: # - python: '3.9' From 4fc49d40298d698409dbbddb3ac91ffea9549017 Mon Sep 17 00:00:00 2001 From: Wes Okes Date: Tue, 24 Jan 2023 15:50:04 -0500 Subject: [PATCH 05/11] bump back to 100 coverage --- .github/workflows/tests.yml | 2 +- .travis.yml | 39 ------------------------------------- tox.ini | 31 ----------------------------- 3 files changed, 1 insertion(+), 71 deletions(-) delete mode 100644 .travis.yml delete mode 100644 tox.ini diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ead1fd9..f5e7178 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -78,7 +78,7 @@ jobs: } run: | coverage run manage.py test entity_event - coverage report --fail-under=99 + coverage report --fail-under=100 continue-on-error: ${{ matrix.experimental }} - name: Check style run: flake8 entity_event diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index db541c2..0000000 --- a/.travis.yml +++ /dev/null @@ -1,39 +0,0 @@ -dist: xenial -language: python -sudo: false - -python: - - "3.6" - - "3.7" - -env: - matrix: - - DJANGO=2.2 - - DJANGO=3.0 - - DJANGO=3.1 - - DJANGO=master - -addons: - postgresql: '9.6' - -matrix: - include: - - { python: "3.6", env: TOXENV=flake8 } - - allow_failures: - - env: DJANGO=master - -install: - - pip install tox-travis - -before_script: - - psql -c 'CREATE DATABASE entity_event;' -U postgres - -script: - - tox - -after_success: - coveralls - -notifications: - email: false diff --git a/tox.ini b/tox.ini deleted file mode 100644 index f09dd18..0000000 --- a/tox.ini +++ /dev/null @@ -1,31 +0,0 @@ -[tox] -envlist = - flake8 - py{36,37}-django22 - py{36,37}-django30 - py{36,37}-django31 - py{36,37}-djangomaster - -[testenv] -setenv = - DB = postgres -deps = - django22: Django>=2.2,<3.0 - django30: Django>=3.0,<3.1 - django31: Django>=3.1,<3.2 - djangomaster: https://github.com/django/django/archive/master.tar.gz - -rrequirements/requirements-testing.txt -commands = - coverage run setup.py test - coverage report --fail-under=100 - -[testenv:flake8] -deps = flake8 -commands = flake8 entity_event - -[travis:env] -DJANGO = - 2.2: django22 - 3.0: django30 - 3.1: django31 - master: djangomaster From 7e5b64c2f2f1e25d37e9ca212ccd244e870d870e Mon Sep 17 00:00:00 2001 From: Wes Okes Date: Tue, 24 Jan 2023 15:59:49 -0500 Subject: [PATCH 06/11] checking queries --- .github/workflows/tests.yml | 11 ++++++----- entity_event/tests/context_loader_tests.py | 9 ++++++++- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f5e7178..7ed83eb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,18 +12,19 @@ jobs: strategy: fail-fast: false matrix: - python: ['3.7', '3.8', '3.9'] + python: ['3.7'] +# python: ['3.7', '3.8', '3.9'] # Time to switch to pytest or nose2?? # nosetests is broken on 3.10 # AttributeError: module 'collections' has no attribute 'Callable' # https://github.com/nose-devs/nose/issues/1099 django: - - 'Django~=2.2.0' - - 'Django~=3.0.0' +# - 'Django~=2.2.0' +# - 'Django~=3.0.0' - 'Django~=3.1.0' - 'Django~=3.2.0' - - 'Django~=4.0.0' - - 'Django~=4.1.0' +# - 'Django~=4.0.0' +# - 'Django~=4.1.0' experimental: [false] # include: # - python: '3.9' diff --git a/entity_event/tests/context_loader_tests.py b/entity_event/tests/context_loader_tests.py index 7216f91..4a9addb 100644 --- a/entity_event/tests/context_loader_tests.py +++ b/entity_event/tests/context_loader_tests.py @@ -766,7 +766,14 @@ def test_optimal_queries(self): e3 = G(models.Event, context={'key2': test_fk_m1.id, 'key': test_m1.id}, source=s2) e4 = G(models.Event, context={'key2': test_fk_m2.id}, source=s2) - with self.assertNumQueries(5): + num_queries = 5 + + # It appears that django >= 3.2 only needs 4 queries + # TODO: check version + if False: + num_queries = 4 + + with self.assertNumQueries(6): context_loader.load_contexts_and_renderers([e1, e2, e3, e4], [medium1, medium2]) self.assertEquals(e1.context['key'].fk, fk1) self.assertEquals(e2.context['key'][0].fk, fk1) From c4a464b0099447e175a59e72cecba03d78facb9c Mon Sep 17 00:00:00 2001 From: Wes Okes Date: Tue, 24 Jan 2023 16:05:16 -0500 Subject: [PATCH 07/11] query handling --- entity_event/tests/context_loader_tests.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/entity_event/tests/context_loader_tests.py b/entity_event/tests/context_loader_tests.py index 4a9addb..3742d7c 100644 --- a/entity_event/tests/context_loader_tests.py +++ b/entity_event/tests/context_loader_tests.py @@ -1,3 +1,4 @@ +import django from django.test import TestCase from django.test.utils import override_settings from django_dynamic_fixture import N, G @@ -766,14 +767,15 @@ def test_optimal_queries(self): e3 = G(models.Event, context={'key2': test_fk_m1.id, 'key': test_m1.id}, source=s2) e4 = G(models.Event, context={'key2': test_fk_m2.id}, source=s2) + # It appears that django >= 3.2 only needs 4 queries because it ignores an "Id in (NULL)" query for the source + # group + # SELECT "entity_event_sourcegroup".* FROM "entity_event_sourcegroup" + # WHERE "entity_event_sourcegroup"."id" IN (NULL) num_queries = 5 - - # It appears that django >= 3.2 only needs 4 queries - # TODO: check version - if False: + if (django.VERSION[0] == 3 and django.VERSION[1] >= 2) or django.VERSION[0] >= 4: num_queries = 4 - with self.assertNumQueries(6): + with self.assertNumQueries(num_queries): context_loader.load_contexts_and_renderers([e1, e2, e3, e4], [medium1, medium2]) self.assertEquals(e1.context['key'].fk, fk1) self.assertEquals(e2.context['key'][0].fk, fk1) From 9d8a1b16370ae788d74cc208244fc74cdf0b358c Mon Sep 17 00:00:00 2001 From: Wes Okes Date: Tue, 24 Jan 2023 16:17:51 -0500 Subject: [PATCH 08/11] nocover in test for version check --- entity_event/tests/context_loader_tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/entity_event/tests/context_loader_tests.py b/entity_event/tests/context_loader_tests.py index 3742d7c..c88026f 100644 --- a/entity_event/tests/context_loader_tests.py +++ b/entity_event/tests/context_loader_tests.py @@ -1,4 +1,4 @@ -import django +from django import VERSION from django.test import TestCase from django.test.utils import override_settings from django_dynamic_fixture import N, G @@ -772,7 +772,7 @@ def test_optimal_queries(self): # SELECT "entity_event_sourcegroup".* FROM "entity_event_sourcegroup" # WHERE "entity_event_sourcegroup"."id" IN (NULL) num_queries = 5 - if (django.VERSION[0] == 3 and django.VERSION[1] >= 2) or django.VERSION[0] >= 4: + if (VERSION[0] == 3 and VERSION[1] >= 2) or VERSION[0] >= 4: # pragma: nocover num_queries = 4 with self.assertNumQueries(num_queries): From f1df9fd623906dea9b960eafbf5244f2bef72057 Mon Sep 17 00:00:00 2001 From: Wes Okes Date: Tue, 24 Jan 2023 16:21:48 -0500 Subject: [PATCH 09/11] fix no cover --- entity_event/tests/context_loader_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entity_event/tests/context_loader_tests.py b/entity_event/tests/context_loader_tests.py index c88026f..aedea8a 100644 --- a/entity_event/tests/context_loader_tests.py +++ b/entity_event/tests/context_loader_tests.py @@ -772,7 +772,7 @@ def test_optimal_queries(self): # SELECT "entity_event_sourcegroup".* FROM "entity_event_sourcegroup" # WHERE "entity_event_sourcegroup"."id" IN (NULL) num_queries = 5 - if (VERSION[0] == 3 and VERSION[1] >= 2) or VERSION[0] >= 4: # pragma: nocover + if (VERSION[0] == 3 and VERSION[1] >= 2) or VERSION[0] >= 4: # pragma: no cover num_queries = 4 with self.assertNumQueries(num_queries): From aa44f6167704c8865a6a09f8d5132b06e638c055 Mon Sep 17 00:00:00 2001 From: Wes Okes Date: Tue, 24 Jan 2023 16:35:50 -0500 Subject: [PATCH 10/11] versions --- .github/workflows/tests.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7ed83eb..f5e7178 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,19 +12,18 @@ jobs: strategy: fail-fast: false matrix: - python: ['3.7'] -# python: ['3.7', '3.8', '3.9'] + python: ['3.7', '3.8', '3.9'] # Time to switch to pytest or nose2?? # nosetests is broken on 3.10 # AttributeError: module 'collections' has no attribute 'Callable' # https://github.com/nose-devs/nose/issues/1099 django: -# - 'Django~=2.2.0' -# - 'Django~=3.0.0' + - 'Django~=2.2.0' + - 'Django~=3.0.0' - 'Django~=3.1.0' - 'Django~=3.2.0' -# - 'Django~=4.0.0' -# - 'Django~=4.1.0' + - 'Django~=4.0.0' + - 'Django~=4.1.0' experimental: [false] # include: # - python: '3.9' From 864bb64bf550c9d63e5ef4e97cd5a16ed80e3b29 Mon Sep 17 00:00:00 2001 From: Wes Okes Date: Tue, 24 Jan 2023 16:45:19 -0500 Subject: [PATCH 11/11] unpine testing requirements --- requirements/requirements-testing.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements/requirements-testing.txt b/requirements/requirements-testing.txt index 1dd1fa8..073bc4f 100644 --- a/requirements/requirements-testing.txt +++ b/requirements/requirements-testing.txt @@ -1,7 +1,7 @@ -coverage==4.5.1 +coverage django-dynamic-fixture -django-nose==1.4.5 -flake8==3.5.0 -freezegun==0.3.12 -mock==2.0.0 -psycopg2>=2.7.7 +django-nose +flake8 +freezegun +mock +psycopg2