From ad27cac52761a52f100cab1fcd0722ceb2e03b86 Mon Sep 17 00:00:00 2001 From: Rob Knop Date: Mon, 15 Jul 2024 09:35:55 -0700 Subject: [PATCH] Update rkwebutil --- conductor/Makefile | 2 +- conductor/db.py | 8 --- conductor/flaskauth.py | 1 - conductor/rkauth_flask.py | 1 + conductor/webservice.py | 29 ++++++--- docker/application/Dockerfile | 2 +- docs/testing.md | 4 +- models/user.py | 60 +------------------ tests/docker-compose.yaml | 18 +++--- .../test_pipeline_exposure_launcher.py | 2 +- webap/rkwebutil | 2 +- 11 files changed, 38 insertions(+), 91 deletions(-) delete mode 100644 conductor/db.py delete mode 120000 conductor/flaskauth.py create mode 120000 conductor/rkauth_flask.py diff --git a/conductor/Makefile b/conductor/Makefile index d4d9ae8e..78e557b8 100644 --- a/conductor/Makefile +++ b/conductor/Makefile @@ -1,6 +1,6 @@ INSTALLDIR = test_install -toinstall = webservice.py db.py flaskauth.py updater.py run_conductor.sh \ +toinstall = webservice.py rkauth_flask.py updater.py run_conductor.sh \ static/conductor.js static/conductor_start.js static/rkwebutil.js \ static/resetpasswd_start.js static/rkauth.js static/seechange.css \ templates/base.html templates/conductor_root.html diff --git a/conductor/db.py b/conductor/db.py deleted file mode 100644 index 4c68571e..00000000 --- a/conductor/db.py +++ /dev/null @@ -1,8 +0,0 @@ -from models.base import SmartSession -import models.user - -AuthUser = models.user.AuthUser -PasswordLink = models.user.PasswordLink - -def DBSession( *args, **kwargs ): - return SmartSession( *args, **kwargs ) diff --git a/conductor/flaskauth.py b/conductor/flaskauth.py deleted file mode 120000 index 56dc4576..00000000 --- a/conductor/flaskauth.py +++ /dev/null @@ -1 +0,0 @@ -../webap/rkwebutil/flaskauth.py \ No newline at end of file diff --git a/conductor/rkauth_flask.py b/conductor/rkauth_flask.py new file mode 120000 index 00000000..3d415a79 --- /dev/null +++ b/conductor/rkauth_flask.py @@ -0,0 +1 @@ +../webap/rkwebutil/rkauth_flask.py \ No newline at end of file diff --git a/conductor/webservice.py b/conductor/webservice.py index 4899067b..e3a16f51 100644 --- a/conductor/webservice.py +++ b/conductor/webservice.py @@ -387,16 +387,29 @@ def do_the_things( self ): # Import and configure the auth subapp sys.path.insert( 0, pathlib.Path(__name__).parent ) -import flaskauth +import rkauth_flask + +kwargs = { + 'db_host': cfg.value( 'db.host' ), + 'db_port': cfg.value( 'db.port' ), + 'db_name': cfg.value( 'db.database' ), + 'db_user': cfg.value( 'db.user' ) +} +password = cfg.value( 'db.password' ) +if password is None: + if cfg.value( 'db.password_file' ) is None: + raise RuntimeError( 'In config, one of db.password or db.password_file must be specified' ) + with open( cfg.value( 'db.password_file' ) ) as ifp: + password = ifp.readline().strip() +kwargs[ 'db_password' ] = password + for attr in [ 'email_from', 'email_subject', 'email_system_name', 'smtp_server', 'smtp_port', 'smtp_use_ssl', 'smtp_username', 'smtp_password' ]: - setattr( flaskauth.RKAuthConfig, attr, cfg.value( f'conductor.{attr}' ) ) -flaskauth.RKAuthConfig.webap_url = cfg.value('conductor.conductor_url') -if flaskauth.RKAuthConfig.webap_url[-1] != '/': - flaskauth.RKAuthConfig.webap_url += '/' -flaskauth.RKAuthConfig.webap_url += "auth" -# app.logger.debug( f'webap_url is {flaskauth.RKAuthConfig.webap_url}' ) -app.register_blueprint( flaskauth.bp ) + kwargs[ attr ] = cfg.value( f'conductor.{attr}' ) + +rkauth_flask.RKAuthConfig.setdbparams( **kwargs ) + +app.register_blueprint( rkauth_flask.bp ) # Configure urls diff --git a/docker/application/Dockerfile b/docker/application/Dockerfile index 54c6bdd6..cd93767b 100755 --- a/docker/application/Dockerfile +++ b/docker/application/Dockerfile @@ -285,7 +285,7 @@ ADD conductor/ /usr/src/seechange/conductor/ # Need special handling of symlinks, because the # ADD above copied the links as is, but they point # to something that won't exist at the destination. -ADD conductor/flaskauth.py /usr/src/seechange/conductor/flaskauth.py +ADD conductor/rkauth_flask.py /usr/src/seechange/conductor/rkauth_flask.py ADD conductor/static/rkwebutil.js /usr/src/seechange/conductor/static/rkwebutil.js ADD conductor/static/rkauth.js /usr/src/seechange/conductor/static/rkauth.js ADD conductor/static/seechange.css /usr/src/seechange/conductor/static/seechange.css diff --git a/docs/testing.md b/docs/testing.md index 4e1583b5..bd4a1a81 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -63,7 +63,7 @@ Normally, all of these image should already be present in the `ghcr.io` archive, To build and push the docker images, in the `tests` subdirectory run: ``` - IMTGAG=[tag] docker compose build + IMGTAG=[tag] docker compose build ``` where [tag] is exactly what you see in the `docker-compose.yaml` file, followed by: ``` @@ -84,4 +84,4 @@ This set of docker images depend on the following files: * `webap/*` * `requirements.text` -If you change any of those files, you will need to build and push new docker images. Before doing that, edit `tests/docker-compose.yaml` and bump the date part of the tag for _every_ image, so that your changed images will only get used for your branch while you're still finalizing your pull request, and so that the updated images will get used by everybody else once your branch has been merged to main. +If you change any of those files, you will need to build and push new docker images. Before doing that, edit `tests/docker-compose.yaml` and bump the date part of the tag for _every_ image (search and replace is your friend), so that your changed images will only get used for your branch while you're still finalizing your pull request, and so that the updated images will get used by everybody else once your branch has been merged to main. diff --git a/models/user.py b/models/user.py index e0050941..48857197 100644 --- a/models/user.py +++ b/models/user.py @@ -1,14 +1,8 @@ -from datetime import datetime, timedelta -import dateutil.parser -import pytz import uuid - import sqlalchemy as sa from sqlalchemy.dialects.postgresql import UUID as sqlUUID from sqlalchemy.dialects.postgresql import JSONB -from models.base import Base, SmartSession -from util.util import as_UUID, as_datetime - +from models.base import Base class AuthUser(Base): __tablename__ = "authuser" @@ -20,61 +14,9 @@ class AuthUser(Base): pubkey = sa.Column( sa.Text ) privkey = sa.Column( JSONB ) - # This is here rather than just using things already defined in base - # because this table is designed to work with the pre-existing auth - # library in rkwebutil. - @classmethod - def get( cls, id, session=None ): - id = id if isinstance( id, uuid.UUID) else uuid.UUID( id ) - with SmartSession(session) as sess: - q = sess.query(cls).filter( cls.id==id ) - if q.count() > 1: - raise ErrorMsg( f'Error, {cls.__name__} {id} multiply defined! This shouldn\'t happen.' ) - if q.count() == 0: - return None - return q[0] - - @classmethod - def getbyusername( cls, name, session=None ): - with SmartSession(session) as sess: - q = sess.query(cls).filter( cls.username==name ) - return q.all() - - @classmethod - def getbyemail( cls, email, session=None ): - with SmartSession(session) as sess: - q = sess.query(cls).filter( cls.email==email ) - return q.all() - - class PasswordLink(Base): __tablename__ = "passwordlink" id = sa.Column( sqlUUID(as_uuid=True), primary_key=True, default=uuid.uuid4 ) userid = sa.Column( sqlUUID(as_uuid=True), sa.ForeignKey("authuser.id", ondelete="CASCADE"), index=True ) expires = sa.Column( sa.DateTime(timezone=True) ) - - @classmethod - def new( cls, userid, expires=None, session=None ): - if expires is None: - expires = datetime.now(pytz.utc) + timedelta(hours=1) - else: - expires = as_datetime( expires ) - with SmartSession(session) as sess: - link = PasswordLink( userid = as_UUID(userid), - expires = expires ) - sess.add( link ) - sess.commit() - return link - - # This is here rather than just using things already defined in base - # because this table is designed to work with the pre-existing auth - # library in rkwebutil. - @classmethod - def get( cls, uuid, session=None ): - with SmartSession(session) as sess: - q = sess.query( PasswordLink ).filter( PasswordLink.id==uuid ) - if q.count() == 0: - return None - else: - return q.first() diff --git a/tests/docker-compose.yaml b/tests/docker-compose.yaml index d191980c..d27ce198 100644 --- a/tests/docker-compose.yaml +++ b/tests/docker-compose.yaml @@ -1,6 +1,6 @@ services: make-archive-directories: - image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-c3-time-domain}/upload-connector:${IMGTAG:-tests20240628} + image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-c3-time-domain}/upload-connector:${IMGTAG:-test20240715} build: context: ../extern/nersc-upload-connector args: @@ -20,7 +20,7 @@ services: depends_on: make-archive-directories: condition: service_completed_successfully - image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-c3-time-domain}/upload-connector:${IMGTAG:-tests20240628} + image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-c3-time-domain}/upload-connector:${IMGTAG:-test20240715} build: context: ../extern/nersc-upload-connector args: @@ -47,7 +47,7 @@ services: user: ${USERID:-0}:${GROUPID:-0} postgres: - image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-c3-time-domain}/postgres:${IMGTAG:-tests20240628} + image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-c3-time-domain}/postgres:${IMGTAG:-test20240715} build: context: ../docker/postgres environment: @@ -60,7 +60,7 @@ services: retries: 5 setuptables: - image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-c3-time-domain}/seechange:${IMGTAG:-tests20240628} + image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-c3-time-domain}/seechange:${IMGTAG:-test20240715} build: context: ../ dockerfile: ./docker/application/Dockerfile @@ -84,7 +84,7 @@ services: - "${MAILHOG_PORT:-8025}:8025" conductor: - image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-c3-time-domain}/conductor:${IMGTAG:-tests20240628} + image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-c3-time-domain}/conductor:${IMGTAG:-test20240715} build: context: ../ dockerfile: ./docker/application/Dockerfile @@ -114,7 +114,7 @@ services: condition: service_completed_successfully make-archive-directories: condition: service_completed_successfully - image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-c3-time-domain}/seechange-webap:${IMGTAG:-tests20240628} + image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-c3-time-domain}/seechange-webap:${IMGTAG:-test20240715} build: context: ../webap user: ${USERID:-0}:${GROUPID:-0} @@ -135,7 +135,7 @@ services: entrypoint: [ "gunicorn", "-w", "4", "-b", "0.0.0.0:8081", "--timeout", "0", "seechange_webap:app" ] runtests: - image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-c3-time-domain}/seechange:${IMGTAG:-tests20240628} + image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-c3-time-domain}/seechange:${IMGTAG:-test20240715} build: context: ../ dockerfile: ./docker/application/Dockerfile @@ -165,7 +165,7 @@ services: entrypoint: "pytest -v /seechange/$TEST_SUBFOLDER" runalltests: - image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-c3-time-domain}/seechange:${IMGTAG:-tests20240628} + image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-c3-time-domain}/seechange:${IMGTAG:-test20240715} build: context: ../ dockerfile: ./docker/application/Dockerfile @@ -195,7 +195,7 @@ services: entrypoint: "pytest -v /seechange/tests" shell: - image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-c3-time-domain}/seechange:${IMGTAG:-tests20240628} + image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-c3-time-domain}/seechange:${IMGTAG:-test20240715} build: context: ../ dockerfile: ./docker/application/Dockerfile diff --git a/tests/pipeline/test_pipeline_exposure_launcher.py b/tests/pipeline/test_pipeline_exposure_launcher.py index 90363408..78ef9c37 100644 --- a/tests/pipeline/test_pipeline_exposure_launcher.py +++ b/tests/pipeline/test_pipeline_exposure_launcher.py @@ -85,7 +85,7 @@ def test_exposure_launcher( conductor_connector, measq = session.query( Measurements ).join( Cutouts ).join( SourceList ).join( Image ) meas0 = measq.filter( Image.id==sub0.id ).all() meas1 = measq.filter( Image.id==sub1.id ).all() - assert len(meas0) == 3 + assert len(meas0) == 2 assert len(meas1) == 6 finally: diff --git a/webap/rkwebutil b/webap/rkwebutil index 809a6163..ab245392 160000 --- a/webap/rkwebutil +++ b/webap/rkwebutil @@ -1 +1 @@ -Subproject commit 809a61636e5f70baefc6af4ef2a0e9de6760a897 +Subproject commit ab245392e4bfaafc5c7bf42a162e35707088c520