wip: v2 metadata URL (#699)

* wip: v2 metadata URL

* wip: rework, update deps

* wip: fix failing tests

* fix: failing tests

* fix: workflow to install python deps from poetry

* fix: python version + missing python packages

* fix: install dev dependencies as well in the ci

* fix: schema for DetailedScoreResponse

* feat: have updated Dockerfile to use poetry for python libs

* feat: removing unused dependencies

* feat: adding lambda handler for get score request in v2

* feat: fix test workflow

* fix: dockerfile for building lambda image

* fix: updating python version in workflow and Dockerfile

* fix: Dockerfile for lambdas

* fix: handler for lambda getting score

* feat: add tests for get_score handler

* fix: failing tests

* feat: adding the infra files for v2 of the api

* fix: failing tests, but historical endpoint for v2 is still wip (and commented out for now)

---------

Co-authored-by: Gerald Iakobinyi-Pich <[email protected]>

.github/workflows/ci.yml

@@ -41,14 +41,17 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.12"
 
-      - name: Install pipenv
-        run: pip3 install pipenv
+      - name: Install poetry
+        run: |
+          pip3 install pipx
+          pipx ensurepath
+          pipx install poetry
 
       - name: Generate requirements.txt
         working-directory: ./api
-        run: pipenv requirements --dev > requirements.txt
+        run: poetry export -f requirements.txt -o requirements.txt --with dev
 
       - name: Install API dependencies
         working-directory: ./api

.github/workflows/test_generic.yml

@@ -54,14 +54,17 @@ jobs:
           fetch-depth: 0
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.12"
 
-      - name: Install pipenv
-        run: pip3 install pipenv
+      - name: Install poetry
+        run: |
+          pip3 install pipx
+          pipx ensurepath
+          pipx install poetry
 
       - name: Generate requirements.txt
         working-directory: ./api
-        run: pipenv requirements --dev > requirements.txt
+        run: poetry export -f requirements.txt -o requirements.txt --with dev
 
       - name: Install API dependencies
         working-directory: ./api

api/Dockerfile

@@ -1,5 +1,5 @@
 # https://pipenv.pypa.io/en/latest/basics/#pipenv-and-docker-containers
-FROM --platform=linux/amd64 docker.io/python:3.11 AS base
+FROM --platform=linux/amd64 docker.io/python:3.12 AS base
 
 RUN apt update && \
     apt install -y python3-dev libpq-dev
@@ -11,12 +11,15 @@ RUN apt update && \
 FROM base AS builder
 
 RUN  pip install --upgrade pip && \
-    pip install --user pipenv
+    pip install --user pipx && \
+    /root/.local/bin/pipx ensurepath && \
+    /root/.local/bin/pipx install poetry
 
 # Tell pipenv to create venv in the current directory
-ENV PIPENV_VENV_IN_PROJECT=1
+ENV POETRY_VIRTUALENVS_IN_PROJECT=1
 
-ADD Pipfile.lock /usr/src/
+ADD poetry.lock /usr/src/
+ADD pyproject.toml /usr/src/
 
 WORKDIR /usr/src
 
@@ -26,10 +29,7 @@ WORKDIR /usr/src
 # you need to have pycurl build dependencies libcurl4-gnutls-dev and libcurl3-gnutls
 # In the runtime container you need only libcurl3-gnutls
 
-# RUN apt install -y libcurl3-gnutls libcurl4-gnutls-dev
-# RUN /root/.local/bin/pipenv lock
-
-RUN /root/.local/bin/pipenv sync
+RUN /root/.local/bin/poetry install
 RUN /usr/src/.venv/bin/python -c "import django; print(django.__version__)"
 
 

api/Pipfile

@@ -22,7 +22,6 @@ web3 = "*"
 siwe = "*"
 psycopg2-binary = "*"
 djangorestframework-api-key = "==2.*"
-celery = {extras = ["redis"], version = "*"}
 requests = "*"
 social-auth-app-django = "*"
 dag-cbor = "*"
@@ -36,17 +35,16 @@ django-debug-toolbar = "*"
 django-filter = "*"
 boto3 = "*"
 tqdm = "*"
-gql = "*"
 requests-toolbelt = "*"
 pyarrow = "*"
 pynacl = "*"
-faker = "*"
 python-jose = "*"
 uvarint = "*"
 django-ace = "*"
 async-timeout = "*"
 aiohttp = "*"
-freezegun = "*"
+poetry = "*"
+pipenv-poetry-migrate = "*"
 
 [dev-packages]
 black = "*"
@@ -59,6 +57,8 @@ pytest-bdd = "*"
 pytest-mock = "*"
 django-stubs = "*"
 pandas = "*"
+faker = "*"
+freezegun = "*"
 
 [requires]
-python_version = "3.11"
+python_version = "3.12"

api/account/api.py

@@ -197,7 +197,7 @@ def submit_signed_challenge(request, payload: SiweVerifySubmit):
         raise InvalidNonceException()
 
     try:
-        message: SiweMessage = SiweMessage(payload.message)
+        message: SiweMessage = SiweMessage(**payload.message)
         verifyParams = {
             "signature": payload.signature,
             # See note in /nonce function above

api/account/test/test_account.py

@@ -3,14 +3,15 @@
 from copy import deepcopy
 from datetime import datetime
 
-from account.models import Account
 from django.conf import settings
 from django.test import Client, TestCase
 from eth_account.messages import encode_defunct
 from siwe import SiweMessage
 from web3 import Web3
 from web3.auto import w3
 
+from account.models import Account
+
 my_mnemonic = settings.TEST_MNEMONIC
 
 # Create your tests here.
@@ -59,7 +60,7 @@ def test_nonce(self):
 
     def test_create_account_with_SIWE(self):
         """Test creation of an account wit SIWE"""
-        siwe = SiweMessage(self.siwe_data_pay)
+        siwe = SiweMessage(**self.siwe_data_pay)
         data_to_sign = siwe.prepare_message()
 
         private_key = account.key
@@ -86,7 +87,7 @@ def test_create_account_with_SIWE(self):
         self.assertTrue("access" in data)
 
     def test_account_address_is_lower_case(self):
-        siwe = SiweMessage(self.siwe_data_pay)
+        siwe = SiweMessage(**self.siwe_data_pay)
         data_to_sign = siwe.prepare_message()
 
         private_key = account.key
@@ -111,7 +112,7 @@ def test_account_address_is_lower_case(self):
         self.assertEqual(created_account.address, account.address.lower())
 
     def test_create_account_with_www_domain(self):
-        siwe = SiweMessage(self.siwe_data_pay)
+        siwe = SiweMessage(**self.siwe_data_pay)
         data_to_sign = siwe.prepare_message()
         data_to_sign = siwe.prepare_message()
 

api/aws_lambdas/scorer_api_passport/tests/test_bulk_stamp_lambdas.py

@@ -1,4 +1,5 @@
 import json
+from decimal import Decimal
 
 import pytest
 from django.conf import settings

api/aws_lambdas/scorer_api_passport/tests/test_stamp_lambdas.py

@@ -1,14 +1,15 @@
 import json
 
 import pytest
+from django.conf import settings
+
 from aws_lambdas.scorer_api_passport.v1 import (
     score_GET,
     score_POST,
     stamp_GET,
     weights_GET,
 )
 from ceramic_cache.models import CeramicCache
-from django.conf import settings
 from registry.models import Passport, Score
 from registry.utils import get_utc_time
 
@@ -33,7 +34,7 @@ def test_score_get(
         status=Score.Status.DONE,
         last_score_timestamp=get_utc_time(),
         error=None,
-        stamp_scores=[],
+        stamp_scores={},
         evidence={
             "rawScore": 10,
             "type": "binary",

api/aws_lambdas/submit_passport/Dockerfile

@@ -1,6 +1,6 @@
 
 # https://pipenv.pypa.io/en/latest/basics/#pipenv-and-docker-containers
-FROM --platform=linux/amd64 public.ecr.aws/lambda/python:3.11 AS base
+FROM --platform=linux/amd64 public.ecr.aws/lambda/python:3.12 AS base
 
 
 #########################################################
@@ -9,16 +9,19 @@ FROM --platform=linux/amd64 public.ecr.aws/lambda/python:3.11 AS base
 FROM base AS builder
 
 RUN  pip install --upgrade pip && \
-    pip install pipenv
+    pip install --user pipx && \
+    /root/.local/bin/pipx ensurepath && \
+    /root/.local/bin/pipx install poetry
 
 # Tell pipenv to create venv in the current directory
-ENV PIPENV_VENV_IN_PROJECT=1
+ENV POETRY_VIRTUALENVS_IN_PROJECT=1
 
-ADD Pipfile.lock /usr/src/
+ADD poetry.lock /usr/src/
+ADD pyproject.toml /usr/src/
 
 WORKDIR /usr/src
 
-RUN pipenv requirements > requirements.txt
+RUN /root/.local/bin/poetry export -f requirements.txt -o requirements.txt
 
 #########################################################
 # Runtime

api/aws_lambdas/submit_passport/submit_passport.py

@@ -2,18 +2,21 @@
 This module provides a handler to manage API requests in AWS Lambda.
 """
 
+import logging
+
 from asgiref.sync import async_to_sync
+from django.db import close_old_connections
+
+from aws_lambdas.exceptions import InvalidRequest
 from aws_lambdas.utils import (
     with_api_request_exception_handling,
 )
 from registry.api.v1 import (
     SubmitPassportPayload,
     ahandle_submit_passport,
 )
-from django.db import close_old_connections
 
-# Now this script or any imported module can use any part of Django it needs.
-# from myapp import models
+logger = logging.getLogger(__name__)
 
 
 @with_api_request_exception_handling
@@ -32,3 +35,38 @@ def _handler(event, _context, request, user_account, body):
 def handler(*args, **kwargs):
     close_old_connections()
     return _handler(*args, **kwargs)
+
+
+@with_api_request_exception_handling
+def _handler_get_score(event, _context, request, user_account, body):
+    path = event.get("path", "").split("/")
+
+    if len(path) != 6:
+        raise InvalidRequest(f"Invalid path: '{path}'")
+
+    scorer_id = path[3]
+    address = path[5]
+
+    logger.info(f"path: '%s', scorer_id: '%s', address: %s", path, scorer_id, address)
+
+    # unquote(event.get("queryStringParameters", {}).get("model_list", ""))
+
+    score = async_to_sync(ahandle_submit_passport)(
+        SubmitPassportPayload(
+            scorer_id=scorer_id,
+            address=address,
+        ),
+        user_account,
+    )
+
+    return score
+
+
+def handler_get_score(*args, **kwargs):
+    """
+    This handles the new GET request for the score, that also implies doing the same score
+    calculation as on the previous POST `submit-passport` request
+    """
+
+    close_old_connections()
+    return _handler_get_score(*args, **kwargs)