fix: remove '=' padding in pagination token (#741)

* fix: remove '=' padding in pagination token

* wip: testing v2 api

* fix: handle failing tests and wrong value returned in  in V2 api

* fix: failing tests

* fix: setting FF_V2_API to on before running the get_unmonitored_urls command

* fix: add missing argument 'created_at' to http request for historic endpoint monitor

* feat: adjust response time trhesholds for alarms

* feat: consistent formatting of score & threshold with 5 decimal places

* fix: return 404 message if score does not exist in historical endpoint

* fix: adding missing file

---------

Co-authored-by: Gerald Iakobinyi-Pich <[email protected]>

.github/workflows/ci.yml

@@ -113,6 +113,7 @@ jobs:
           REGISTRY_ADDRESS: ${{ env.REGISTRY_ADDRESS }}
           CERAMIC_CACHE_JWT_TOKEN: ${{ env.CERAMIC_CACHE_JWT_TOKEN }}
           CERAMIC_CACHE_ADDRESS: ${{ env.CERAMIC_CACHE_ADDRESS }}
+          FF_V2_API: on
 
         run:
           python manage.py get_unmonitored_urls --base-url https://api.scorer.gitcoin.co/ --base-url-xyz https://api.passport.xyz/

.github/workflows/test_generic.yml

@@ -110,6 +110,7 @@ jobs:
           REGISTRY_ADDRESS: ${{ env.REGISTRY_ADDRESS }}
           CERAMIC_CACHE_JWT_TOKEN: ${{ env.CERAMIC_CACHE_JWT_TOKEN }}
           CERAMIC_CACHE_ADDRESS: ${{ env.CERAMIC_CACHE_ADDRESS }}
+          FF_V2_API: on
 
         run: python manage.py get_unmonitored_urls --base-url https://api.scorer.gitcoin.co/ --base-url-xyz https://api.passport.xyz/ ${{ inputs.uptime-robot-monitor-dry-run }}
 

api/registry/management/commands/get_unmonitored_urls_config.py

@@ -183,7 +183,7 @@ def get_config(base_url: str, base_url_xyz: str) -> dict:
                     "success_http_statues": [200],
                 },
                 ("GET", "/v2/stamps/{scorer_id}/score/{address}/history"): {
-                    "url": f"{base_url_xyz}v2/stamps/{REGISTRY_SCORER_ID}/score/{REGISTRY_ADDRESS}/history",
+                    "url": f"{base_url_xyz}v2/stamps/{REGISTRY_SCORER_ID}/score/{REGISTRY_ADDRESS}/history?created_at=2024-12-01",
                     "http_headers": {"X-API-Key": REGISTRY_API_KEY},
                     "success_http_statues": [200],
                 },

api/registry/utils.py

@@ -141,13 +141,24 @@ def wrapped(request, *args, **kwargs):
 
 
 def encode_cursor(**kwargs) -> str:
-    encoded_bytes = base64.urlsafe_b64encode(json.dumps(dict(**kwargs)).encode("utf-8"))
+    encoded_bytes = base64.urlsafe_b64encode(
+        json.dumps(dict(**kwargs)).encode("utf-8")
+        # Remove the "=" padding ...
+    ).rstrip(b"=")
     return encoded_bytes
 
 
+BASE64_PADDING = "===="
+
+
 def decode_cursor(token: str) -> dict:
     if token:
-        return json.loads(base64.urlsafe_b64decode(token).decode("utf-8"))
+        return json.loads(
+            base64.urlsafe_b64decode(
+                # We add back the "=" padding ...
+                token + BASE64_PADDING[: -(len(token) % 4)]
+            ).decode("utf-8")
+        )
     return {}
 
 

api/v2/api/__init__.py

@@ -2,6 +2,7 @@
 from django_ratelimit.exceptions import Ratelimited
 from ninja_extra import NinjaExtraAPI
 
+from ..exceptions import ScoreDoesNotExist
 from .api_models import *
 from .api_stamps import *
 from .router import api_router
@@ -26,3 +27,12 @@ def service_unavailable(request, _):
         {"detail": "You have been rate limited!"},
         status=429,
     )
+
+
+@api.exception_handler(ScoreDoesNotExist)
+def score_not_found(request, exc):
+    return api.create_response(
+        request,
+        {"detail": exc.detail, "address": exc.address},
+        status=exc.status_code
+    )

api/v2/api/api_stamps.py

@@ -1,4 +1,3 @@
-from datetime import datetime, time
 from decimal import Decimal
 from typing import Any, Dict, List
 from urllib.parse import urljoin
@@ -13,11 +12,9 @@
 from ceramic_cache.models import CeramicCache
 from registry.api.schema import (
     CursorPaginatedStampCredentialResponse,
-    DetailedScoreResponse,
     ErrorMessageResponse,
     NoScoreResponse,
     StampDisplayResponse,
-    SubmitPassportPayload,
 )
 from registry.api.utils import (
     ApiKey,
@@ -29,33 +26,95 @@
     with_read_db,
 )
 from registry.api.v1 import (
-    ahandle_submit_passport,
+    aget_scorer_by_id,
     fetch_all_stamp_metadata,
 )
+from registry.atasks import ascore_passport
 from registry.exceptions import (
     CreatedAtIsRequiredException,
-    CreatedAtMalFormedException,
     InternalServerErrorException,
     InvalidAddressException,
     InvalidAPIKeyPermissions,
     InvalidLimitException,
     api_get_object_or_404,
 )
-from registry.models import Event, Score
+from registry.models import Event, Passport, Score
 from registry.utils import (
     decode_cursor,
     encode_cursor,
     reverse_lazy_with_query,
 )
+from scorer_weighted.models import Scorer
 from v2.schema import V2ScoreResponse
 
+from ..exceptions import ScoreDoesNotExist
 from .router import api_router
 
 METADATA_URL = urljoin(settings.PASSPORT_PUBLIC_URL, "stampMetadata.json")
 
 log = logging.getLogger(__name__)
 
 
+async def handle_scoring(address: str, scorer_id: str, user_account):
+    address_lower = address.lower()
+
+    if not is_valid_address(address_lower):
+        raise InvalidAddressException()
+
+    # Get community object
+    user_community = await aget_scorer_by_id(scorer_id, user_account)
+
+    scorer = await user_community.aget_scorer()
+    scorer_type = scorer.type
+
+    # Create an empty passport instance, only needed to be able to create a pending Score
+    # The passport will be updated by the score_passport task
+    db_passport, _ = await Passport.objects.aupdate_or_create(
+        address=address_lower,
+        community=user_community,
+    )
+
+    score, _ = await Score.objects.select_related("passport").aget_or_create(
+        passport=db_passport,
+        defaults=dict(score=None, status=Score.Status.PROCESSING),
+    )
+
+    await ascore_passport(user_community, db_passport, address_lower, score)
+    await score.asave()
+
+    raw_score = 0
+    threshold = 20
+
+    if scorer_type == Scorer.Type.WEIGHTED:
+        raw_score = score.score
+    elif score.evidence and "rawScore":
+        raw_score = score.evidence.get("rawScore")
+        threshold = score.evidence.get("threshold")
+
+    if raw_score is None:
+        raw_score = 0
+
+    if threshold is None:
+        threshold = 0
+
+    return V2ScoreResponse(
+        address=address_lower,
+        score=raw_score,
+        passing_score=(Decimal(raw_score) >= Decimal(threshold)),
+        threshold=threshold,
+        last_score_timestamp=(
+            score.last_score_timestamp.isoformat()
+            if score.last_score_timestamp
+            else None
+        ),
+        expiration_timestamp=(
+            score.expiration_date.isoformat() if score.expiration_date else None
+        ),
+        error=score.error,
+        stamp_scores=score.stamp_scores if score.stamp_scores is not None else {},
+    )
+
+
 @api_router.get(
     "/stamps/{scorer_id}/score/{address}",
     auth=aapi_key,
@@ -74,35 +133,14 @@
 async def a_submit_passport(request, scorer_id: int, address: str) -> V2ScoreResponse:
     check_rate_limit(request)
     try:
-        if not request.api_key.submit_passports:
-            raise InvalidAPIKeyPermissions()
-
-        v1_score = await ahandle_submit_passport(
-            SubmitPassportPayload(address=address, scorer_id=str(scorer_id)),
-            request.auth,
-        )
-        threshold = v1_score.evidence.threshold if v1_score.evidence else "20"
-        score = v1_score.evidence.rawScore if v1_score.evidence else v1_score.score
-
-        return V2ScoreResponse(
-            address=v1_score.address,
-            score=score,
-            passing_score=(
-                Decimal(v1_score.score) >= Decimal(threshold)
-                if v1_score.score
-                else False
-            ),
-            threshold=threshold,
-            last_score_timestamp=v1_score.last_score_timestamp,
-            expiration_timestamp=v1_score.expiration_date,
-            error=v1_score.error,
-            stamp_scores=v1_score.stamp_scores,
-        )
+        return await handle_scoring(address, str(scorer_id), request.auth)
     except APIException as e:
         raise e
     except Exception as e:
         log.exception("Error submitting passport: %s", e)
-        raise InternalServerErrorException("Unexpected error while submitting passport")
+        raise InternalServerErrorException(
+            "Unexpected error while submitting passport"
+        ) from e
 
 
 def extract_score_data(event_data: Dict[str, Any]) -> Dict[str, Any]:
@@ -141,10 +179,10 @@ class Meta:
     "/stamps/{scorer_id}/score/{address}/history",
     auth=ApiKey(),
     response={
-        200: V2ScoreResponse | NoScoreResponse,
+        200: V2ScoreResponse,
         401: ErrorMessageResponse,
         400: ErrorMessageResponse,
-        404: ErrorMessageResponse,
+        404: ErrorMessageResponse | NoScoreResponse,
     },
     operation_id="v2_api_api_stamps_get_score_history",
     summary="Retrieve historical Stamp-based unique humanity score for a specified address",
@@ -180,9 +218,7 @@ def get_score_history(
         score_event = filterset.qs.order_by("-created_at").first()
 
         if not score_event:
-            return NoScoreResponse(
-                address=address, status=f"No Score Found for {address} at {created_at}"
-            )
+            raise ScoreDoesNotExist(address, f"No Score Found for {address} at {created_at}")
 
         # Extract and normalize score data from either format
         score_data = extract_score_data(score_event.data)

api/v2/aws_lambdas/stamp_score_GET.py

@@ -2,19 +2,14 @@
 This module provides a handler to manage API requests in AWS Lambda.
 """
 
-from decimal import Decimal
-
 from asgiref.sync import async_to_sync
 from django.db import close_old_connections
 
 from aws_lambdas.utils import (
     with_api_request_exception_handling,
 )
-from registry.api.v1 import (
-    SubmitPassportPayload,
-    ahandle_submit_passport,
-)
-from v2.schema import V2ScoreResponse
+
+from ..api.api_stamps import handle_scoring
 
 # Now this script or any imported module can use any part of Django it needs.
 # from myapp import models
@@ -28,29 +23,7 @@ def _handler(event, _context, request, user_account, body):
     split_url = event["path"].split("/")
     address = split_url[-1]
     scorer_id = split_url[-3]
-
-    v1_score = async_to_sync(ahandle_submit_passport)(
-        SubmitPassportPayload(
-            address=address,
-            scorer_id=scorer_id,
-        ),
-        user_account,
-    )
-
-    threshold = v1_score.evidence.threshold if v1_score.evidence else "20"
-
-    return V2ScoreResponse(
-        address=v1_score.address,
-        score=v1_score.score,
-        passing_score=(
-            Decimal(v1_score.score) >= Decimal(threshold) if v1_score.score else False
-        ),
-        threshold=threshold,
-        last_score_timestamp=v1_score.last_score_timestamp,
-        expiration_timestamp=v1_score.expiration_date,
-        error=v1_score.error,
-        stamp_scores=v1_score.stamp_scores,
-    )
+    return async_to_sync(handle_scoring)(address, scorer_id, user_account)
 
 
 def handler(*args, **kwargs):

api/v2/aws_lambdas/tests/test_stamp_score_get.py

@@ -109,9 +109,9 @@ def test_successful_authentication(
             body = json.loads(response["body"])
 
             assert body["address"] == address
-            assert body["score"] == "0"
+            assert body["score"] == "0.93300"
             assert body["passing_score"] == False
-            assert body["threshold"] == "20.0"
+            assert body["threshold"] == "20.00000"
 
             assert body["error"] is None
             assert body["stamp_scores"] == {"Ens": "0.408", "Google": "0.525"}
@@ -151,9 +151,9 @@ def test_successful_authentication_and_base64_encoded_body(
             body = json.loads(response["body"])
 
             assert body["address"] == address
-            assert body["score"] == "0"
+            assert body["score"] == "0.93300"
             assert body["passing_score"] == False
-            assert body["threshold"] == "20.0"
+            assert body["threshold"] == "20.00000"
             assert body["error"] is None
             assert body["stamp_scores"] == {"Ens": "0.408", "Google": "0.525"}
             # We just check that something != None was recorded for the last timestamp

api/v2/exceptions.py

@@ -0,0 +1,11 @@
+from ninja_extra import status
+from ninja_extra.exceptions import APIException
+
+
+class ScoreDoesNotExist(APIException):
+    status_code = status.HTTP_404_NOT_FOUND
+    default_detail = "No score exists."
+
+    def __init__(self, address: str, *args, **kwargs):
+        self.address = address
+        super().__init__(*args, **kwargs)

api/v2/schema.py

@@ -2,6 +2,7 @@
 from typing import Dict, Optional
 
 from ninja import Schema
+from pydantic import field_serializer
 
 
 class V2ScoreResponse(Schema):
@@ -13,3 +14,11 @@ class V2ScoreResponse(Schema):
     threshold: Decimal
     error: Optional[str]
     stamp_scores: Optional[Dict[str, Decimal]]
+
+    @field_serializer("score")
+    def serialize_score(self, score: Decimal, _info):
+        return format(score, ".5f")
+
+    @field_serializer("threshold")
+    def serialize_threshold(self, threshold: Decimal, _info):
+        return format(threshold, ".5f")