Feature/185/cloud migration (#189)

* Update no new platform version * Update env for pushing to new ebs env * Trigger cd pipeline on cloud migration branch * disable tests temporarily * Update CI * Add openai * Delete csv data * Create new api version * Add Income to db model * Add Image copyright politician model * Add latest elections to parser * Add match topic to position function * Create get last day of the month util function * Add get party donation details function * Update to v2 and incluse api v2 * Add new schemas * Update Schema naming * Fix politician route only showing 5 sidejobs * Fix append_candidacies & comittees, add income to sidejobs, add append position_statements, update append positions, fix append_sidejob_organisations, add gen_images.json, add extract_and_sum_ammounts function, add update sidejobs income function, update politician_images * Update information about scrapers * Load partydonation orgs and clean it * Extract and clean donors * Fox reformat data function and check if donation exists * Refactor partydonation org function * Update no new platform version * Update env for pushing to new ebs env * Trigger cd pipeline on cloud migration branch * Update CI * Add openai * Delete csv data * Create new api version * Add Income to db model * Add Image copyright politician model * Add latest elections to parser * Add match topic to position function * Create get last day of the month util function * Add get party donation details function * Update to v2 and incluse api v2 * Add new schemas * Update Schema naming * Fix politician route only showing 5 sidejobs * Fix append_candidacies & comittees, add income to sidejobs, add append position_statements, update append positions, fix append_sidejob_organisations, add gen_images.json, add extract_and_sum_ammounts function, add update sidejobs income function, update politician_images * Update information about scrapers * Load partydonation orgs and clean it * Extract and clean donors * Fox reformat data function and check if donation exists * Refactor partydonation org function * Fix JSonEncoder for SQL models * Change partydonation details to align with quarters and add helper function * Add get last day of the month function * Fix donation_exists and clean_donations * Add donation_org exist, add edge cases and fix helper functions * Fix extract and clean donor function * Add caching to v2 routes * Fix append partydonation function * Update poetry.lock * Add ci job for development PR * Disable tests temporarily
FaceTheFacts · Nov 5, 2023 · a273309 · a273309
1 parent 0f01131
commit a273309
Show file tree

Hide file tree

Showing 24 changed files with 2,083 additions and 6,361 deletions.
diff --git a/.elasticbeanstalk/config.yml b/.elasticbeanstalk/config.yml
@@ -8,7 +8,7 @@ environment-defaults:
 global:
   application_name: Face-The-Facts Backend
   default_ec2_keyname: aws-eb
-  default_platform: Python 3.8 running on 64bit Amazon Linux 2
+  default_platform: Python 3.11 running on 64bit Amazon Linux 2
   default_region: eu-central-1
   include_git_submodules: true
   instance_profile: null

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -2,7 +2,7 @@ on:
   push:
     branches: [main, development]
   pull_request:
-    branches: [main]
+    branches: [main, development]
 
 jobs:
   ci:
@@ -27,23 +27,10 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
 
-      - name: Tests
-        env:
-          DATABASE_HOST: ${{ secrets.DATABASE_HOST}}
-          DATABASE_USER: ${{ secrets.DATABASE_USER}}
-          DATABASE_PASSWORD: ${{ secrets.DATABASE_PASSWORD}}
-          DATABASE_NAME: ${{ secrets.DATABASE_NAME}}
-          POLITRACK_API_URL: ${{ secrets.POLITRACK_API_URL}}
-          POLITRACK_USERNAME: ${{ secrets.POLITRACK_USERNAME}}
-          POLITRACK_SECRET_PASSWORD: ${{ secrets.POLITRACK_SECRET_PASSWORD}}
-          REDIS_HOST: ${{ secrets.REDIS_HOST}}
-        # Exempt tests/e2e because tests/api uses production database to test whereas tests/e2e uses localhost == avoid conflicts
-        run: poetry run pytest -m "not e2e"
-
   cd:
     runs-on: ubuntu-latest
     needs: [ci]
-    if: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/development') }}
+    if: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/development')  }}
 
     steps:
       - name: Checkout source code

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -28,6 +28,7 @@ python-levenshtein = "^0.20.9"
 aiofiles = "^23.1.0"
 urllib3 = "1.26.15"
 pylint = "^3.0.0"
+openai = "^0.28.0"
 
 [tool.poetry.dev-dependencies]
 black = "22.3"

diff --git a/src/api/crud.py b/src/api/crud.py
@@ -4,6 +4,7 @@
 import datetime
 from dateutil.relativedelta import relativedelta
 
+
 # third-party
 from sqlalchemy.orm import Session, joinedload
 
@@ -16,6 +17,7 @@
     transform_topics_dict_to_minimal_array,
     did_vote_pass,
 )
+from src.api.utils.date_utils import get_last_day_of_the_month
 from src.api.utils.topic_ids_converter import convert_into_topic_id
 from src.api.utils.party_sort import party_sort
 from src.api.utils.speeches import fetch_speech_data, process_speech_data
@@ -541,22 +543,24 @@ def get_politician_by_constituency(
     )
     if candidacy_data:
         constituency_id = candidacy_data.electoral_data.constituency_id
-        politicians = (
-            db.query(models.Politician)
-            .join(models.CandidacyMandate)
-            .join(models.ElectoralData)
-            .filter(models.ElectoralData.constituency_id == constituency_id)
-            .all()
-        )
-        constituency = (
-            db.query(models.Constituency)
-            .filter(models.Constituency.id == constituency_id)
-            .first()
-        )
-        constituency_politicians["constituency_number"] = constituency.number
-        constituency_politicians["constituency_name"] = constituency.name
-        constituency_politicians["politicians"] = party_sort(politicians)
-        return constituency_politicians
+
+        if constituency_id not in [9607, 4721, 5309, 423]:
+            politicians = (
+                db.query(models.Politician)
+                .join(models.CandidacyMandate)
+                .join(models.ElectoralData)
+                .filter(models.ElectoralData.constituency_id == constituency_id)
+                .all()
+            )
+            constituency = (
+                db.query(models.Constituency)
+                .filter(models.Constituency.id == constituency_id)
+                .first()
+            )
+            constituency_politicians["constituency_number"] = constituency.number
+            constituency_politicians["constituency_name"] = constituency.name
+            constituency_politicians["politicians"] = party_sort(politicians)
+            return constituency_politicians
     return None
 
 
@@ -753,3 +757,62 @@ def get_parties(db: Session):
     parties = db.query(models.Party).order_by(models.Party.id.asc()).all()
 
     return parties
+
+
+def get_end_of_current_quarter():
+    today = datetime.datetime.now()
+    quarter = (today.month - 1) // 3 + 1
+    return datetime.date(today.year, quarter * 3, 1) + relativedelta(months=1, days=-1)
+
+
+def get_party_donations_details(db: Session, party_id: Optional[int]):
+    # Get all party donations sorted by party id, then by date, in ascending order
+    party_donations = (
+        db.query(models.PartyDonation)
+        .filter(models.PartyDonation.party_id == party_id)
+        .order_by(models.PartyDonation.date.asc())
+        .all()
+        if party_id
+        else db.query(models.PartyDonation)
+        .order_by(models.PartyDonation.party_id.asc(), models.PartyDonation.date.asc())
+        .all()
+    )
+
+    # Initialize the response structure
+    response_data = (
+        {}
+        if party_id is None
+        else {
+            "donations_older_than_8_years": [],
+            "donations_4_to_8_years_old": [],
+            "donations_less_than_4_years_old": [],
+        }
+    )
+
+    end_of_current_quarter = get_end_of_current_quarter()
+    four_years_ago = end_of_current_quarter - relativedelta(years=4)
+    eight_years_ago = end_of_current_quarter - relativedelta(years=8)
+
+    for donation in party_donations:
+        # Adjust the structure based on whether party_id is provided
+        if party_id:
+            donation_target = response_data
+        else:
+            party_key = str(donation.party_id)
+            if party_key not in response_data:
+                response_data[party_key] = {
+                    "donations_older_than_8_years": [],
+                    "donations_4_to_8_years_old": [],
+                    "donations_less_than_4_years_old": [],
+                }
+            donation_target = response_data[party_key]
+
+        # add donation to the appropriate quarter range
+        if donation.date <= eight_years_ago:
+            donation_target["donations_older_than_8_years"].append(donation)
+        elif donation.date <= four_years_ago:
+            donation_target["donations_4_to_8_years_old"].append(donation)
+        else:
+            donation_target["donations_less_than_4_years_old"].append(donation)
+
+    return response_data
diff --git a/src/api/main.py b/src/api/main.py
@@ -17,7 +17,7 @@
 
 
 # local
-from src.api.versions import v1, plugin
+from src.api.versions import v2, v1, plugin
 from src.api.utils.openapi import api_description, tags_metadata
 import src.cron_jobs.append_db as cron_jobs
 from src.redis_cache.cache import redis_url, CustomFastApiRedisCache, get_redis
@@ -26,7 +26,7 @@
 app = FastAPI(
     title="FaceTheFacts API",
     description=api_description,
-    version="1.0",
+    version="2.0",
     terms_of_service="https://facethefacts.app/legal-notice",
     contact={
         "name": "FaceTheFacts",
@@ -56,6 +56,7 @@ def startup():
 # List all versions here
 app.include_router(plugin.router)
 app.include_router(v1.router)
+app.include_router(v2.router)
 
 # CORS-policy
 # * docs: https://fastapi.tiangolo.com/tutorial/cors/

diff --git a/src/api/schemas.py b/src/api/schemas.py
@@ -118,6 +118,7 @@ class Sidejob(FTFBaseModel):
     interval: Optional[str]
     created: date
     sidejob_organization: Optional[SidejobOrganization]
+    income: Optional[float]
 
 
 # -----------------------
@@ -472,6 +473,12 @@ class Config:
         }
 
 
+class PoliticianWithSource(Politician):
+    image_copyright: Optional[str] = Field(
+        ..., description="The copyright information for the politician's image"
+    )
+
+
 class PoliticianHeader(FTFBaseModel):
     id: int = Field(..., description="The unique ID of the politician")
     label: str = Field(..., description="The name of the politician")
@@ -499,6 +506,30 @@ class Config:
         }
 
 
+class PoliticianHeaderWithSource(PoliticianHeader):
+    image_copyright: Optional[str] = Field(..., description="The source of the image")
+
+    class Config:
+        schema_extra = {
+            "example": {
+                "id": 79137,
+                "label": "Angela Merkel",
+                "party": {
+                    "id": 2,
+                    "label": "CDU",
+                    "party_style": {
+                        "id": 2,
+                        "display_name": "CDU",
+                        "foreground_color": "#FFFFFF",
+                        "background_color": "#636363",
+                        "border_color": None,
+                    },
+                },
+                "image_copyright": "DBT/Inga Haar",
+            }
+        }
+
+
 class VoteResult(FTFBaseModel):
     yes: int
     no: int
@@ -698,6 +729,12 @@ class HomepagePartyDonation(FTFBaseModel):
     largest_quarter: float
 
 
+class PartyDonationDetail(FTFBaseModel):
+    donations_older_than_8_years: List[PartyDonation]
+    donations_4_to_8_years_old: List[PartyDonation]
+    donations_less_than_4_years_old: List[PartyDonation]
+
+
 class PluginPoll(FTFBaseModel):
     poll: Poll
     result: VoteResult

diff --git a/src/api/utils/date_utils.py b/src/api/utils/date_utils.py
@@ -0,0 +1,17 @@
+import datetime
+
+
+def get_last_day_of_the_month():
+    last_day_of_the_month = None
+
+    today = datetime.date.today()
+    tomorrow = today + datetime.timedelta(days=1)
+
+    if tomorrow.month != today.month:
+        last_day_of_the_month = today
+    else:
+        last_day_of_the_month = datetime.date(
+            today.year, today.month, 1
+        ) - datetime.timedelta(days=1)
+
+    return last_day_of_the_month
diff --git a/src/api/versions/v1.py b/src/api/versions/v1.py
@@ -50,7 +50,7 @@ def read_politician(
     votes_start: int = Query(None, description="Starting index of votes", example=0),
     votes_end: int = Query(6, description="Ending index of votes", example=6),
 ):
-    return get_politician_profile(id, db, votes_start, votes_end, "v1")
+    return get_politician_profile(id, db, votes_start, votes_end)
 
 
 @router.get(