Improve db setup cache pool and link insertions (#57)

* added redis image to docker-compose * added connection pool to Redis instance usage * added __await__ hanlding for Database class, allowing better annotation of .create method & current functionality of await Database.create to continue triggering migrations * improved link insertions & reducing logging noise from duplicate attempts to insert into link tables * corrected return type & missing pk definition * fixed raising Exceptoin on non-link related insertion exceptions * appended max connections for postgres container * added assertions to ensure raise error when pytest fails within in sequence call * appended max connections for postgres container * appended max connections for postgres container * appended max connections for postgres container * appended max connections for postgres container * appended max connections for postgres container * appended max connections for postgres container * appended max connections for postgres container * updated max connections in docker-compose * replaced services definitions of containers with docker-compose usage * ensure rc is 0 when running test migrations in sequence --------- Co-authored-by: Joshua (codemation) <[email protected]>
codemation · Apr 15, 2023 · 8cc1392 · 8cc1392
1 parent 6b7d19e
commit 8cc1392
Show file tree

Hide file tree

Showing 8 changed files with 90 additions and 137 deletions.
diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
@@ -27,38 +27,16 @@ jobs:
       matrix:
         python-version: [3.7, 3.8, 3.9, 3.10.8]
 
-    services:
-      # Label used to access the service container
-      redis:
-        # Docker Hub image
-        image: redis
-        # Provide the password for postgres
-        ports:
-          - 6379:6379
-      postgres:
-        # Docker Hub image
-        image: postgres
-        # Provide the password for postgres
-        env:
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_USER: postgres
-          POSTGRES_DB: database
-        ports:
-          - 5432:5432
-
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
     steps:
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
       - name: Check out repository code
         uses: actions/checkout@v2
+      - name: start postgres & redis
+        run: |
+          docker-compose up -d postgres redis
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -77,26 +55,7 @@ jobs:
     strategy:
       matrix:
         python-version: [3.7, 3.8, 3.9]
-    # Service containers to run with `container-job`
-    services:
-      # Label used to access the service container
-      redis:
-        # Docker Hub image
-        image: redis
-        # Provide the password for postgres
-        ports:
-          - 6379:6379
-      mysql:
-        # Docker Hub image
-        image: mysql
-        # Provide the password for postgres
-        env:
-          MYSQL_USER: mysqltestuser
-          MYSQL_PASSWORD: abcd1234
-          MYSQL_ROOT_PASSWORD: abcd1234
-          MYSQL_DATABASE: database
-        ports:
-          - 3306:3306
+
     steps:
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v2
@@ -105,6 +64,9 @@ jobs:
       # Downloads a copy of the code in your repository before running CI tests
       - name: Check out repository code
         uses: actions/checkout@v2
+      - name: start mysql & redis
+        run: |
+          docker-compose up -d mysql redis
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

diff --git a/.github/workflows/package.yaml b/.github/workflows/package.yaml
@@ -12,26 +12,6 @@ jobs:
     strategy:
       matrix:
         python-version: [3.7, 3.8, 3.9]
-    # Service containers to run with `container-job`
-    services:
-      # Label used to access the service container
-      redis:
-        # Docker Hub image
-        image: redis
-        # Provide the password for postgres
-        ports:
-          - 6379:6379
-      mysql:
-        # Docker Hub image
-        image: mysql
-        # Provide the password for postgres
-        env:
-          MYSQL_USER: mysqltestuser
-          MYSQL_PASSWORD: abcd1234
-          MYSQL_ROOT_PASSWORD: abcd1234
-          MYSQL_DATABASE: database
-        ports:
-          - 3306:3306
     steps:
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v2
@@ -40,6 +20,9 @@ jobs:
       # Downloads a copy of the code in your repository before running CI tests
       - name: Check out repository code
         uses: actions/checkout@v2
+      - name: start mysql & redis
+        run: |
+          docker-compose up -d mysql redis
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -96,38 +79,16 @@ jobs:
       matrix:
         python-version: [3.7, 3.8, 3.9, 3.10.8]
 
-    services:
-      # Label used to access the service container
-      redis:
-        # Docker Hub image
-        image: redis
-        # Provide the password for postgres
-        ports:
-          - 6379:6379
-      postgres:
-        # Docker Hub image
-        image: postgres
-        # Provide the password for postgres
-        env:
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_USER: postgres
-          POSTGRES_DB: database
-        ports:
-          - 5432:5432
-
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
     steps:
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
       - name: Check out repository code
         uses: actions/checkout@v2
+      - name: start postgres & redis
+        run: |
+          docker-compose up -d postgres redis
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -9,7 +9,7 @@ services:
       POSTGRES_USER: postgres
       POSTGRES_PASSWORD: postgres
       POSTGRES_DB: database
-    command: postgres -c fsync=no -c full_page_writes=no
+    command: postgres -c fsync=no -c full_page_writes=no -c 'max_connections=250'
     ports:
       - "5432:5432"
 
@@ -24,3 +24,7 @@ services:
       MYSQL_DATABASE: database
     ports:
       - "3306:3306"
+  redis:
+    image: redis
+    ports:
+      - "6379:6379"
diff --git a/pydbantic/cache.py b/pydbantic/cache.py
@@ -13,7 +13,8 @@ def __init__(
         redis_url: str = "redis://localhost",
         log: logging.Logger = logging.getLogger(__name__),
     ):
-        self.redis = aioredis.from_url(redis_url)
+        self.pool = aioredis.ConnectionPool.from_url(redis_url)
+        self.redis = aioredis.Redis(connection_pool=self.pool)
         self.log = log
 
     async def get(self, key):