Merge pull request #670 from biigle/vector-db

Implement separate vector database

.docker/pgvector.dockerfile

@@ -0,0 +1,13 @@
+FROM postgres:14-alpine
+
+RUN apk add --no-cache --virtual .build-deps \
+        git \
+        build-base \
+        clang \
+        llvm13-dev \
+    && git clone --branch v0.5.0 https://github.com/pgvector/pgvector.git /tmp/pgvector \
+    && cd /tmp/pgvector \
+    && make OPTFLAGS="" \
+    && make install \
+    && rm -r /tmp/pgvector \
+    && apk del --purge .build-deps

.env.example

@@ -21,6 +21,12 @@ DB_DATABASE="biigle"
 DB_USERNAME="biigle"
 DB_PASSWORD="secret"
 
+VECTOR_DB_HOST="vector_database"
+VECTOR_DB_PORT=5432
+VECTOR_DB_DATABASE="biigle"
+VECTOR_DB_USERNAME="biigle"
+VECTOR_DB_PASSWORD="secret"
+
 VOLUME_ADMIN_STORAGE_DISKS=local
 
 BROADCAST_DRIVER=pusher

.github/workflows/release.yml

@@ -126,3 +126,41 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
+
+  build-and-push-pgvector:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/[email protected]
+
+      - name: Log in to the Container registry
+        uses: docker/[email protected]
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/[email protected]
+        with:
+          images: ${{ env.REGISTRY }}/biigle/pgvector
+
+      - name: Build and push Docker image
+        uses: docker/[email protected]
+        with:
+          context: .
+          file: .docker/pgvector.dockerfile
+          build-args: BIIGLE_VERSION=${{ github.event.release.tag_name }}
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/test.yml

@@ -32,9 +32,12 @@ jobs:
       run: |
         docker pull ghcr.io/biigle/app:latest
         docker pull ghcr.io/biigle/worker:latest
+        docker pull ghcr.io/biigle/pgvector:latest
 
     - name: Start test database
-      run: docker-compose up -d --no-build database_testing
+      run: |
+        docker-compose up -d --no-build database_testing
+        docker-compose up -d --no-build vector_database_testing
 
     - name: Run tests
       run: docker-compose run --rm -u 1001 worker php -d memory_limit=1G vendor/bin/phpunit --random-order
@@ -71,9 +74,12 @@ jobs:
       run: |
         docker pull ghcr.io/biigle/app:latest
         docker pull ghcr.io/biigle/worker:latest
+        docker pull ghcr.io/biigle/pgvector:latest
 
     - name: Start test database
-      run: docker-compose up -d --no-build database_testing
+      run: |
+        docker-compose up -d --no-build database_testing
+        docker-compose up -d --no-build vector_database_testing
 
     - name: Run tests
       run: docker-compose run --rm -u 1001 worker php -d memory_limit=1G vendor/bin/phpunit --random-order

composer.json

@@ -23,6 +23,7 @@
         "ext-soap": "*",
         "ext-vips": "*",
         "ext-yaml": "*",
+        "ankane/pgvector": "^0.1.3",
         "biigle/laravel-file-cache": "^4.0",
         "doctrine/dbal": "^3.0",
         "duncan3dc/bom-string": "^1.1",

config/database.php

@@ -78,6 +78,21 @@
             'sslmode' => 'prefer',
         ],
 
+        'pgvector' => [
+            'driver' => 'pgsql',
+            'url' => env('VECTOR_DATABASE_URL'),
+            'host' => env('VECTOR_DB_HOST', 'localhost'),
+            'port' => env('VECTOR_DB_PORT', '5432'),
+            'database' => env('VECTOR_DB_DATABASE', 'forge'),
+            'username' => env('VECTOR_DB_USERNAME', 'forge'),
+            'password' => env('VECTOR_DB_PASSWORD', ''),
+            'charset' => 'utf8',
+            'prefix' => '',
+            'prefix_indexes' => true,
+            'search_path' => 'public',
+            'sslmode' => 'prefer',
+        ],
+
         'sqlsrv' => [
             'driver' => 'sqlsrv',
             'url' => env('DATABASE_URL'),

database/migrations/2022_08_03_000000_create_vector_extension.php

@@ -0,0 +1,26 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Support\Facades\DB;
+
+return new class extends Migration {
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        DB::connection('pgvector')->statement('CREATE EXTENSION IF NOT EXISTS vector');
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        DB::connection('pgvector')->statement('DROP EXTENSION IF EXISTS vector');
+    }
+};

docker-compose.yml

@@ -22,6 +22,7 @@ services:
     depends_on:
       - app
       - database_testing
+      - vector_database_testing
     build:
       context: ./
       dockerfile: .docker/worker.dockerfile
@@ -76,6 +77,21 @@ services:
     ports:
       - "54320:5432"
 
+  vector_database:
+    image: ghcr.io/biigle/pgvector
+    build:
+      context: ./
+      dockerfile: .docker/pgvector.dockerfile
+    volumes:
+      - vecdbdata:/var/lib/postgresql/data
+      - ./:/data
+    environment:
+      - "POSTGRES_DB=biigle"
+      - "POSTGRES_USER=biigle"
+      - "POSTGRES_PASSWORD=secret"
+    ports:
+      - "54321:5432"
+
   database_testing:
     image: postgres:14-alpine
     tmpfs:
@@ -85,6 +101,16 @@ services:
       - "POSTGRES_USER=biigle"
       - "POSTGRES_PASSWORD=secret"
 
+  vector_database_testing:
+    image: ghcr.io/biigle/pgvector
+    tmpfs:
+      - /var/lib/postgresql/data
+    environment:
+      - "POSTGRES_DB=biigle"
+      - "POSTGRES_USER=biigle"
+      - "POSTGRES_PASSWORD=secret"
+
 volumes:
   dbdata:
+  vecdbdata:
 

phpunit.xml

@@ -25,9 +25,13 @@
         <env name="TELESCOPE_ENABLED" value="false"/>
         <env name="QUEUE_CONNECTION" value="sync"/>
         <env name="DB_CONNECTION" value="pgsql"/>
-        <env name="DB_DATABASE" value="biigle"/>
         <env name="DB_HOST" value="database_testing"/>
+        <env name="DB_DATABASE" value="biigle"/>
         <env name="DB_USERNAME" value="biigle"/>
         <env name="DB_PASSWORD" value="secret"/>
+        <env name="VECTOR_DB_HOST" value="vector_database_testing"/>
+        <env name="VECTOR_DB_DATABASE" value="biigle"/>
+        <env name="VECTOR_DB_USERNAME" value="biigle"/>
+        <env name="VECTOR_DB_PASSWORD" value="secret"/>
     </php>
 </phpunit>