From 72fdbd4d1338f003666b7691a7c045726c3e7e45 Mon Sep 17 00:00:00 2001 From: Martin Zurowietz Date: Tue, 12 Dec 2023 15:26:53 +0100 Subject: [PATCH 1/3] Move feature vector models to default database connection --- src/AnnotationCandidateFeatureVector.php | 7 --- ...nnotationCandidateFeatureVectorFactory.php | 3 +- .../TrainingProposalFeatureVectorFactory.php | 3 +- ...4_164500_create_feature_vectors_tables.php | 29 ++-------- ...2_150900_create_feature_vectors_tables.php | 53 +++++++++++++++++++ src/TrainingProposalFeatureVector.php | 7 --- .../AnnotationCandidateFeatureVectorTest.php | 8 +++ tests/TrainingProposalFeatureVectorTest.php | 8 +++ 8 files changed, 77 insertions(+), 41 deletions(-) create mode 100644 src/Database/migrations/2023_12_12_150900_create_feature_vectors_tables.php diff --git a/src/AnnotationCandidateFeatureVector.php b/src/AnnotationCandidateFeatureVector.php index 2c035ac..e10c92f 100644 --- a/src/AnnotationCandidateFeatureVector.php +++ b/src/AnnotationCandidateFeatureVector.php @@ -12,13 +12,6 @@ class AnnotationCandidateFeatureVector extends Model { use HasFactory, HasNeighbors; - /** - * The database connection associated with the model. - * - * @var string - */ - protected $connection = 'pgvector'; - /** * The table associated with the model. * diff --git a/src/Database/Factories/AnnotationCandidateFeatureVectorFactory.php b/src/Database/Factories/AnnotationCandidateFeatureVectorFactory.php index 201fe67..36e518e 100644 --- a/src/Database/Factories/AnnotationCandidateFeatureVectorFactory.php +++ b/src/Database/Factories/AnnotationCandidateFeatureVectorFactory.php @@ -3,6 +3,7 @@ namespace Biigle\Modules\Maia\Database\Factories; use Biigle\Modules\Maia\MaiaJob; +use Biigle\Modules\Maia\AnnotationCandidate; use Biigle\Modules\Maia\AnnotationCandidateFeatureVector; use Illuminate\Database\Eloquent\Factories\Factory; @@ -23,7 +24,7 @@ class AnnotationCandidateFeatureVectorFactory extends Factory public function definition() { return [ - 'id' => $this->faker->unique()->randomDigit(), + 'id' => AnnotationCandidate::factory(), 'job_id' => MaiaJob::factory(), 'vector' => range(0, 383), ]; diff --git a/src/Database/Factories/TrainingProposalFeatureVectorFactory.php b/src/Database/Factories/TrainingProposalFeatureVectorFactory.php index 7b09073..5aed2ac 100644 --- a/src/Database/Factories/TrainingProposalFeatureVectorFactory.php +++ b/src/Database/Factories/TrainingProposalFeatureVectorFactory.php @@ -3,6 +3,7 @@ namespace Biigle\Modules\Maia\Database\Factories; use Biigle\Modules\Maia\MaiaJob; +use Biigle\Modules\Maia\TrainingProposal; use Biigle\Modules\Maia\TrainingProposalFeatureVector; use Illuminate\Database\Eloquent\Factories\Factory; @@ -23,7 +24,7 @@ class TrainingProposalFeatureVectorFactory extends Factory public function definition() { return [ - 'id' => $this->faker->unique()->randomDigit(), + 'id' => TrainingProposal::factory(), 'job_id' => MaiaJob::factory(), 'vector' => range(0, 383), ]; diff --git a/src/Database/migrations/2023_10_04_164500_create_feature_vectors_tables.php b/src/Database/migrations/2023_10_04_164500_create_feature_vectors_tables.php index 3a20974..d49a1e4 100644 --- a/src/Database/migrations/2023_10_04_164500_create_feature_vectors_tables.php +++ b/src/Database/migrations/2023_10_04_164500_create_feature_vectors_tables.php @@ -13,27 +13,9 @@ */ public function up() { - Schema::connection('pgvector') - ->create('maia_training_proposal_feature_vectors', function (Blueprint $table) { - // Don't use increments() because it should throw an error if this is not - // manually provided. It should be the ID of the model in the main DB. - $table->unsignedInteger('id'); - $table->unsignedInteger('job_id')->index(); - $table->vector('vector', 384); - - $table->primary('id'); - }); - - Schema::connection('pgvector') - ->create('maia_annotation_candidate_feature_vectors', function (Blueprint $table) { - // Don't use increments() because it should throw an error if this is not - // manually provided. It should be the ID of the model in the main DB. - $table->unsignedInteger('id'); - $table->unsignedInteger('job_id')->index(); - $table->vector('vector', 384); - - $table->primary('id'); - }); + // This migration set up tables in a separate pgvector database. The database was + // removed later. The migration was cleared so data in the separate database + // cannot be accidentally deleted. } /** @@ -43,9 +25,6 @@ public function up() */ public function down() { - Schema::connection('pgvector') - ->dropIfExists('maia_training_proposal_feature_vectors'); - Schema::connection('pgvector') - ->dropIfExists('maia_annotation_candidate_feature_vectors'); + // } }; diff --git a/src/Database/migrations/2023_12_12_150900_create_feature_vectors_tables.php b/src/Database/migrations/2023_12_12_150900_create_feature_vectors_tables.php new file mode 100644 index 0000000..66027e2 --- /dev/null +++ b/src/Database/migrations/2023_12_12_150900_create_feature_vectors_tables.php @@ -0,0 +1,53 @@ +unsignedBigInteger('id'); + $table->foreign('id') + ->references('id') + ->on('maia_training_proposals') + ->onDelete('cascade'); + + $table->unsignedInteger('job_id')->index(); + $table->vector('vector', 384); + + $table->primary('id'); + }); + + Schema::create('maia_annotation_candidate_feature_vectors', function (Blueprint $table) { + $table->unsignedBigInteger('id'); + $table->foreign('id') + ->references('id') + ->on('maia_annotation_candidates') + ->onDelete('cascade'); + + $table->unsignedInteger('job_id')->index(); + $table->vector('vector', 384); + + $table->primary('id'); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::dropIfExists('maia_training_proposal_feature_vectors'); + Schema::dropIfExists('maia_annotation_candidate_feature_vectors'); + } +}; diff --git a/src/TrainingProposalFeatureVector.php b/src/TrainingProposalFeatureVector.php index 373b4ee..62e7a85 100644 --- a/src/TrainingProposalFeatureVector.php +++ b/src/TrainingProposalFeatureVector.php @@ -12,13 +12,6 @@ class TrainingProposalFeatureVector extends Model { use HasFactory, HasNeighbors; - /** - * The database connection associated with the model. - * - * @var string - */ - protected $connection = 'pgvector'; - /** * The table associated with the model. * diff --git a/tests/AnnotationCandidateFeatureVectorTest.php b/tests/AnnotationCandidateFeatureVectorTest.php index f93e206..68be5cc 100644 --- a/tests/AnnotationCandidateFeatureVectorTest.php +++ b/tests/AnnotationCandidateFeatureVectorTest.php @@ -2,6 +2,7 @@ namespace Biigle\Tests\Modules\Maia; +use Biigle\Modules\Maia\AnnotationCandidate; use Biigle\Modules\Maia\AnnotationCandidateFeatureVector; use Biigle\Shape; use TestCase; @@ -14,4 +15,11 @@ public function testAttributes() $this->assertNotNull($model->job_id); $this->assertNotNull($model->vector); } + + public function testCascadeDelete() + { + $model = AnnotationCandidateFeatureVector::factory()->create(); + AnnotationCandidate::find($model->id)->delete(); + $this->assertNull($model->fresh()); + } } diff --git a/tests/TrainingProposalFeatureVectorTest.php b/tests/TrainingProposalFeatureVectorTest.php index 6af42f1..c00129f 100644 --- a/tests/TrainingProposalFeatureVectorTest.php +++ b/tests/TrainingProposalFeatureVectorTest.php @@ -2,6 +2,7 @@ namespace Biigle\Tests\Modules\Maia; +use Biigle\Modules\Maia\TrainingProposal; use Biigle\Modules\Maia\TrainingProposalFeatureVector; use Biigle\Shape; use TestCase; @@ -14,4 +15,11 @@ public function testAttributes() $this->assertNotNull($model->job_id); $this->assertNotNull($model->vector); } + + public function testCascadeDelete() + { + $model = TrainingProposalFeatureVector::factory()->create(); + TrainingProposal::find($model->id)->delete(); + $this->assertNull($model->fresh()); + } } From 50699e0428cc943717a7ea631ff575c1bc9d4029 Mon Sep 17 00:00:00 2001 From: Martin Zurowietz Date: Wed, 13 Dec 2023 11:53:42 +0100 Subject: [PATCH 2/3] Add job foreign key constraint to feature vector tables --- ...2_150900_create_feature_vectors_tables.php | 14 ++++++--- .../DeleteAnnotationFeatureVectors.php | 22 -------------- src/MaiaServiceProvider.php | 2 -- .../DeleteAnnotationFeatureVectorsTest.php | 29 ------------------- tests/MaiaJobTest.php | 16 ++++++++++ 5 files changed, 26 insertions(+), 57 deletions(-) delete mode 100644 src/Listeners/DeleteAnnotationFeatureVectors.php delete mode 100644 tests/Listeners/DeleteAnnotationFeatureVectorsTest.php diff --git a/src/Database/migrations/2023_12_12_150900_create_feature_vectors_tables.php b/src/Database/migrations/2023_12_12_150900_create_feature_vectors_tables.php index 66027e2..e91851c 100644 --- a/src/Database/migrations/2023_12_12_150900_create_feature_vectors_tables.php +++ b/src/Database/migrations/2023_12_12_150900_create_feature_vectors_tables.php @@ -21,9 +21,12 @@ public function up() ->onDelete('cascade'); $table->unsignedInteger('job_id')->index(); - $table->vector('vector', 384); + $table->foreign('job_id') + ->references('id') + ->on('maia_jobs') + ->onDelete('cascade'); - $table->primary('id'); + $table->vector('vector', 384); }); Schema::create('maia_annotation_candidate_feature_vectors', function (Blueprint $table) { @@ -34,9 +37,12 @@ public function up() ->onDelete('cascade'); $table->unsignedInteger('job_id')->index(); - $table->vector('vector', 384); + $table->foreign('job_id') + ->references('id') + ->on('maia_jobs') + ->onDelete('cascade'); - $table->primary('id'); + $table->vector('vector', 384); }); } diff --git a/src/Listeners/DeleteAnnotationFeatureVectors.php b/src/Listeners/DeleteAnnotationFeatureVectors.php deleted file mode 100644 index b2e6526..0000000 --- a/src/Listeners/DeleteAnnotationFeatureVectors.php +++ /dev/null @@ -1,22 +0,0 @@ -job->id)->delete(); - AnnotationCandidateFeatureVector::where('job_id', $event->job->id)->delete(); - } -} diff --git a/src/MaiaServiceProvider.php b/src/MaiaServiceProvider.php index 9bbdb9e..511227a 100644 --- a/src/MaiaServiceProvider.php +++ b/src/MaiaServiceProvider.php @@ -6,7 +6,6 @@ use Biigle\Modules\Maia\Events\MaiaJobContinued; use Biigle\Modules\Maia\Events\MaiaJobCreated; use Biigle\Modules\Maia\Events\MaiaJobDeleting; -use Biigle\Modules\Maia\Listeners\DeleteAnnotationFeatureVectors; use Biigle\Modules\Maia\Listeners\DispatchMaiaJob; use Biigle\Modules\Maia\Listeners\DispatchObjectDetection; use Biigle\Modules\Maia\Listeners\PrepareDeleteAnnotationPatches; @@ -66,7 +65,6 @@ public function boot(Modules $modules, Router $router) Event::listen(MaiaJobContinued::class, PruneTrainingProposalPatches::class); Event::listen(MaiaJobContinued::class, PruneTrainingProposalFeatureVectors::class); Event::listen(MaiaJobDeleting::class, PrepareDeleteAnnotationPatches::class); - Event::listen(MaiaJobDeleting::class, DeleteAnnotationFeatureVectors::class); } /** diff --git a/tests/Listeners/DeleteAnnotationFeatureVectorsTest.php b/tests/Listeners/DeleteAnnotationFeatureVectorsTest.php deleted file mode 100644 index bcc8ad0..0000000 --- a/tests/Listeners/DeleteAnnotationFeatureVectorsTest.php +++ /dev/null @@ -1,29 +0,0 @@ -create(); - $tp = TrainingProposalFeatureVector::factory()->create([ - 'job_id' => $job->id, - ]); - $ac = AnnotationCandidateFeatureVector::factory()->create([ - 'job_id' => $job->id, - ]); - - $event = new MaiaJobDeleting($job); - (new DeleteAnnotationFeatureVectors)->handle($event); - $this->assertNull($tp->fresh()); - $this->assertNull($ac->fresh()); - } -} diff --git a/tests/MaiaJobTest.php b/tests/MaiaJobTest.php index 5e2e0b5..c3b77d9 100644 --- a/tests/MaiaJobTest.php +++ b/tests/MaiaJobTest.php @@ -2,6 +2,8 @@ namespace Biigle\Tests\Modules\Maia; +use Biigle\Modules\Maia\AnnotationCandidateFeatureVector; +use Biigle\Modules\Maia\TrainingProposalFeatureVector; use Biigle\Modules\Maia\Events\MaiaJobCreated; use Biigle\Modules\Maia\Events\MaiaJobDeleting; use Biigle\Modules\Maia\MaiaJob; @@ -144,4 +146,18 @@ public function testShouldShowTrainingProposals() ]; $this->assertTrue($this->model->shouldShowTrainingProposals()); } + + public function testDeleteFeatureVectorsCascade() + { + $tp = TrainingProposalFeatureVector::factory()->create([ + 'job_id' => $this->model->id, + ]); + $ac = AnnotationCandidateFeatureVector::factory()->create([ + 'job_id' => $this->model->id, + ]); + + $this->model->delete(); + $this->assertNull($tp->fresh()); + $this->assertNull($ac->fresh()); + } } From 3eb56de399d19b64b624cb8172bc95c62a2d7dc7 Mon Sep 17 00:00:00 2001 From: Martin Zurowietz Date: Wed, 13 Dec 2023 11:59:16 +0100 Subject: [PATCH 3/3] Remove vector database from test script --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cbf233e..88c6d68 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -57,7 +57,7 @@ jobs: docker pull ghcr.io/biigle/worker:latest - name: Start test database - run: docker-compose up -d --no-build database_testing vector_database_testing && sleep 5 + run: docker-compose up -d --no-build database_testing && sleep 5 working-directory: ../core - name: Run tests